diff --git a/src/tools/embeddings.ts b/src/tools/embeddings.ts index b9dbae5..5be37cc 100644 --- a/src/tools/embeddings.ts +++ b/src/tools/embeddings.ts @@ -89,7 +89,7 @@ async function getBatchEmbeddingsWithRetry( const key = Object.keys(item)[0]; return key === 'text' ? { text: trimSymbols(item[key]) } : item; } - }); // Copy the original texts + }).filter(item => typeof item !== 'string' || item.trim()); // Copy the original texts let indexMap = new Map(); // Map to keep track of original indices // Initialize indexMap with original indices @@ -251,7 +251,7 @@ function truncateInputString(input: string | Record): string { } } -function trimSymbols(str: string): string { +export function trimSymbols(str: string): string { const regex = /[\p{S}\p{P}\p{Z}\p{C}\p{Emoji}]+/gu; return str.replace(regex, ' '); } \ No newline at end of file