feat: late chunking

src/app.ts

@@ -500,6 +500,7 @@ app.post('/v1/chat/completions', (async (req: Request, res: Response) => {
     const actionListener = async (step: StepAction) => {
       // Add content to queue for both thinking steps and final answer
       if (step.action === 'visit') {
+        // emit every url in the visit action in url field
         (step as VisitAction).URLTargets.forEach((url) => {
           const chunk: ChatCompletionChunk = {
           id: requestId,

src/tools/jina-latechunk.ts

@@ -3,7 +3,7 @@ import axios from 'axios';
 import {JINA_API_KEY} from "../config";
 import {Schemas} from "../utils/schemas";
 
-export async function cherryPick(question: string, longContext: string, options: any = {}, trackers: TrackerContext, schemaGen: Schemas) {
+export async function cherryPick(question: string, longContext: string, options: any = {}, trackers: TrackerContext, schemaGen: Schemas, url: string) {
 
   const {
     snippetLength = 3000,
@@ -27,7 +27,7 @@ export async function cherryPick(question: string, longContext: string, options:
 
   console.log('late chunking enabled! num chunks:', chunks.length);
 
-  trackers.actionTracker.trackThink('late_chunk', schemaGen.languageCode);
+  trackers.actionTracker.trackThink('late_chunk', schemaGen.languageCode, {url});
 
   try {
     // Estimate the number of tokens per chunk
@@ -61,7 +61,8 @@ export async function cherryPick(question: string, longContext: string, options:
           late_chunking: true,
           dimensions: 1024,
           embedding_type: "float",
-          input: batch
+          input: batch,
+          truncate: true
         },
         {
           headers: {
@@ -97,7 +98,8 @@ export async function cherryPick(question: string, longContext: string, options:
         task: "retrieval.query",
         dimensions: 1024,
         embedding_type: "float",
-        input: [question]
+        input: [question],
+        truncate: true
       },
       {
         headers: {

src/utils/i18n.json

@@ -4,97 +4,97 @@
     "search_for": "Let me search for ${keywords} to gather more information.",
     "read_for": "Let me read ${urls} to gather more information.",
     "read_for_verify": "Let me fetch the source content to verify the answer.",
-    "late_chunk": "Source is too long, I'm cherry-picking the relevant parts."
+    "late_chunk": "Content of ${url} is too long, let me cherry-pick the relevant parts."
   },
   "zh-CN": {
     "eval_first": "等等，让我先自己评估一下答案。",
     "search_for": "让我搜索${keywords}来获取更多信息。",
-    "read_for": "让我读取网页${urls}来获取更多信息。",
+    "read_for": "让我读取网页 ${urls} 来获取更多信息。",
     "read_for_verify": "让我读取源网页内容来验证答案。",
-    "late_chunk": "源内容太长，我正在挑选相关部分。"
+    "late_chunk": "网页 ${url} 内容太长，我正在筛选精华部分。"
   },
   "zh-TW": {
     "eval_first": "等等，讓我先評估一下答案。",
     "search_for": "讓我搜索${keywords}來獲取更多信息。",
-    "read_for": "讓我閱讀${urls}來獲取更多信息。",
+    "read_for": "讓我閱讀 ${urls} 來獲取更多信息。",
     "read_for_verify": "讓我獲取源內容來驗證答案。",
-    "late_chunk": "源內容太長，我正在挑選相關部分。"
+    "late_chunk": "網頁 ${url} 內容太長，我正在挑選相關部分。"
   },
   "ja": {
     "eval_first": "ちょっと待って、まず答えを評価します。",
     "search_for": "キーワード${keywords}で検索して、情報を集めます。",
-    "read_for": "URL${urls}を読んで、情報を集めます。",
+    "read_for": "${urls} を読んで、情報を集めます。",
     "read_for_verify": "答えを確認するために、ソースコンテンツを取得します。",
-    "late_chunk": "ソースが長すぎるため、関連部分を抜粋しています。"
+    "late_chunk": "${url} のコンテンツが長すぎるため、関連部分を選択します。"
   },
   "ko": {
     "eval_first": "잠시만요, 먼저 답변을 평가해 보겠습니다.",
     "search_for": "키워드 ${keywords}로 검색하여 더 많은 정보를 수집하겠습니다.",
-    "read_for": "URL ${urls}을 읽어 더 많은 정보를 수집하겠습니다.",
+    "read_for": "${urls} 을 읽어 더 많은 정보를 수집하겠습니다.",
     "read_for_verify": "답변을 확인하기 위해 소스 콘텐츠를 가져오겠습니다.",
-    "late_chunk": "소스가 너무 길어서 관련 부분만 추출하고 있습니다."
+    "late_chunk": "${url} 의 콘텐츠가 너무 길어, 관련 부분을 선택하겠습니다."
   },
   "fr": {
     "eval_first": "Un instant, je vais d'abord évaluer la réponse.",
     "search_for": "Je vais rechercher ${keywords} pour obtenir plus d'informations.",
     "read_for": "Je vais lire ${urls} pour obtenir plus d'informations.",
     "read_for_verify": "Je vais récupérer le contenu source pour vérifier la réponse.",
-    "late_chunk": "La source est trop longue, je sélectionne les parties pertinentes."
+    "late_chunk": "Le contenu de ${url} est trop long, je vais sélectionner les parties pertinentes."
   },
   "de": {
     "eval_first": "Einen Moment, ich werde die Antwort zuerst evaluieren.",
     "search_for": "Ich werde nach ${keywords} suchen, um weitere Informationen zu sammeln.",
     "read_for": "Ich werde ${urls} lesen, um weitere Informationen zu sammeln.",
     "read_for_verify": "Ich werde den Quellinhalt abrufen, um die Antwort zu überprüfen.",
-    "late_chunk": "Die Quelle ist zu lang, ich wähle die relevanten Teile aus."
+    "late_chunk": "Der Inhalt von ${url} ist zu lang, ich werde die relevanten Teile auswählen."
   },
   "es": {
     "eval_first": "Un momento, voy a evaluar la respuesta primero.",
     "search_for": "Voy a buscar ${keywords} para recopilar más información.",
     "read_for": "Voy a leer ${urls} para recopilar más información.",
     "read_for_verify": "Voy a obtener el contenido fuente para verificar la respuesta.",
-    "late_chunk": "La fuente es demasiado larga, estoy seleccionando las partes relevantes."
+    "late_chunk": "El contenido de ${url} es demasiado largo, voy a seleccionar las partes relevantes."
   },
   "it": {
     "eval_first": "Un attimo, valuterò prima la risposta.",
     "search_for": "Cercherò ${keywords} per raccogliere ulteriori informazioni.",
     "read_for": "Leggerò ${urls} per raccogliere ulteriori informazioni.",
     "read_for_verify": "Recupererò il contenuto sorgente per verificare la risposta.",
-    "late_chunk": "La fonte è troppo lunga, sto selezionando le parti rilevanti."
+    "late_chunk": "Il contenuto di ${url} è troppo lungo, selezionerò le parti rilevanti."
   },
   "pt": {
     "eval_first": "Um momento, vou avaliar a resposta primeiro.",
     "search_for": "Vou pesquisar ${keywords} para reunir mais informações.",
     "read_for": "Vou ler ${urls} para reunir mais informações.",
     "read_for_verify": "Vou buscar o conteúdo da fonte para verificar a resposta.",
-    "late_chunk": "A fonte é muito longa, estou selecionando as partes relevantes."
+    "late_chunk": "O conteúdo de ${url} é muito longo, vou selecionar as partes relevantes."
   },
   "ru": {
     "eval_first": "Подождите, я сначала оценю ответ.",
     "search_for": "Дайте мне поискать ${keywords} для сбора дополнительной информации.",
     "read_for": "Дайте мне прочитать ${urls} для сбора дополнительной информации.",
     "read_for_verify": "Дайте мне получить исходный контент для проверки ответа.",
-    "late_chunk": "Источник слишком длинный, я выбираю только значимые части."
+    "late_chunk": "Содержимое ${url} слишком длинное, я выберу только значимые части."
   },
   "ar": {
     "eval_first": "لكن انتظر، دعني أقوم بتقييم الإجابة أولاً.",
     "search_for": "دعني أبحث عن ${keywords} لجمع المزيد من المعلومات.",
     "read_for": "دعني أقرأ ${urls} لجمع المزيد من المعلومات.",
     "read_for_verify": "دعني أحضر محتوى المصدر للتحقق من الإجابة.",
-    "late_chunk": "المصدر طويل جدًا، أنا أختار الأجزاء ذات الصلة."
+    "late_chunk": "محتوى ${url} طويل جدًا، سأختار الأجزاء ذات الصلة."
   },
   "nl": {
     "eval_first": "Een moment, ik zal het antwoord eerst evalueren.",
     "search_for": "Ik zal zoeken naar ${keywords} om meer informatie te verzamelen.",
     "read_for": "Ik zal ${urls} lezen om meer informatie te verzamelen.",
     "read_for_verify": "Ik zal de broninhoud ophalen om het antwoord te verifiëren.",
-    "late_chunk": "De bron is te lang, ik selecteer de relevante delen."
+    "late_chunk": "De inhoud van ${url} is te lang, ik zal de relevante delen selecteren."
   },
   "zh": {
     "eval_first": "等等，让我先评估一下答案。",
     "search_for": "让我搜索${keywords}来获取更多信息。",
-    "read_for": "让我阅读${urls}来获取更多信息。",
+    "read_for": "让我阅读 ${urls} 来获取更多信息。",
     "read_for_verify": "让我获取源内容来验证答案。",
-    "late_chunk": "源内容太长，我正在挑选相关部分。"
+    "late_chunk": "网页 ${url} 内容太长，我正在筛选精华部分。"
   }
 }

src/utils/url-tools.ts

@@ -1,5 +1,5 @@
 import {BoostedSearchSnippet, KnowledgeItem, SearchResult, SearchSnippet, TrackerContext, VisitAction} from "../types";
-import {smartMergeStrings} from "./text-tools";
+import {getI18nText, smartMergeStrings} from "./text-tools";
 import {rerankDocuments} from "../tools/jina-rerank";
 import {readUrl} from "../tools/read";
 import {Schemas} from "./schemas";
@@ -401,7 +401,12 @@ export async function processURLs(
   }
 
   // Track the reading action
-  context.actionTracker.trackThink('read_for', schemaGen.languageCode, {urls: urls.join(', ')});
+  const thisStep: VisitAction = {
+    action: 'visit',
+    think: getI18nText('read_for', schemaGen.languageCode, {urls: urls.join(', ')}),
+    URLTargets: urls
+  }
+  context.actionTracker.trackAction({thisStep})
 
   // Process each URL in parallel
   const urlResults = await Promise.all(
@@ -423,7 +428,7 @@ export async function processURLs(
         // Add to knowledge base
         allKnowledge.push({
           question: `What do expert say about "${data.title}"?`,
-          answer: await cherryPick(question, data.content, {}, context, schemaGen),
+          answer: await cherryPick(question, data.content, {}, context, schemaGen, url),
           references: [data.url],
           type: 'url',
           updated: guessedTime