fix: improve eval

README.md

@@ -68,7 +68,8 @@ You can use it with any OpenAI-compatible client. For the authentication Bearer,
 If you are building a web/local/mobile client that uses `Jina DeepSearch API`, here are some design guidelines:
 - Our API is fully compatible with [OpenAI API schema](https://platform.openai.com/docs/api-reference/chat/create), this should greatly simplify the integration process. The model name is `jina-deepsearch-v1`.
 - Our DeepSearch API is a reasoning+search grounding LLM, so it's best for questions that require deep reasoning and search.
-- Two special tokens are introduced `<think>...</think>`, `<references>...</references>`. Please render them with care.
+- Two special tokens are introduced `<think>...</think>`. Please render them with care.
+- Citations are often provided, and in [Github-flavored markdown footnote format](https://github.blog/changelog/2021-09-30-footnotes-now-supported-in-markdown-fields/), e.g. `[^1]`, `[^2]`, ...
 - Guide the user to get a Jina API key from https://jina.ai, with 1M free tokens for new API key.
 - There are rate limits, [between 10RPM to 30RPM depending on the API key tier](https://jina.ai/contact-sales#rate-limit).
 - [Download Jina AI logo here](https://jina.ai/logo-Jina-1024.zip)

src/agent.ts

@@ -76,6 +76,7 @@ function getSchema(allowReflect: boolean, allowRead: boolean, allowAnswer: boole
 function getPrompt(
   question: string,
   context?: string[],
+  allChatHistory?: { user: string, assistant: string }[],
   allQuestions?: string[],
   allKeywords?: string[],
   allowReflect: boolean = true,
@@ -101,15 +102,20 @@ ${question}
 </question>
 `);
 
-  // Add context section if exists
+  // add chat history if exist
-  if (context?.length) {
+  if (allChatHistory?.length) {
     sections.push(`
-You have conducted the following actions:
+You have conducted the following chat history with the user:
-<context>
+<chat-history>
-${context.join('\n')}
+${allChatHistory.map(({user, assistant}) => `
-
+<user>
-</context>
+${user}
-`);
+</user>
+<you>
+${assistant}
+</you>
+`).join('\n')}
+</chat-history>`);
   }
 
   // Add knowledge section if exists
@@ -142,6 +148,19 @@ ${knowledgeItems}
 `);
   }
 
+
+
+  // Add context section if exists
+  if (context?.length) {
+    sections.push(`
+You have conducted the following actions:
+<context>
+${context.join('\n')}
+
+</context>
+`);
+  }
+
   // Add bad context section if exists
   if (badContext?.length) {
     const attempts = badContext
@@ -159,7 +178,7 @@ ${knowledgeItems}
     const learnedStrategy = badContext.map(c => c.improvement).join('\n');
 
     sections.push(`
-Your have tried the following actions but failed to find the answer to the question:
+Also, you have tried the following actions but failed to find the answer to the question:
 <bad-attempts>    
 
 ${attempts}
@@ -303,6 +322,7 @@ export async function getResponse(question: string,
   const gaps: string[] = [question];  // All questions to be answered including the orginal question
   const allQuestions = [question];
   const allKeywords = [];
+  const allChatHistory: { user: string, assistant: string }[] = [];
   const allKnowledge: KnowledgeItem[] = [];  // knowledge are intermedidate questions that are answered
   // iterate over historyMessages
   // if role is user and content is question, add to allQuestions, the next assistant content should be the answer
@@ -310,12 +330,15 @@ export async function getResponse(question: string,
   historyMessages?.forEach((message, i) => {
     if (message.role === 'user' && message.content && historyMessages[i + 1]?.role === 'assistant') {
       allQuestions.push(message.content as string)
-      allKnowledge.push({
+      const answerContent = (historyMessages[i + 1]?.content || '') as string;
-        question: message.content as string,
+      // Remove <think></think> tags and their content using regex
-        answer: (historyMessages[i + 1]?.content || '') as string,
+      const cleanedAnswer = answerContent.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
-        type: 'chat-history',
+      if (cleanedAnswer) {
-        updated: new Date().toISOString()
+        allChatHistory.push({
-      });
+          user: message.content as string,
+          assistant: cleanedAnswer,
+        });
+      }
     }
   })
 
@@ -353,6 +376,7 @@ export async function getResponse(question: string,
     prompt = getPrompt(
       currentQuestion,
       diaryContext,
+      allChatHistory,
       allQuestions,
       allKeywords,
       allowReflect,
@@ -693,6 +717,7 @@ You decided to think out of the box or cut from a completely different angle.`);
     const prompt = getPrompt(
       question,
       diaryContext,
+      allChatHistory,
       allQuestions,
       allKeywords,
       false,

src/app.ts

@@ -28,35 +28,21 @@ app.get('/health', (req, res) => {
 });
 
 function buildMdFromAnswer(answer: AnswerAction) {
-  if (!answer.references?.length || !answer.references.some(ref => ref.url.startsWith('http'))) {
+  if (!answer.references?.length) {
     return answer.answer;
   }
 
   const references = answer.references.map((ref, i) => {
     // Clean up the quote text
     const cleanQuote = ref.exactQuote
-      // Remove HTML artifacts
+      .replace(/[^a-zA-Z0-9]+/g, ' ')
-      .replace(/<[^>]+>/g, '')
-      // Remove multiple spaces
-      .replace(/\s+/g, ' ')
-      // Remove special characters and markdown conflicts
-      .replace(/[[\]_*`]/g, '')
-      // Clean up any remaining artifacts
-      .replace(/=+/g, '')
-      .replace(/_[^_]+_/g, '$1')
-      // Remove navigation artifacts
-      .replace(/arrow_drop_down/g, '')
-      .replace(/language/g, '')
-      // Trim whitespace
       .trim();
 
-    // If the quote is too long, truncate it
+    if (ref.url.startsWith('http')) {
-    const maxLength = 20;
+      return `[^${i + 1}]: [${cleanQuote}](${ref.url})`;
-    const truncatedQuote = cleanQuote.length > maxLength
+    } else {
-      ? cleanQuote.substring(0, maxLength) + '...'
+      return `[^${i + 1}]: ${cleanQuote}`;
-      : cleanQuote;
+    }
-
-    return `[^${i + 1}]: [${truncatedQuote}](${ref.url})`;
   }).join('\n\n');
 
   return `