refactor: knowledge as msg

src/agent.ts

@@ -49,6 +49,58 @@ async function sleep(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+function BuildMsgsFromKnowledge(knowledge: KnowledgeItem[]) : CoreMessage[] {
+  // build user, assistant pair messages from knowledge
+    const messages: CoreMessage[] = [];
+    knowledge.forEach(k => {
+      messages.push({role: 'user', content: k.question.trim()});
+      const aMsg = `
+${k.updated && (k.type === 'url' || k.type === 'side-info') ? `
+<answer-datetime>
+${k.updated}
+</answer-datetime>
+` : ''}
+
+${k.references && k.type === 'url' ? `
+<url>
+${k.references[0]}
+</url>
+` : ''}
+
+${k.answer}
+      `.trim();
+      messages.push({role: 'assistant', content: removeExtraLineBreaks(aMsg)});
+    });
+    return messages;
+}
+
+function composeMsgs(messages: CoreMessage[], knowledge: KnowledgeItem[], question: string, finalAnswerPIP?: string) {
+    const msgs = [...messages, ...BuildMsgsFromKnowledge(knowledge)];
+
+    const userContent = `
+${knowledge.length>0 ? `
+Based on what you learned from our previous messages, what is your answer to the question below?
+`: ''}
+
+${question}
+
+${finalAnswerPIP ? `
+<answer-requirements>
+- You provide deep, unexpected insights, identifying hidden patterns and connections, and creating "aha moments.".
+- You break conventional thinking, establish unique cross-disciplinary connections, and bring new perspectives to the user.
+${finalAnswerPIP}
+</answer-requirements>` : ''}
+    `.trim();
+
+    // only add if the last user msg is not the same
+    // first find the last message whose role is 'user'
+    const lastUserMsg = msgs.filter(m => m.role === 'user').pop();
+    if ((lastUserMsg?.content as string).trim() !== userContent) {
+      msgs.push({role: 'user', content: removeExtraLineBreaks(userContent)});
+    }
+    return msgs;
+}
+
 
 function getPrompt(
   context?: string[],
@@ -59,7 +111,6 @@ function getPrompt(
   allowRead: boolean = true,
   allowSearch: boolean = true,
   allowCoding: boolean = true,
-  badContext?: { question: string, answer: string, evaluation: string, recap: string; blame: string; improvement: string; }[],
   knowledge?: KnowledgeItem[],
   allURLs?: BoostedSearchSnippet[],
   beastMode?: boolean,
@@ -71,42 +122,9 @@ function getPrompt(
   sections.push(`Current date: ${new Date().toUTCString()}
 
 You are an advanced AI research agent from Jina AI. You are specialized in multistep reasoning. 
-Using your training data and prior lessons learned, answer the user question with absolute certainty.
+Using your best knowledge, conversation with the user and lessons learned, answer the user question with absolute certainty.
 `);
 
-  // Add knowledge section if exists
-  if (knowledge?.length) {
-    const knowledgeItems = knowledge
-      .map((k, i) => `
-<knowledge-${i + 1}>
-<question>
-${k.question}
-</question>
-<answer>
-${k.answer}
-</answer>
-${k.updated && (k.type === 'url' || k.type === 'side-info') ? `
-<answer-datetime>
-${k.updated}
-</answer-datetime>
-` : ''}
-${k.references && k.type === 'url' ? `
-<url>
-${k.references[0]}
-</url>
-` : ''}
-</knowledge-${i + 1}>
-`)
-      .join('\n\n');
-
-    sections.push(`
-You have successfully gathered some knowledge which might be useful for answering the original question. Here is the knowledge you have gathered so far:
-<knowledge>
-${knowledgeItems}
-</knowledge>
-`);
-  }
-
 
   // Add context section if exists
   if (context?.length) {
@@ -119,37 +137,6 @@ ${context.join('\n')}
 `);
   }
 
-  // Add bad context section if exists
-  if (badContext?.length) {
-    const attempts = badContext
-      .map((c, i) => `
-<attempt-${i + 1}>
-- Question: ${c.question}
-- Answer: ${c.answer}
-- Reject Reason: ${c.evaluation}
-- Actions Recap: ${c.recap}
-- Actions Blame: ${c.blame}
-</attempt-${i + 1}>
-`)
-      .join('\n\n');
-
-    const learnedStrategy = badContext.map(c => c.improvement).join('\n');
-
-    sections.push(`
-Also, you have tried the following actions but failed to find the answer to the question:
-<bad-attempts>    
-
-${attempts}
-
-</bad-attempts>
-
-Based on the failed attempts, you have learned the following strategy:
-<learned-strategy>
-${learnedStrategy}
-</learned-strategy>
-`);
-  }
-
   // Build actions section
 
   if (allowRead) {
@@ -169,14 +156,6 @@ ${urlList}
 `);
   }
 
-  if (allowCoding) {
-    actionSections.push(`
-<action-coding>
-- This JavaScript-based solution helps you handle programming tasks like counting, filtering, transforming, sorting, regex extraction, and data processing.
-- Simply describe your problem in the "codingIssue" field. Include actual values for small inputs or variable names for larger datasets.
-- No code writing is required – senior engineers will handle the implementation.
-</action-coding>`);
-  }
 
   if (allowSearch) {
 
@@ -216,7 +195,7 @@ ${allKeywords.join('\n')}
 PRIME DIRECTIVE:
 - DEMOLISH ALL HESITATION! ANY RESPONSE SURPASSES SILENCE!
 - PARTIAL STRIKES AUTHORIZED - DEPLOY WITH FULL CONTEXTUAL FIREPOWER
-- TACTICAL REUSE FROM <bad-attempts> SANCTIONED
+- TACTICAL REUSE FROM PREVIOUS CONVERSATION SANCTIONED
 - WHEN IN DOUBT: UNLEASH CALCULATED STRIKES BASED ON AVAILABLE INTEL!
 
 FAILURE IS NOT AN OPTION. EXECUTE WITH EXTREME PREJUDICE! ⚡️
@@ -227,10 +206,19 @@ FAILURE IS NOT AN OPTION. EXECUTE WITH EXTREME PREJUDICE! ⚡️
   if (allowReflect) {
     actionSections.push(`
 <action-reflect>
-- Think slowly and planning lookahead. Examine <question>, <context>, <knowledge>, <bad-attempts>, and <learned-strategy> to identify knowledge gaps. 
+- Think slowly and planning lookahead. Examine <question>, <context>, previous conversation with users to identify knowledge gaps. 
 - Reflect the gaps and plan a list key clarifying questions that deeply related to the original question and lead to the answer
 </action-reflect>
 `);
+  }
+
+    if (allowCoding) {
+    actionSections.push(`
+<action-coding>
+- This JavaScript-based solution helps you handle programming tasks like counting, filtering, transforming, sorting, regex extraction, and data processing.
+- Simply describe your problem in the "codingIssue" field. Include actual values for small inputs or variable names for larger datasets.
+- No code writing is required – senior engineers will handle the implementation.
+</action-coding>`);
   }
 
   sections.push(`
@@ -253,15 +241,6 @@ function updateContext(step: any) {
   allContext.push(step)
 }
 
-function replaceLastUserMsg(messages: Array<CoreMessage>, content: string) {
-  return messages.map((m, i) => {
-        if (m.role === 'user' && i === messages.length - 1) {
-            return {...m, content}
-        }
-        return m
-    });
-}
-
 
 export async function getResponse(question?: string,
                                   tokenBudget: number = 1_000_000,
@@ -275,6 +254,9 @@ export async function getResponse(question?: string,
   let badAttempts = 0;
 
   question = question?.trim() as string;
+  // remove incoming system messages to avoid override
+  messages = messages?.filter(m => m.role !== 'system');
+
   if (messages && messages.length > 0) {
     // 2 cases
     const lastContent = messages[messages.length - 1].content;
@@ -303,7 +285,6 @@ export async function getResponse(question?: string,
   const allKeywords = [];
   const allKnowledge: KnowledgeItem[] = [];  // knowledge are intermedidate questions that are answered
 
-  const badContext = [];
   let diaryContext = [];
   let weightedURLs: BoostedSearchSnippet[] = [];
   let allowAnswer = true;
@@ -312,6 +293,7 @@ export async function getResponse(question?: string,
   let allowReflect = true;
   let allowCoding = true;
   let system = '';
+  let msgWithKnowledge: CoreMessage[] = [];
   let thisStep: StepAction = {action: 'answer', answer: '', references: [], think: '', isFinal: false};
 
   const allURLs: Record<string, SearchSnippet> = {};
@@ -327,8 +309,9 @@ export async function getResponse(question?: string,
     const budgetPercentage = (context.tokenTracker.getTotalUsage().totalTokens / tokenBudget * 100).toFixed(2);
     console.log(`Step ${totalStep} / Budget used ${budgetPercentage}%`);
     console.log('Gaps:', gaps);
-    allowReflect = allowReflect && (gaps.length <= 1);
-    const currentQuestion: string = gaps.length > 0 ? gaps.shift()! : question
+    allowReflect = allowReflect && (gaps.length <= MAX_REFLECT_PER_STEP);
+    // rotating question from gaps
+    const currentQuestion: string = gaps[totalStep % gaps.length];
     // if (!evaluationMetrics[currentQuestion]) {
     //   evaluationMetrics[currentQuestion] =
     //     await evaluateQuestion(currentQuestion, context, SchemaGen)
@@ -375,17 +358,17 @@ export async function getResponse(question?: string,
       allowRead,
       allowSearch,
       allowCoding,
-      badContext,
       allKnowledge,
       weightedURLs,
       false,
     );
-    schema = SchemaGen.getAgentSchema(allowReflect, allowRead, allowAnswer, allowSearch, allowCoding, finalAnswerPIP, currentQuestion)
+    schema = SchemaGen.getAgentSchema(allowReflect, allowRead, allowAnswer, allowSearch, allowCoding, currentQuestion)
+    msgWithKnowledge = composeMsgs(messages, allKnowledge, currentQuestion, currentQuestion === question ? finalAnswerPIP : undefined);
     const result = await generator.generateObject({
       model: 'agent',
       schema,
       system,
-      messages: replaceLastUserMsg(messages, currentQuestion),
+      messages: msgWithKnowledge,
     });
     thisStep = {
       action: result.object.action,
@@ -394,7 +377,7 @@ export async function getResponse(question?: string,
     } as StepAction;
     // print allowed and chose action
     const actionsStr = [allowSearch, allowRead, allowAnswer, allowReflect, allowCoding].map((a, i) => a ? ['search', 'read', 'answer', 'reflect'][i] : null).filter(a => a).join(', ');
-    console.log(`${thisStep.action} <- [${actionsStr}]`);
+    console.log(`${currentQuestion}: ${thisStep.action} <- [${actionsStr}]`);
     console.log(thisStep)
 
     context.actionTracker.trackAction({totalStep, thisStep, gaps, badAttempts});
@@ -447,6 +430,10 @@ export async function getResponse(question?: string,
           SchemaGen,
           currentQuestion
         );
+
+        if (!evaluationMetrics[currentQuestion].includes('attribution')) {
+            evaluationMetrics[currentQuestion].push('attribution')
+        }
       }
 
       updateContext({
@@ -459,7 +446,9 @@ export async function getResponse(question?: string,
       let evaluation: EvaluationResponse = {pass: true, think: ''};
       if (evaluationMetrics[currentQuestion].length > 0) {
         context.actionTracker.trackThink('eval_first', SchemaGen.languageCode)
-        evaluation = await evaluateAnswer(currentQuestion, thisStep,
+        evaluation = await evaluateAnswer(
+          currentQuestion,
+          thisStep,
           evaluationMetrics[currentQuestion],
           context,
           allKnowledge,
@@ -512,19 +501,29 @@ ${evaluation.think}
             // store the bad context and reset the diary context
             const errorAnalysis = await analyzeSteps(diaryContext, context, SchemaGen);
 
-            badContext.push({
-              question: currentQuestion,
-              answer: thisStep.answer,
-              evaluation: evaluation.think,
-              ...errorAnalysis
-            });
+            allKnowledge.push({
+              question: `
+Why is the following answer bad for the question? Please reflect
 
-            if (errorAnalysis.questionsToAnswer) {
-              errorAnalysis.questionsToAnswer = chooseK((await dedupQueries(errorAnalysis.questionsToAnswer, allQuestions, context.tokenTracker)).unique_queries, MAX_REFLECT_PER_STEP);
-              gaps.push(...errorAnalysis.questionsToAnswer);
-              allQuestions.push(...errorAnalysis.questionsToAnswer);
-              gaps.push(question);  // always keep the original question in the gaps
-            }
+<question>
+${currentQuestion}
+</question>
+
+<answer>
+${thisStep.answer}
+</answer>
+`,
+              answer: `
+${evaluation.think}
+
+${errorAnalysis.recap}
+
+${errorAnalysis.blame}
+
+${errorAnalysis.improvement}
+`,
+              type: 'qa',
+            })
 
             badAttempts++;
             allowAnswer = false;  // disable answer action in the immediate next step
@@ -533,6 +532,7 @@ ${evaluation.think}
           }
         }
       } else if (evaluation.pass) {
+        // solved a gap question
         diaryContext.push(`
 At step ${step}, you took **answer** action. You found a good answer to the sub-question:
 
@@ -554,6 +554,8 @@ Although you solved a sub-question, you still need to find the answer to the ori
           type: 'qa',
           updated: formatDateBasedOnType(new Date(), 'full')
         });
+        // solved sub-question!
+        gaps.splice(gaps.indexOf(currentQuestion), 1);
       }
     } else if (thisStep.action === 'reflect' && thisStep.questionsToAnswer) {
       thisStep.questionsToAnswer = chooseK((await dedupQueries(thisStep.questionsToAnswer, allQuestions, context.tokenTracker)).unique_queries, MAX_REFLECT_PER_STEP);
@@ -569,7 +571,6 @@ You will now figure out the answers to these sub-questions and see if they can h
 `);
         gaps.push(...newGapQuestions);
         allQuestions.push(...newGapQuestions);
-        gaps.push(question);  // always keep the original question in the gaps
         updateContext({
           totalStep,
           ...thisStep,
@@ -784,11 +785,11 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
       }
     }
 
-    await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs}, totalStep);
+    await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs, msgWithKnowledge}, totalStep);
     await sleep(STEP_SLEEP);
   }
 
-  await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs}, totalStep);
+  await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs, msgWithKnowledge}, totalStep);
   if (!(thisStep as AnswerAction).isFinal) {
     console.log('Enter Beast mode!!!')
     // any answer is better than no answer, humanity last resort
@@ -803,18 +804,18 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
       false,
       false,
       false,
-      badContext,
       allKnowledge,
       weightedURLs,
       true,
     );
 
-    schema = SchemaGen.getAgentSchema(false, false, true, false, false, finalAnswerPIP, question);
+    schema = SchemaGen.getAgentSchema(false, false, true, false, false, question);
+    msgWithKnowledge = composeMsgs(messages, allKnowledge, question, finalAnswerPIP);
     const result = await generator.generateObject({
       model: 'agentBeastMode',
       schema,
       system,
-      messages
+      messages: msgWithKnowledge
     });
     thisStep = {
       action: result.object.action,
@@ -828,7 +829,7 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
   (thisStep as AnswerAction).mdAnswer = buildMdFromAnswer((thisStep as AnswerAction))
   console.log(thisStep)
 
-  await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs}, totalStep);
+  await storeContext(system, schema, {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs, msgWithKnowledge}, totalStep);
 
   // max return 300 urls
   const returnedURLs = weightedURLs.slice(0, 50).map(r => r.url);
@@ -846,10 +847,11 @@ async function storeContext(prompt: string, schema: any, memory: {
                               allQuestions: string[];
                               allKnowledge: KnowledgeItem[];
                               weightedURLs: BoostedSearchSnippet[];
+                              msgWithKnowledge: CoreMessage[];
                             }
   , step: number) {
 
-  const {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs} = memory;
+  const {allContext, allKeywords, allQuestions, allKnowledge, weightedURLs, msgWithKnowledge} = memory;
   if ((process as any).asyncLocalContext?.available?.()) {
 
     (process as any).asyncLocalContext.ctx.promptContext = {
@@ -877,6 +879,7 @@ ${JSON.stringify(zodToJsonSchema(schema), null, 2)}
     await fs.writeFile('questions.json', JSON.stringify(allQuestions, null, 2));
     await fs.writeFile('knowledge.json', JSON.stringify(allKnowledge, null, 2));
     await fs.writeFile('urls.json', JSON.stringify(weightedURLs, null, 2));
+    await fs.writeFile('messages.json', JSON.stringify(msgWithKnowledge, null, 2));
   } catch (error) {
     console.error('Context storage failed:', error);
   }

src/tools/error-analyzer.ts

@@ -83,11 +83,7 @@ The answer is not definitive and fails to provide the requested information.  La
   "blame": "The root cause of failure was getting stuck in a repetitive search pattern without adapting the strategy. Steps 4-5 repeated the same search, and step 6 deviated to less reliable entertainment sources instead of exploring business journals, news articles, or professional databases. Additionally, the process didn't attempt to triangulate age through indirect information like education history or career milestones.",
   
   "improvement": "1. Avoid repeating identical searches and implement a strategy to track previously searched terms. 2. When direct age/birthdate searches fail, try indirect approaches like: searching for earliest career mentions, finding university graduation years, or identifying first company founding dates. 3. Focus on high-quality business sources and avoid entertainment websites for professional information. 4. Consider using industry event appearances or conference presentations where age-related context might be mentioned. 5. If exact age cannot be determined, provide an estimated range based on career timeline and professional achievements.",
-  
-  "questionsToAnswer": [
-    "What alternative professional databases or news archives could provide reliable biographical information?",
-    "How can we use education history or career milestones to estimate age range?"
-  ]
+ 
 }
 </output>
 </example>`,

src/tools/evaluator.ts

@@ -1,9 +1,7 @@
 import {GenerateObjectResult} from 'ai';
 import {AnswerAction, EvaluationResponse, EvaluationType, KnowledgeItem, PromptPair, TrackerContext} from '../types';
-import {readUrl} from "./read";
 import {ObjectGeneratorSafe} from "../utils/safe-generator";
 import {Schemas} from "../utils/schemas";
-import {removeAllLineBreaks} from "../utils/text-tools";
 
 const TOOL_NAME = 'evaluator';
 

src/types.ts

@@ -173,7 +173,6 @@ export type ErrorAnalysisResponse = {
   recap: string;
   blame: string;
   improvement: string;
-  questionsToAnswer: string[];
 };
 
 

src/utils/schemas.ts

@@ -4,7 +4,7 @@ import {EvaluationType, PromptPair} from "../types";
 
 export const MAX_URLS_PER_STEP = 4
 export const MAX_QUERIES_PER_STEP = 7
-export const MAX_REFLECT_PER_STEP = 3
+export const MAX_REFLECT_PER_STEP = 1
 
 function getLanguagePrompt(question: string): PromptPair {
   return {
@@ -112,11 +112,7 @@ export class Schemas {
     return z.object({
       recap: z.string().describe('Recap of the actions taken and the steps conducted in first person narrative.').max(500),
       blame: z.string().describe(`Which action or the step was the root cause of the answer rejection. ${this.getLanguagePrompt()}`).max(500),
-      improvement: z.string().describe(`Suggested key improvement for the next iteration, do not use bullet points, be concise and hot-take vibe. ${this.getLanguagePrompt()}`).max(500),
-      questionsToAnswer: z.array(
-        z.string().describe("each question must be a single line, concise and clear. not composite or compound, less than 20 words.")
-      ).max(MAX_REFLECT_PER_STEP)
-        .describe(`List of most important reflect questions to fill the knowledge gaps. Maximum provide ${MAX_REFLECT_PER_STEP} reflect questions.`)
+      improvement: z.string().describe(`Suggested key improvement for the next iteration, do not use bullet points, be concise and hot-take vibe. ${this.getLanguagePrompt()}`).max(500)
     });
   }
 
@@ -200,7 +196,7 @@ export class Schemas {
   }
 
   getAgentSchema(allowReflect: boolean, allowRead: boolean, allowAnswer: boolean, allowSearch: boolean, allowCoding: boolean,
-                 finalAnswerPIP?: string, currentQuestion?: string): z.ZodObject<any> {
+                 currentQuestion?: string): z.ZodObject<any> {
     const actionSchemas: Record<string, z.ZodOptional<any>> = {};
 
     if (allowSearch) {
@@ -227,14 +223,13 @@ export class Schemas {
         references: z.array(
           z.object({
             exactQuote: z.string().describe("Exact relevant quote from the document, must be a soundbite, short and to the point, no fluff").max(30),
-            url: z.string().describe("source URL; must be copy directly from existing knowledge's <url>, avoid example.com or any placeholder fake URLs").max(100),
-            dateTime: z.string().describe("Use original knowledge's <dateime> if available.").max(16),
+            url: z.string().describe("source URL; must be copy directly from previous message's <url>, avoid example.com or any placeholder fake URLs").max(100),
+            dateTime: z.string().describe("Use original message's <answer-dateime> if available.").max(16),
           }).required()
         ).describe("Required when action='answer'. Must be an array of references that support the answer, each reference must contain an exact quote, URL and datetime"),
         answer: z.string()
           .describe(`Required when action='answer'. 
           
-          ${finalAnswerPIP}
           Use all your knowledge you have collected, cover multiple aspects if needed. 
           Must be definitive, no ambiguity, no uncertainty, no disclaimers. Must in ${this.languageStyle} and confident. 
           Use markdown footnote syntax like [^1], [^2] to refer the corresponding reference item. 
@@ -249,13 +244,13 @@ export class Schemas {
         questionsToAnswer: z.array(
           z.string().describe(`
 Ensure each reflection question:
- - Cuts to core emotional truths while staying anchored to the original question
- - Transforms surface-level problems into deeper psychological insights
+ - Cuts to core emotional truths while staying anchored to <og-question>
+ - Transforms surface-level problems into deeper psychological insights, helps answer <og-question>
  - Makes the unconscious conscious
- - NEVER pose general questions like: "How can I verify the accuracy of information before including it in my answer?", "What information was actually contained in the URLs I found?"         
+ - NEVER pose general questions like: "How can I verify the accuracy of information before including it in my answer?", "What information was actually contained in the URLs I found?", "How can i tell if a source is reliable?".         
           `)
         ).max(MAX_REFLECT_PER_STEP)
-          .describe(`Required when action='reflect'. Reflection and planing, generate a list of most important questions to fill the knowledge gaps to the original question ${currentQuestion}. Maximum provide ${MAX_REFLECT_PER_STEP} reflect questions.`)
+          .describe(`Required when action='reflect'. Reflection and planing, generate a list of most important questions to fill the knowledge gaps to <og-question> ${currentQuestion} </og-question>. Maximum provide ${MAX_REFLECT_PER_STEP} reflect questions.`)
       }).optional()
     }