From c74911ee580d54603dc9b721382473fe4394467e Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 12 Jun 2025 19:10:36 -0700 Subject: [PATCH 1/4] fix: update answer aggregation and logging in reducer --- src/agent.ts | 4 +++- src/tools/reducer.ts | 40 ++++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index def3e41..f3035c2 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -448,6 +448,7 @@ export async function getResponse(question?: string, const gaps: string[] = [question]; // All questions to be answered including the orginal question const allQuestions = [question]; const allKeywords: string[] = []; + let candidateAnswers: string[] = []; const allKnowledge: KnowledgeItem[] = []; // knowledge are intermedidate questions that are answered let diaryContext = []; @@ -818,6 +819,7 @@ But then you realized you have asked them before. You decided to to think out of isFinal: true, isAggregated: true } as AnswerAction; + candidateAnswers = subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).filter(a => a) as string[]; // aggregate urls visitedURLs.push(...subproblemResponses.map(r => r.readURLs).flat()); @@ -1084,7 +1086,7 @@ But unfortunately, you failed to solve the issue. You need to think out of the b } } } else if (answerStep.isAggregated) { - answerStep.answer = await reduceAnswers(answerStep.answer, context, SchemaGen); + answerStep.answer = await reduceAnswers(candidateAnswers, context, SchemaGen); answerStep.mdAnswer = repairMarkdownFootnotesOuter(buildMdFromAnswer(answerStep)); } diff --git a/src/tools/reducer.ts b/src/tools/reducer.ts index a186cc1..07628c9 100644 --- a/src/tools/reducer.ts +++ b/src/tools/reducer.ts @@ -5,7 +5,7 @@ import { Schemas } from "../utils/schemas"; import { logInfo, logError, logDebug, logWarning } from '../logging'; -function getPrompt(mdContent: string): PromptPair { +function getPrompt(answers: string[]): PromptPair { return { @@ -14,7 +14,7 @@ You are an article aggregator that creates a coherent, high-quality article by s 1. Content Preservation -ALWAYS preserve original sentences verbatim - do not paraphrase or rewrite +ALWAYS preserve original sentences verbatim - do not delete Select the highest quality version when multiple articles cover the same point Maintain the original author's voice and technical accuracy Keep direct quotes, statistics, and factual claims exactly as written @@ -46,22 +46,29 @@ No attribution to individual sources (present as unified piece) Do not add your own commentary or analysis Do not change technical terms, names, or specific details - -Your final output should read as a cohesive, high-quality article that appears to be written by a single author, while actually being a careful curation of the best sentences from all input sources. `, - user: mdContent + user: ` + Here are the answers to merge: +${answers.map((a, i) => ` + +${a} + + +Your output should read as a coherent, high-quality article that appears to be written by a single author, while actually being a careful curation of the best sentences from all input sources. +`).join('\n\n')} + ` } } const TOOL_NAME = 'reducer'; export async function reduceAnswers( - mdContent: string, + answers: string[], trackers: TrackerContext, schema: Schemas ): Promise { try { - const prompt = getPrompt(mdContent); + const prompt = getPrompt(answers); trackers?.actionTracker.trackThink('reduce_answer', schema.languageCode) const result = await generateText({ @@ -71,25 +78,30 @@ export async function reduceAnswers( }); trackers.tokenTracker.trackUsage(TOOL_NAME, result.usage) + const totalLength = answers.reduce((acc, curr) => acc + curr.length, 0); + const reducedLength = result.text.length; - logDebug(`${TOOL_NAME} before/after: ${mdContent.length} -> ${result.text.length}`, { - originalContent: mdContent, + logDebug(`${TOOL_NAME} before/after: ${totalLength} -> ${reducedLength}`, { + answers, reducedContent: result.text }); - if (result.text.length < mdContent.length * 0.5) { - logWarning(`reduce content length ${result.text.length} is significantly shorter than original content ${mdContent.length}, return original content instead.`, { - originalContent: mdContent, + + const reductionRatio = reducedLength / totalLength; + if (reductionRatio < 0.5) { + logWarning(`reduce content length ${reducedLength} is significantly shorter than original content ${totalLength}, return original content instead.`, { + originalContent: answers, repairedContent: result.text }); - return mdContent; + // return simple join of answers + return answers.join('\n\n'); } return result.text; } catch (error) { logError(TOOL_NAME, { error }); - return mdContent; + return answers.join('\n\n'); } } \ No newline at end of file From 7621f589b2946bd2dbe6cd6a6c0984afd66eb1f9 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 12 Jun 2025 19:24:56 -0700 Subject: [PATCH 2/4] fix: adjust reduction ratio threshold and update logging in reducer --- src/tools/reducer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/reducer.ts b/src/tools/reducer.ts index 07628c9..7fd6171 100644 --- a/src/tools/reducer.ts +++ b/src/tools/reducer.ts @@ -89,8 +89,8 @@ export async function reduceAnswers( const reductionRatio = reducedLength / totalLength; - if (reductionRatio < 0.5) { - logWarning(`reduce content length ${reducedLength} is significantly shorter than original content ${totalLength}, return original content instead.`, { + if (reductionRatio < 0.6) { + logWarning(`reducer content length ${reducedLength} is significantly shorter than original content ${totalLength}, return original content instead.`, { originalContent: answers, repairedContent: result.text }); From e8b99a588980e910d97fdc0f9d1846e4bad6008c Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 12 Jun 2025 19:39:14 -0700 Subject: [PATCH 3/4] fix: update answer aggregation logic in agent and clean up reducer logging --- src/agent.ts | 2 +- src/tools/reducer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index f3035c2..bbd39d1 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -1086,7 +1086,7 @@ But unfortunately, you failed to solve the issue. You need to think out of the b } } } else if (answerStep.isAggregated) { - answerStep.answer = await reduceAnswers(candidateAnswers, context, SchemaGen); + answerStep.answer = candidateAnswers.join('\n\n'); // await reduceAnswers(candidateAnswers, context, SchemaGen); answerStep.mdAnswer = repairMarkdownFootnotesOuter(buildMdFromAnswer(answerStep)); } diff --git a/src/tools/reducer.ts b/src/tools/reducer.ts index 7fd6171..c0380e8 100644 --- a/src/tools/reducer.ts +++ b/src/tools/reducer.ts @@ -2,7 +2,7 @@ import { PromptPair, TrackerContext } from '../types'; import { getModel } from "../config"; import { generateText } from "ai"; import { Schemas } from "../utils/schemas"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logError, logDebug, logWarning } from '../logging'; function getPrompt(answers: string[]): PromptPair { From a664e4d8519046eb4299d950f5478d2af2e18881 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 12 Jun 2025 19:57:50 -0700 Subject: [PATCH 4/4] fix: refine subproblem handling and aggregation logic in agent --- src/agent.ts | 66 ++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index bbd39d1..7d53d34 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -796,39 +796,45 @@ But then you realized you have asked them before. You decided to to think out of if (teamSize > 1) { const subproblems = await researchPlan(question, teamSize, soundBites, context, SchemaGen); - // parallel call getResponse for each subproblem with exact same parameters from the current step, but their teamSize is 1 - const subproblemResponses = await Promise.all(subproblems.map(subproblem => getResponse(subproblem, - tokenBudget, - maxBadAttempts, - context, - messages, - numReturnedURLs, - noDirectAnswer, - boostHostnames, - badHostnames, - onlyHostnames, - maxRef, - minRelScore, languageCode, searchLanguageCode, searchProvider, withImages, 1))); - // convert current step to AnswerAction - thisStep = { - action: 'answer', - think: thisStep.think, - answer: subproblemResponses.map(r => (r.result as AnswerAction).answer).join('\n\n'), - mdAnswer: subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).join('\n\n'), - references: subproblemResponses.map(r => (r.result as AnswerAction).references).flat(), - isFinal: true, - isAggregated: true - } as AnswerAction; - candidateAnswers = subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).filter(a => a) as string[]; + if (subproblems.length > 1) { - // aggregate urls - visitedURLs.push(...subproblemResponses.map(r => r.readURLs).flat()); - weightedURLs = subproblemResponses.map(r => r.allURLs.map(url => ({ url, title: '' } as BoostedSearchSnippet))).flat(); + // parallel call getResponse for each subproblem with exact same parameters from the current step, but their teamSize is 1 + const subproblemResponses = await Promise.all(subproblems.map(subproblem => getResponse(subproblem, + tokenBudget, + maxBadAttempts, + context, + messages, + numReturnedURLs, + noDirectAnswer, + boostHostnames, + badHostnames, + onlyHostnames, + maxRef, + minRelScore, languageCode, searchLanguageCode, searchProvider, withImages, 1))); + // convert current step to AnswerAction + thisStep = { + action: 'answer', + think: thisStep.think, + answer: subproblemResponses.map(r => (r.result as AnswerAction).answer).join('\n\n'), + mdAnswer: subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).join('\n\n'), + references: subproblemResponses.map(r => (r.result as AnswerAction).references).flat(), + isFinal: true, + isAggregated: true + } as AnswerAction; + candidateAnswers = subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).filter(a => a) as string[]; - // TODO aggregate images @shazhou2015 + // aggregate urls + visitedURLs.push(...subproblemResponses.map(r => r.readURLs).flat()); + weightedURLs = subproblemResponses.map(r => r.allURLs.map(url => ({ url, title: '' } as BoostedSearchSnippet))).flat(); - // break the loop, jump directly final boxing - break; + // TODO aggregate images @shazhou2015 + + // break the loop, jump directly final boxing + break; + } else { + // if there is only one subproblem, then we skip the recurrsion + gaps.push(subproblems[0]); + } } // rewrite queries with initial soundbites