From 37b71eb86f5344f009fda8434ef5ab2b38deb23f Mon Sep 17 00:00:00 2001
From: Han Xiao <han.xiao@jina.ai>
Date: Fri, 13 Jun 2025 10:25:36 -0700
Subject: [PATCH] fix: dedup references by URL in agent

---
 src/agent.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/agent.ts b/src/agent.ts
index 4ad6b4b..f386b39 100644
--- a/src/agent.ts
+++ b/src/agent.ts
@@ -823,6 +823,9 @@ But then you realized you have asked them before. You decided to to think out of
             isAggregated: true
           } as AnswerAction;
           candidateAnswers = subproblemResponses.map(r => (r.result as AnswerAction).mdAnswer).filter(a => a) as string[];
+          // dedup references by their urls
+          const uniqueURLs = new Set(thisStep.references.map(r => r.url));
+          thisStep.references = Array.from(uniqueURLs).map(url => (thisStep as AnswerAction).references.find(r => r?.url === url)) as Reference[];
 
           // aggregate urls
           visitedURLs.push(...subproblemResponses.map(r => r.readURLs).flat());