From ed5ba0f4fd77e1c3bc1c5f4ce82d69381f0f3f16 Mon Sep 17 00:00:00 2001
From: Han Xiao <han.xiao@jina.ai>
Date: Tue, 11 Mar 2025 15:37:36 +0800
Subject: [PATCH] fix: eval

---
 src/agent.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/agent.ts b/src/agent.ts
index e82dd40..d0c1532 100644
--- a/src/agent.ts
+++ b/src/agent.ts
@@ -330,11 +330,14 @@ export async function getResponse(question?: string,
     console.log('Gaps:', gaps);
     allowReflect = allowReflect && (gaps.length <= 1);
     const currentQuestion: string = gaps.length > 0 ? gaps.shift()! : question
-    if (!evaluationMetrics[currentQuestion]) {
+    // if (!evaluationMetrics[currentQuestion]) {
+    //   evaluationMetrics[currentQuestion] =
+    //     await evaluateQuestion(currentQuestion, context, SchemaGen)
+    // }
+    if (currentQuestion.trim() === question && step === 1) {
+      // only add evaluation for initial question, once at step 1
       evaluationMetrics[currentQuestion] =
         await evaluateQuestion(currentQuestion, context, SchemaGen)
-    }
-    if (currentQuestion.trim() === question && !evaluationMetrics[currentQuestion].includes('strict') && step === 1) {
       // force strict eval for the original question, only once.
       evaluationMetrics[currentQuestion].push('strict')
     }