fix: chinese references and duplicate eval

src/app.ts

@@ -17,6 +17,7 @@ const app = express();
 // Get secret from command line args for optional authentication
 const secret = process.argv.find(arg => arg.startsWith('--secret='))?.split('=')[1];
 
+
 app.use(cors());
 app.use(express.json({
   limit: '10mb'
@@ -45,13 +46,13 @@ function buildMdFromAnswer(answer: AnswerAction) {
   // No footnotes in answer but we have references - append them at the end
   if (footnotes.length === 0) {
     const appendedCitations = Array.from(
-      { length: answer.references.length },
+      {length: answer.references.length},
       (_, i) => `[^${i + 1}]`
     ).join('');
 
     const references = answer.references.map((ref, i) => {
       const cleanQuote = ref.exactQuote
-        .replace(/[^a-zA-Z0-9]+/g, ' ')
+        .replace(/[^\p{L}\p{N}\s]/gu, ' ').replace(/\s+/, ' ')
         .trim();
 
       if (ref.url.startsWith('http')) {
@@ -71,7 +72,7 @@ ${references}
   // Check if correction is needed
   const needsCorrection = (() => {
     if (footnotes.length === answer.references.length &&
-        footnotes.every(n => n === footnotes[0])) {
+      footnotes.every(n => n === footnotes[0])) {
       return true;
     }
 

src/tools/evaluator.ts

@@ -476,13 +476,26 @@ export async function evaluateAnswer(
         // Safely handle references and ensure we have content
         const urls = action.references?.filter(ref => ref.url.startsWith('http') && !visitedURLs.includes(ref.url)).map(ref => ref.url) || [];
         const uniqueURLs = [...new Set(urls)];
+
+        if (uniqueURLs.length === 0) {
+          // all URLs have been read, or there is no valid urls. no point to read them.
+          result = {
+            object: {
+              pass: true,
+              think: "All provided references have been visited and no new URLs were found to read. The answer is considered valid without further verification.",
+              type: 'attribution',
+            } as EvaluationResponse
+          }
+          break;
+        }
+
         const allKnowledge = await fetchSourceContent(uniqueURLs, trackers);
 
         if (!allKnowledge.trim()) {
           return {
             response: {
               pass: false,
-              think: "The answer does not provide any valid attribution references that could be verified. No accessible source content was found to validate the claims made in the answer.",
+              think: `The answer does provide URL references ${JSON.stringify(uniqueURLs)}, but the content could not be fetched or is empty. Need to found some other references and URLs`,
               type: 'attribution',
             }
           };