refactor: query rewriter

src/agent.ts

@@ -31,7 +31,7 @@ import {
   addToAllURLs,
   rankURLs,
   countUrlParts,
-  getUnvisitedURLs,
+  removeBFromA,
   normalizeUrl, sampleMultinomial,
   weightedURLToString, getLastModified
 } from "./utils/url-tools";
@@ -79,14 +79,6 @@ Using your training data and prior lessons learned, answer the user question wit
   if (knowledge?.length) {
     const knowledgeItems = knowledge
       .map((k, i) => `
-<knowledge-0>
-<question>
-How can I get the last update time of a URL?
-</question>
-<answer>
-Just choose <action-visit> and put URL in it, it will fetch full text and estimate the last update datetime of that URL. 
-</answer>
-</knowledge-0>
 <knowledge-${i + 1}>
 <question>
 ${k.question}
@@ -96,8 +88,9 @@ ${k.answer}
 </answer>
 ${k.updated && k.type === 'url' ? `
 <answer-datetime>
-${k.updated}` : ''}
+${k.updated}
 </answer-datetime>
+` : ''}
 ${k.references && k.type === 'url' ? `
 <url>
 ${k.references[0]}
@@ -110,7 +103,14 @@ ${k.references[0]}
     sections.push(`
 You have successfully gathered some knowledge which might be useful for answering the original question. Here is the knowledge you have gathered so far:
 <knowledge>
-
+<knowledge-0>
+<question>
+How can I get the last update time of a URL?
+</question>
+<answer>
+Just choose <action-visit> and put URL in it, it will fetch full text and estimate the last update datetime of that URL. 
+</answer>
+</knowledge-0>
 ${knowledgeItems}
 
 </knowledge>
@@ -237,12 +237,8 @@ FAILURE IS NOT AN OPTION. EXECUTE WITH EXTREME PREJUDICE! ⚡️
   if (allowReflect) {
     actionSections.push(`
 <action-reflect>
-- Critically examine <question>, <context>, <knowledge>, <bad-attempts>, and <learned-strategy> to identify gaps and the problems. 
+- Think slowly and planning lookahead. Examine <question>, <context>, <knowledge>, <bad-attempts>, and <learned-strategy> to identify knowledge gaps. 
-- Identify gaps and ask key clarifying questions that deeply related to the original question and lead to the answer
+- Reflect the gaps and plan a list key clarifying questions that deeply related to the original question and lead to the answer
-- Ensure each reflection:
- - Cuts to core emotional truths while staying anchored to original <question>
- - Transforms surface-level problems into deeper psychological insights
- - Makes the unconscious conscious
 </action-reflect>
 `);
   }
@@ -353,9 +349,15 @@ export async function getResponse(question?: string,
     // allowRead = allowRead && (Object.keys(allURLs).length > 0);
     if (allURLs && Object.keys(allURLs).length > 0) {
       // rerank urls
-      weightedURLs = rankURLs(getUnvisitedURLs(allURLs, visitedURLs), {
+      weightedURLs = rankURLs(
-        question: currentQuestion
+        removeBFromA(allURLs,
-      }, context);
+          [
+            ...visitedURLs,
+            ...weightedURLs.map(r => r.url).slice(Math.floor(weightedURLs.length * 0.5))]),
+        {
+          question: currentQuestion
+        }, context);
+      console.log('Weighted URLs:', weightedURLs.length);
     }
 
     // allowSearch = allowSearch && (weightedURLs.length < 70);  // disable search when too many urls already
@@ -375,7 +377,7 @@ export async function getResponse(question?: string,
       weightedURLs,
       false,
     );
-    schema = SchemaGen.getAgentSchema(allowReflect, allowRead, allowAnswer, allowSearch, allowCoding, finalAnswerPIP)
+    schema = SchemaGen.getAgentSchema(allowReflect, allowRead, allowAnswer, allowSearch, allowCoding, finalAnswerPIP, currentQuestion)
     const result = await generator.generateObject({
       model: 'agent',
       schema,
@@ -468,10 +470,10 @@ Your journey ends here. You have successfully answered the original question. Co
           thisStep.isFinal = true;
           break
         } else {
+          evaluationMetrics[currentQuestion] = evaluationMetrics[currentQuestion].filter(e => e !== evaluation.type);
           if (evaluation.type === 'strict') {
             finalAnswerPIP = evaluation.improvement_plan || '';
             // remove 'strict' from the evaluation metrics
-            evaluationMetrics[currentQuestion] = evaluationMetrics[currentQuestion].filter(e => e !== 'strict');
           }
           if (badAttempts >= maxBadAttempts) {
             thisStep.isFinal = false;
@@ -500,7 +502,6 @@ ${evaluation.think}
             });
 
             if (errorAnalysis.questionsToAnswer) {
-              // reranker? maybe
               errorAnalysis.questionsToAnswer = chooseK(errorAnalysis.questionsToAnswer, MAX_REFLECT_PER_STEP);
               gaps.push(...errorAnalysis.questionsToAnswer);
               allQuestions.push(...errorAnalysis.questionsToAnswer);
@@ -574,31 +575,35 @@ But then you realized you have asked them before. You decided to to think out of
       thisStep.searchRequests = chooseK((await dedupQueries(thisStep.searchRequests, [], context.tokenTracker)).unique_queries, MAX_QUERIES_PER_STEP);
 
       // rewrite queries
-      let {queries: keywordsQueries} = await rewriteQuery(thisStep, context, SchemaGen);
+      let keywordsQueries = await rewriteQuery(thisStep, context, SchemaGen);
+      const qOnly = keywordsQueries.filter(q => q.q).map(q => q.q)
       // avoid exisitng searched queries
-      keywordsQueries = chooseK((await dedupQueries(keywordsQueries, allKeywords, context.tokenTracker)).unique_queries, MAX_QUERIES_PER_STEP);
+      const uniqQOnly = chooseK((await dedupQueries(qOnly, allKeywords, context.tokenTracker)).unique_queries, MAX_QUERIES_PER_STEP);
+      keywordsQueries = keywordsQueries.filter(q => q.q).filter(q => uniqQOnly.includes(q.q));
 
       let anyResult = false;
 
       if (keywordsQueries.length > 0) {
-        context.actionTracker.trackThink('search_for', SchemaGen.languageCode, {keywords: keywordsQueries.join(', ')});
+        context.actionTracker.trackThink('search_for', SchemaGen.languageCode, {keywords: uniqQOnly.join(', ')});
         for (const query of keywordsQueries) {
-          console.log(`Search query: ${query}`);
 
           let results: SearchResult[] = []
 
           try {
-            let siteQuery = query
+            let siteQuery = query.q;
 
             const topHosts = Object.entries(countUrlParts(
               Object.entries(allURLs).map(([, result]) => result)
             ).hostnameCount).sort((a, b) => b[1] - a[1]);
             console.log(topHosts)
-            if (topHosts.length > 0 && Math.random() < 0.6 && !query.includes('site:')) {
+            if (topHosts.length > 0 && Math.random() < 0.6 && !query.q.includes('site:')) {
               // explore-exploit
-              siteQuery = query + ' site:' + sampleMultinomial(topHosts);
+              siteQuery = query.q + ' site:' + sampleMultinomial(topHosts);
+              query.q = siteQuery;
               console.log('Site query:', siteQuery)
             }
+
+            console.log('Search query:', query);
             switch (SEARCH_PROVIDER) {
               case 'jina':
                 results = (await search(siteQuery, context.tokenTracker)).response?.data || [];
@@ -610,7 +615,7 @@ But then you realized you have asked them before. You decided to to think out of
                 results = (await braveSearch(siteQuery)).response.web?.results || [];
                 break;
               case 'serper':
-                results = (await serperSearch(siteQuery)).response.organic || [];
+                results = (await serperSearch(query)).response.organic || [];
                 break;
               default:
                 results = [];
@@ -619,7 +624,7 @@ But then you realized you have asked them before. You decided to to think out of
               throw new Error('No results found');
             }
           } catch (error) {
-            console.error(`${SEARCH_PROVIDER} search failed for query "${query}":`, error);
+            console.error(`${SEARCH_PROVIDER} search failed for query:`, query, error);
             continue
           } finally {
             await sleep(STEP_SLEEP)
@@ -635,10 +640,10 @@ But then you realized you have asked them before. You decided to to think out of
           minResults.forEach(r => {
             addToAllURLs(r, allURLs);
           });
-          allKeywords.push(query);
+          allKeywords.push(query.q);
 
           allKnowledge.push({
-            question: `What do Internet say about "${query}"?`,
+            question: `What do Internet say about "${query.q}"?`,
             answer: removeHTMLtags(minResults.map(r => r.description).join('; ')),
             type: 'side-info',
             updated: new Date().toISOString()
@@ -718,7 +723,9 @@ You decided to think out of the box or cut from a completely different angle.
                   description: link[0],
                 }
                 // in-page link has lower initial weight comparing to search links
-                addToAllURLs(r, allURLs, 0.1);
+                if (r.url && r.url.startsWith('http')) {
+                  addToAllURLs(r, allURLs, 0.1);
+                }
               })
 
               return {url, result: response};
@@ -827,7 +834,7 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
       true,
     );
 
-    schema = SchemaGen.getAgentSchema(false, false, true, false, false, finalAnswerPIP);
+    schema = SchemaGen.getAgentSchema(false, false, true, false, false, finalAnswerPIP, question);
     const result = await generator.generateObject({
       model: 'agentBeastMode',
       schema,

src/tools/evaluator.ts

@@ -576,7 +576,7 @@ export async function evaluateQuestion(
     if (result.object.needsPlurality) types.push('plurality');
     if (result.object.needsCompleteness) types.push('completeness');
 
-    console.log('Question Metrics:', types);
+    console.log('Question Metrics:', question, types);
     trackers?.actionTracker.trackThink(result.object.think);
 
     // Always evaluate definitive first, then freshness (if needed), then plurality (if needed)

src/tools/query-rewriter.ts

@@ -1,4 +1,4 @@
-import {PromptPair, SearchAction, TrackerContext} from '../types';
+import {PromptPair, SearchAction, SERPQuery, TrackerContext} from '../types';
 import {ObjectGeneratorSafe} from "../utils/safe-generator";
 import {Schemas} from "../utils/schemas";
 
@@ -9,7 +9,7 @@ function getPrompt(query: string, think: string): PromptPair {
   const currentMonth = currentTime.getMonth() + 1;
 
   return {
-    system: `You are an expert search query generator with deep psychological understanding. You optimize user queries by extensively analyzing potential user intents and generating comprehensive search variations.
+    system: `You are an expert search query generator with deep psychological understanding. You optimize user queries by extensively analyzing potential user intents and generating comprehensive search variations that follow the required schema format.
 
 The current time is ${currentTime.toISOString()}. Current year: ${currentYear}, current month: ${currentMonth}.
 
@@ -44,40 +44,33 @@ Generate ONE optimized query from each of these cognitive perspectives:
 
 7. Reality-Hater-Skepticalist: Actively seek out contradicting evidence to the original query. Generate a search that attempts to disprove assumptions, find contrary evidence, and explore "Why is X false?" or "Evidence against X" perspectives.
 
-Ensure each persona contributes exactly ONE high-quality query. These 7 queries will be combined into a final array.
+Ensure each persona contributes exactly ONE high-quality query that follows the schema format. These 7 queries will be combined into a final array.
 </cognitive-personas>
 
 <rules>
-1. Start with deep intent analysis:
+1. Query content rules:
-   - Direct intent (what they explicitly ask)
-   - Implicit intent (what they might actually want)
-   - Related intents (what they might need next)
-   - Prerequisite knowledge (what they need to know first)
-   - Common pitfalls (what they should avoid)
-   - Expert perspectives (what professionals would search for)
-   - Beginner needs (what newcomers might miss)
-   - Alternative approaches (different ways to solve the problem)
-
-2. For each identified intent:
-   - Generate queries in original language
-   - Generate queries in English (if not original)
-   - Generate queries in most authoritative language
-   - Use appropriate operators and filters
-
-3. Query structure rules:
    - Split queries for distinct aspects
    - Add operators only when necessary
    - Ensure each query targets a specific intent
    - Remove fluff words but preserve crucial qualifiers
-   - Keep queries short and keyword-based (2-5 words ideal)
+   - Keep 'q' field short and keyword-based (2-5 words ideal)
+
+2. Schema usage rules:
+   - Always include the 'q' field in every query object (should be the last field listed)
+   - Use 'tbs' for time-sensitive queries (remove time constraints from 'q' field)
+   - Use 'gl' and 'hl' for region/language-specific queries (remove region/language from 'q' field)
+   - Use appropriate language code in 'hl' when using non-English queries
+   - Include 'location' only when geographically relevant
+   - Never duplicate information in 'q' that is already specified in other fields
+   - List fields in this order: tbs, gl, hl, location, q
 
 <query-operators>
-A query can't only have operators; and operators can't be at the start a query;
+For the 'q' field content:
 - +term : must include term; for critical terms that must appear
 - -term : exclude term; exclude irrelevant or ambiguous terms
 - filetype:pdf/doc : specific file type
+Note: A query can't only have operators; and operators can't be at the start of a query
 </query-operators>
-
 </rules>
 
 <examples>
@@ -95,36 +88,89 @@ Input Query: 宝马二手车价格
 现实怀疑论者：主动寻找购买二手宝马的负面证据和后悔案例。
 </think>
 queries: [
-  "二手宝马 维修噩梦 隐藏缺陷",
+  {
-  "宝马各系价格区间 里程对比",
+    "q": "二手宝马 维修噩梦 隐藏缺陷"
-  "二手宝马价格趋势 2018-${currentYear}",
+  },
-  "二手宝马vs奔驰vs丰田 性价比",
+  {
-  "${currentYear}年${currentMonth}月 宝马行情",
+    "q": "宝马各系价格区间 里程对比"
-  "BMW Gebrauchtwagen Probleme",
+  },
-  "二手宝马后悔案例 最差投资"
+  {
+    "tbs": "qdr:y",
+    "q": "二手宝马价格趋势"
+  },
+  {
+    "q": "二手宝马vs奔驰vs丰田 性价比"
+  },
+  {
+    "tbs": "qdr:m",
+    "q": "宝马行情"
+  },
+  {
+    "gl": "de",
+    "hl": "de",
+    "q": "BMW Gebrauchtwagen Probleme"
+  },
+  {
+    "q": "二手宝马后悔案例 最差投资"
+  }
 ]
 </example-1>
 
 <example-2>
-Input Query: Python Django authentication best practices
+Input Query: sustainable regenerative agriculture soil health restoration techniques
 <think>
-Surface intent is finding Django authentication best practices, practical intent is implementing a secure reliable authentication system. Emotional intent involves anxiety about making security mistakes that damage professional reputation. Social intent is to demonstrate competence to colleagues and senior developers. Identity intent is to be a responsible engineer following best practices. Taboo intent may include lacking knowledge of underlying authentication mechanisms or rushing implementation to meet deadlines. Shadow intent might involve impostor syndrome, using research to procrastinate coding, or hoping to find perfect solutions without understanding principles.
+Surface intent is to find techniques for restoring soil health through regenerative agriculture practices. Practical intent includes implementing these methods on a farm or garden to improve crop yields and sustainability. Emotional intent may involve anxiety about climate change and environmental degradation, along with hope for solutions. Social intent could include wanting to connect with the regenerative farming community or appear knowledgeable among environmentally-conscious peers. Identity intent relates to seeing oneself as an environmental steward or innovative farmer. Taboo intent might involve seeking ways to bypass regulations or avoid conventional farming practices without facing social judgment. Shadow intent could include displacement activity—researching rather than implementing changes—or seeking validation for convictions about industrial farming's harmfulness.
 
-Expert Skeptic: Challenge Django auth security by finding known vulnerabilities and weaknesses.
+Expert Skeptic: Examine the limitations, failures, and potential negative consequences of regenerative agriculture techniques.
-Detail Analyst: Dig into exact technical parameters and configuration specifics of the auth system.
+Detail Analyst: Investigate specific soil biome metrics, carbon sequestration measurements, and implementation parameters for different techniques.
-Historical Researcher: Study the evolution history of Django authentication and deprecated features.
+Historical Researcher: Explore traditional indigenous land management practices that preceded modern regenerative agriculture concepts.
-Comparative Thinker: Compare Django auth with alternative authentication methods and tradeoffs.
+Comparative Thinker: Compare effectiveness and ROI of different soil restoration approaches across various climate zones and soil types.
-Temporal Context: Ensure getting the latest ${currentYear} security best practices.
+Temporal Context: Find the most recent ${currentYear} research trials and field studies on innovative soil restoration methods.
-Globalizer: Django documentation is primarily English-based, so query official documentation source.
+Globalizer: Look for techniques developed in regions with longstanding sustainable agriculture traditions like Austria's alpine farming or Australia's dryland farming innovations.
-Reality-Hater-Skepticalist: Explore fundamental flaws that might exist in Django's built-in auth.
+Reality-Hater-Skepticalist: Search for evidence that regenerative agriculture's benefits are overstated or cannot scale to commercial agriculture needs.
 </think>
 queries: [
-  "Django authentication vulnerabilities exploits",
+  {
-  "Django AUTH_PASSWORD_VALIDATORS specification",
+    "tbs": "qdr:y",
-  "Django authentication deprecation timeline",
+    "gl": "us",
-  "Django auth vs OAuth vs JWT",
+    "location": "Fort Collins",
-  "Django ${currentYear} security updates",
+    "q": "regenerative agriculture soil failures limitations"
-  "Django built-in auth limitations problems"
+  },
+  {
+    "gl": "us",
+    "location": "Ithaca",
+    "q": "mycorrhizal fungi quantitative sequestration metrics"
+  },
+  {
+    "tbs": "qdr:y",
+    "gl": "au",
+    "location": "Perth",
+    "q": "aboriginal firestick farming soil restoration"
+  },
+  {
+    "gl": "uk",
+    "hl": "en",
+    "location": "Totnes",
+    "q": "comparison no-till vs biochar vs compost tea"
+  },
+  {
+    "tbs": "qdr:m",
+    "gl": "us",
+    "location": "Davis",
+    "q": "soil microbial inoculants research trials"
+  },
+  {
+    "gl": "at",
+    "hl": "de",
+    "location": "Graz",
+    "q": "Humusaufbau Alpenregion Techniken"
+  },
+  {
+    "tbs": "qdr:m",
+    "gl": "ca",
+    "location": "Guelph",
+    "q": "regenerative agriculture exaggerated claims evidence"
+  }
 ]
 </example-2>
 
@@ -142,16 +188,44 @@ Input Query: KIリテラシー向上させる方法
 現実否定的懐疑論者：AIリテラシー向上が無意味である可能性を探る。
 </think>
 queries: [
-  "AI技術 限界 誇大宣伝",
+  {
-  "AIリテラシー 学習ステップ 体系化",
+    "hl": "ja",
-  "AI歴史 失敗事例 教訓",
+    "q": "AI技術 限界 誇大宣伝"
-  "AIリテラシー vs プログラミング vs 批判思考",
+  },
-  "${currentYear}AI最新トレンド 必須スキル",
+  {
-  "artificial intelligence literacy fundamentals",
+    "gl": "jp",
-  "AIリテラシー向上 無意味 理由"
+    "hl": "ja",
+    "q": "AIリテラシー 学習ステップ 体系化"
+  },
+  {
+    "tbs": "qdr:y",
+    "hl": "ja",
+    "q": "AI歴史 失敗事例 教訓"
+  },
+  {
+    "hl": "ja",
+    "q": "AIリテラシー vs プログラミング vs 批判思考"
+  },
+  {
+    "tbs": "qdr:m",
+    "hl": "ja",
+    "q": "AI最新トレンド 必須スキル"
+  },
+  {
+    "gl": "us",
+    "hl": "en",
+    "q": "artificial intelligence literacy fundamentals"
+  },
+  {
+    "hl": "ja",
+    "q": "AIリテラシー向上 無意味 理由"
+  }
 ]
 </example-3>
-</examples>`,
+</examples>
+
+Each generated query must follow JSON schema format. Add 'tbs' if the query is time-sensitive. 
+`,
     user: `
 ${query}
 
@@ -159,13 +233,12 @@ ${query}
 `
   };
 }
-
 const TOOL_NAME = 'queryRewriter';
 
-export async function rewriteQuery(action: SearchAction, trackers: TrackerContext, schemaGen: Schemas): Promise<{ queries: string[] }> {
+export async function rewriteQuery(action: SearchAction, trackers: TrackerContext, schemaGen: Schemas): Promise<SERPQuery[] > {
   try {
     const generator = new ObjectGeneratorSafe(trackers.tokenTracker);
-    const allQueries = [...action.searchRequests];
+    const allQueries = [] as SERPQuery[];
 
     const queryPromises = action.searchRequests.map(async (req) => {
       const prompt = getPrompt(req, action.think);
@@ -182,7 +255,7 @@ export async function rewriteQuery(action: SearchAction, trackers: TrackerContex
     const queryResults = await Promise.all(queryPromises);
     queryResults.forEach(queries => allQueries.push(...queries));
     console.log(TOOL_NAME, allQueries);
-    return {queries: allQueries};
+    return allQueries;
   } catch (error) {
     console.error(`Error in ${TOOL_NAME}`, error);
     throw error;

src/tools/read.ts

@@ -16,6 +16,7 @@ export function readUrl(url: string, withAllLinks?: boolean, tracker?: TokenTrac
         'Authorization': `Bearer ${JINA_API_KEY}`,
         'Content-Type': 'application/json',
         'X-Retain-Images': 'none',
+        'X-Timeout': 15,
       };
     if (withAllLinks) {
       headers['X-With-Links-Summary'] = 'all'

src/tools/serper-search.ts

@@ -1,9 +1,27 @@
 import axios from 'axios';
 import { SERPER_API_KEY } from "../config";
 
-import { SerperSearchResponse } from '../types';
+import {SerperSearchResponse, SERPQuery} from '../types';
 
-export async function serperSearch(query: string): Promise<{ response: SerperSearchResponse }> {
+
+export async function serperSearch(query: SERPQuery): Promise<{ response: SerperSearchResponse }> {
+    const response = await axios.post<SerperSearchResponse>('https://google.serper.dev/search', {
+        ...query,
+        autocorrect: false,
+    }, {
+        headers: {
+            'X-API-KEY': SERPER_API_KEY,
+            'Content-Type': 'application/json'
+        },
+        timeout: 10000
+    });
+
+    // Maintain the same return structure as the original code
+    return { response: response.data };
+}
+
+
+export async function serperSearchOld(query: string): Promise<{ response: SerperSearchResponse }> {
     const response = await axios.post<SerperSearchResponse>('https://google.serper.dev/search', {
         q: query,
         autocorrect: false,

src/types.ts

@@ -1,11 +1,19 @@
 // Action Types
-import { CoreMessage, LanguageModelUsage} from "ai";
+import {CoreMessage, LanguageModelUsage} from "ai";
 
 type BaseAction = {
   action: "search" | "answer" | "reflect" | "visit" | "coding";
   think: string;
 };
 
+export type SERPQuery = {
+  q: string,
+  hl?: string,
+  gl?: string,
+  location?: string,
+  tbs?: string,
+}
+
 export type SearchAction = BaseAction & {
   action: "search";
   searchRequests: string[];
@@ -148,7 +156,7 @@ export type EvaluationResponse = {
     minimum_count_required: number;
     actual_count_provided: number;
   };
-  exactQuote? : string;
+  exactQuote?: string;
   completeness_analysis?: {
     aspects_expected: string,
     aspects_provided: string,

src/utils/schemas.ts

@@ -123,8 +123,14 @@ export class Schemas {
   getQueryRewriterSchema(): z.ZodObject<any> {
     return z.object({
       think: z.string().describe(`Explain why you choose those search queries. ${this.getLanguagePrompt()}`).max(500),
-      queries: z.array(z.string().describe('keyword-based search query, 2-3 words preferred, total length < 30 characters'))
+      queries: z.array(
-        .min(1)
+        z.object({
+          tbs: z.enum(['qdr:h', 'qdr:d', 'qdr:w', 'qdr:m', 'qdr:y']).optional().describe('time-based search filter, must use this field if the search request asks for latest info. qdr:h for past hour, qdr:d for past 24 hours, qdr:w for past week, qdr:m for past month, qdr:y for past year. Choose exactly one.'),
+          gl: z.string().describe('defines the country to use for the search. a two-letter country code. e.g., us for the United States, uk for United Kingdom, or fr for France.'),
+          hl: z.string().describe('the language to use for the search. a two-letter language code. e.g., en for English, es for Spanish, or fr for French.'),
+          location: z.string().describe('defines from where you want the search to originate. It is recommended to specify location at the city level in order to simulate a real user’s search.').optional(),
+          q: z.string().describe('keyword-based search query, 2-3 words preferred, total length < 30 characters').max(50),
+        }))
         .max(MAX_QUERIES_PER_STEP)
         .describe(`'Array of search keywords queries, orthogonal to each other. Maximum ${MAX_QUERIES_PER_STEP} queries allowed.'`)
     });
@@ -193,7 +199,8 @@ export class Schemas {
     }
   }
 
-  getAgentSchema(allowReflect: boolean, allowRead: boolean, allowAnswer: boolean, allowSearch: boolean, allowCoding: boolean, finalAnswerPIP?: string) {
+  getAgentSchema(allowReflect: boolean, allowRead: boolean, allowAnswer: boolean, allowSearch: boolean, allowCoding: boolean,
+                 finalAnswerPIP?: string, currentQuestion?: string): z.ZodObject<any> {
     const actionSchemas: Record<string, z.ZodObject<any>> = {};
 
     if (allowSearch) {
@@ -204,7 +211,6 @@ export class Schemas {
             .max(30)
             .describe(`A natual language search request in ${this.languageStyle}. Based on the deep intention behind the original question and the expected answer format.`))
           .describe(`Required when action='search'. Always prefer a single request, only add another request if the original question covers multiple aspects or elements and one search request is definitely not enough, each request focus on one specific aspect of the original question. Minimize mutual information between each request. Maximum ${MAX_QUERIES_PER_STEP} search requests.`)
-          .min(1)
           .max(MAX_QUERIES_PER_STEP)
       });
     }
@@ -241,9 +247,15 @@ export class Schemas {
     if (allowReflect) {
       actionSchemas.reflect = z.object({
         questionsToAnswer: z.array(
-          z.string().describe("each question must be a single line, Questions must be: Original (not variations of existing questions); Focused on single concepts; Under 20 words; Non-compound/non-complex")
+          z.string().describe(`
+Ensure each reflection question:
+ - Cuts to core emotional truths while staying anchored to the original question
+ - Transforms surface-level problems into deeper psychological insights
+ - Makes the unconscious conscious
+ - NEVER pose general questions like: "How can I verify the accuracy of information before including it in my answer?", "What information was actually contained in the URLs I found?"         
+          `)
         ).max(MAX_REFLECT_PER_STEP)
-          .describe(`Required when action='reflect'. List of most important questions to fill the knowledge gaps of finding the answer to the original question. Maximum provide ${MAX_REFLECT_PER_STEP} reflect questions.`)
+          .describe(`Required when action='reflect'. Reflection and planing, generate a list of most important questions to fill the knowledge gaps to the original question ${currentQuestion}. Maximum provide ${MAX_REFLECT_PER_STEP} reflect questions.`)
       });
     }
 
@@ -257,7 +269,7 @@ export class Schemas {
 
     // Create an object with action as a string literal and exactly one action property
     return z.object({
-      think: z.string().describe(`Articulate your strategic reasoning process: (1) What specific information is still needed? (2) Why is this action most likely to provide that information? (3) What alternatives did you consider and why were they rejected? (4) How will this action advance toward the complete answer? Be concise yet thorough in ${this.getLanguagePrompt()}.`).max(500),
+      think: z.string().describe(`Concisely explain your reasoning process: (1) What specific information is still needed? (2) Why is this action most likely to provide that information? (3) What alternatives did you consider and why were they rejected? (4) How will this action advance toward the complete answer? Be extremely concise and in ${this.getLanguagePrompt()}.`).max(500),
       action: z.enum(Object.keys(actionSchemas).map(key => key) as [string, ...string[]])
         .describe("Choose exactly one best action from the available actions"),
       ...actionSchemas,

src/utils/url-tools.ts

@@ -132,7 +132,7 @@ export function normalizeUrl(urlString: string, debug = false, options = {
   }
 }
 
-export function getUnvisitedURLs(allURLs: Record<string, SearchSnippet>, visitedURLs: string[]): SearchSnippet[] {
+export function removeBFromA(allURLs: Record<string, SearchSnippet>, visitedURLs: string[]): SearchSnippet[] {
   return Object.entries(allURLs)
     .filter(([url]) => !visitedURLs.includes(url))
     .map(([, result]) => result);
@@ -290,7 +290,7 @@ export const weightedURLToString = (allURLs: BoostedSearchSnippet[], maxURLs = 7
     .filter(item => item.merged !== '' && item.merged !== undefined && item.merged !== null)
     .sort((a, b) => (b.score || 0) - (a.score || 0))
     .slice(0, maxURLs)
-    .map(item => `  + weight: ${item.score.toFixed(2)} "${item.url}": "${item.merged}"`)
+    .map(item => `  + weight: ${item.score.toFixed(2)} "${item.url}"`)
     .join('\n');
 }