Merge branch 'main' of github.com:jina-ai/node-DeepResearch

jina-ai/config.json

@@ -10,7 +10,7 @@
     "DEFAULT_MODEL_NAME": ""
   },
   "defaults": {
-    "search_provider": "serper",
+    "search_provider": "jina",
     "llm_provider": "vertex",
     "step_sleep": 500
   },

jina-ai/src/patch-express.ts

@@ -110,7 +110,7 @@ export const jinaAiMiddleware = (req: Request, res: Response, next: NextFunction
             } else {
                 rateLimitPolicy = [
                     RateLimitDesc.from({
-                        occurrence: 2,
+                        occurrence: 1,
                         periodSeconds: 60
                     })
                 ]

src/agent.ts

@@ -302,7 +302,7 @@ async function executeSearchQueries(
       console.log('Search query:', query);
       switch (SEARCH_PROVIDER) {
         case 'jina':
-          results = (await search(query.q, context.tokenTracker)).response?.data || [];
+          results = (await search(query, context.tokenTracker)).response?.data || [];
           break;
         case 'duck':
           results = (await duckSearch(query.q, {safeSearch: SafeSearchType.STRICT})).results;
@@ -1003,11 +1003,7 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
     await updateReferences(answerStep, allURLs)
     answerStep.mdAnswer = repairMarkdownFootnotesOuter(buildMdFromAnswer(answerStep));
   } else {
-    answerStep.mdAnswer =
+    answerStep.mdAnswer = buildMdFromAnswer(answerStep);
-      convertHtmlTablesToMd(
-        fixCodeBlockIndentation(
-          buildMdFromAnswer(answerStep))
-      );
   }
 
   console.log(thisStep)

src/tools/jina-search.ts

@@ -1,26 +1,23 @@
 import axios from 'axios';
 import { TokenTracker } from "../utils/token-tracker";
-import { SearchResponse } from '../types';
+import { SearchResponse, SERPQuery } from '../types';
 import { JINA_API_KEY } from "../config";
 
 export async function search(
-  query: string,
+  query: SERPQuery,
   tracker?: TokenTracker
 ): Promise<{ response: SearchResponse }> {
-  if (!query.trim()) {
-    throw new Error('Query cannot be empty');
-  }
-
   try {
-    const { data } = await axios.get<SearchResponse>(
+    const { data } = await axios.post<SearchResponse>(
-      `https://s.jina.ai/?q=${encodeURIComponent(query)}`,
+      `https://s.jina.ai/`,
+      query,
       {
         headers: {
           'Accept': 'application/json',
           'Authorization': `Bearer ${JINA_API_KEY}`,
           'X-Respond-With': 'no-content',
         },
-        timeout: 30000,
+        timeout: 10000,
         responseType: 'json'
       }
     );
@@ -39,7 +36,7 @@ export async function search(
     const tokenTracker = tracker || new TokenTracker();
     tokenTracker.trackUsage('search', {
       totalTokens,
-      promptTokens: query.length,
+      promptTokens: query.q.length,
       completionTokens: totalTokens
     });
 

src/tools/query-rewriter.ts

@@ -56,11 +56,9 @@ Leverage the soundbites from the context user provides to generate queries that
 2. Schema usage rules:
    - Always include the 'q' field in every query object (should be the last field listed)
    - Use 'tbs' for time-sensitive queries (remove time constraints from 'q' field)
-   - Use 'gl' and 'hl' for region/language-specific queries (remove region/language from 'q' field)
-   - Use appropriate language code in 'hl' when using non-English queries
    - Include 'location' only when geographically relevant
    - Never duplicate information in 'q' that is already specified in other fields
-   - List fields in this order: tbs, gl, hl, location, q
+   - List fields in this order: tbs, location, q
 
 <query-operators>
 For the 'q' field content:
@@ -98,8 +96,6 @@ queries: [
     "q": "宝马行情"
   },
   {
-    "gl": "de",
-    "hl": "de",
     "q": "BMW Gebrauchtwagen Probleme"
   },
   {
@@ -127,13 +123,10 @@ queries: [
   },
   {
     "tbs": "qdr:y",
-    "gl": "au",
     "location": "Perth",
     "q": "aboriginal firestick farming soil restoration"
   },
   {
-    "gl": "uk",
-    "hl": "en",
     "location": "Totnes",
     "q": "comparison no-till vs biochar vs compost tea"
   },
@@ -143,14 +136,11 @@ queries: [
     "q": "soil microbial inoculants research trials"
   },
   {
-    "gl": "at",
-    "hl": "de",
     "location": "Graz",
     "q": "Humusaufbau Alpenregion Techniken"
   },
   {
     "tbs": "qdr:m",
-    "gl": "ca",
     "location": "Guelph",
     "q": "regenerative agriculture exaggerated claims evidence"
   }
@@ -166,33 +156,26 @@ AIリテラシー向上させる方法か...なるほど。最近AIがどんど
 </think>
 queries: [
   {
-    "hl": "ja",
     "q": "AI技術 限界 誇大宣伝"
   },
   {
-    "gl": "jp",
-    "hl": "ja",
     "q": "AIリテラシー 学習ステップ 体系化"
   },
   {
     "tbs": "qdr:y",
-    "hl": "ja",
     "q": "AI歴史 失敗事例 教訓"
   },
   {
-    "hl": "ja",
     "q": "AIリテラシー vs プログラミング vs 批判思考"
   },
   {
     "tbs": "qdr:m",
-    "hl": "ja",
     "q": "AI最新トレンド 必須スキル"
   },
   {
     "q": "artificial intelligence literacy fundamentals"
   },
   {
-    "hl": "ja",
     "q": "AIリテラシー向上 無意味 理由"
   }
 ]

src/types.ts

@@ -8,8 +8,6 @@ type BaseAction = {
 
 export type SERPQuery = {
   q: string,
-  hl?: string,
-  gl?: string,
   location?: string,
   tbs?: string,
 }

src/utils/schemas.ts

@@ -122,8 +122,6 @@ export class Schemas {
       queries: z.array(
         z.object({
           tbs: z.enum(['qdr:h', 'qdr:d', 'qdr:w', 'qdr:m', 'qdr:y']).describe('time-based search filter, must use this field if the search request asks for latest info. qdr:h for past hour, qdr:d for past 24 hours, qdr:w for past week, qdr:m for past month, qdr:y for past year. Choose exactly one.'),
-          gl: z.string().describe('defines the country to use for the search. a two-letter country code. e.g., us for the United States, uk for United Kingdom, or fr for France.'),
-          hl: z.string().describe('the language to use for the search. a two-letter language code. e.g., en for English, es for Spanish, or fr for French.'),
           location: z.string().describe('defines from where you want the search to originate. It is recommended to specify location at the city level in order to simulate a real user’s search.').optional(),
           q: z.string().describe('keyword-based search query, 2-3 words preferred, total length < 30 characters').max(50),
         }))