replace fetch with axios client in url-tools

src/agent.ts

@@ -1012,6 +1012,8 @@ But unfortunately, you failed to solve the issue. You need to think out of the b
 
   // max return 300 urls
   const returnedURLs = weightedURLs.slice(0, numReturnedURLs).map(r => r.url);
+  // reset all context
+  allContext.length = 0;
   return {
     result: thisStep,
     context,

src/utils/axios-client.ts

@@ -1,5 +1,4 @@
 import axios, { AxiosRequestConfig } from 'axios';  
-// import { JINA_API_KEY, SERPER_API_KEY, BRAVE_API_KEY } from "../config";
 
 // Default timeout in milliseconds  
 const DEFAULT_TIMEOUT = 30000;
@@ -45,7 +44,7 @@ const baseConfig: AxiosRequestConfig = {
   headers: {  
     'Accept': 'application/json',  
     'Content-Type': 'application/json',  
-  }
+  },
 };
 
 // Create a single axios instance with the base configuration  
@@ -54,11 +53,12 @@ const axiosClient = axios.create(baseConfig);
 // Add response interceptor for consistent error handling  
 axiosClient.interceptors.response.use(  
   (response) => response,  
-  (error) => {  
+  (error) => {
+    if (error.code === 'ECONNABORTED') {
+      console.error('Request timed out:', error.message);
+      error.request?.destroy?.();
+    }
     if (axios.isAxiosError(error)) {  
-      if (error.code === 'ECONNABORTED') {
-        error.request?.destroy?.();
-      }
       if (error.response) {  
         const status = error.response.status;  
         const errorData = error.response.data as any;  
@@ -68,13 +68,13 @@ axiosClient.interceptors.response.use(
         }  
         throw new Error(errorData?.readableMessage || `HTTP Error ${status}`);  
       } else if (error.request) {  
-        throw new Error(`No response received from server: ${error.message}`);  
+        throw new Error(`No response received from server`);  
       } else {  
         throw new Error(`Request failed: ${error.message}`);  
       }
     }
     throw error;  
-  }  
+  }
 ); 
 
 export default axiosClient;

src/utils/url-tools.ts

@@ -7,6 +7,7 @@ import {cherryPick} from "../tools/jina-latechunk";
 import {formatDateBasedOnType} from "./date-tools";
 import {classifyText} from "../tools/jina-classify-spam";
 import {segmentText} from "../tools/segment";
+import axiosClient from "./axios-client";
 
 export function normalizeUrl(urlString: string, debug = false, options = {
   removeAnchors: true,
@@ -406,22 +407,11 @@ export async function getLastModified(url: string): Promise<string | undefined>
     // Call the API with proper encoding
     const apiUrl = `https://api-beta-datetime.jina.ai?url=${encodeURIComponent(url)}`;
 
-    // Create an AbortController with a timeout
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 10000);
-
-    const response = await fetch(apiUrl, {
-      signal: controller.signal
+    const response = await axiosClient.get(apiUrl, {
+      timeout: 10000,
     });
 
-    // Clear the timeout to prevent memory leaks
-    clearTimeout(timeoutId);
-
-    if (!response.ok) {
-      throw new Error(`API returned ${response.status}`);
-    }
-
-    const data = await response.json();
+    const data = response.data;
 
     // Return the bestGuess date if available
     if (data.bestGuess && data.confidence >= 70) {
@@ -430,7 +420,7 @@ export async function getLastModified(url: string): Promise<string | undefined>
 
     return undefined;
   } catch (error) {
-    console.error('Failed to fetch last modified date:', error);
+    console.error('Failed to fetch last modified date:');
     return undefined;
   }
 }