mirror of
https://github.com/jina-ai/node-DeepResearch.git
synced 2025-12-26 06:28:56 +08:00
refactor: optimize read and search
This commit is contained in:
Han Xiao
parent
c262f1af93
commit
feb918c7b5
@ -1,88 +1,65 @@
|
||||
import https from 'https';
|
||||
import axios from 'axios';
|
||||
import { TokenTracker } from "../utils/token-tracker";
|
||||
import { SearchResponse } from '../types';
|
||||
import { JINA_API_KEY } from "../config";
|
||||
|
||||
export function search(query: string, tracker?: TokenTracker): Promise<{ response: SearchResponse}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!query.trim()) {
|
||||
reject(new Error('Query cannot be empty'));
|
||||
return;
|
||||
export async function search(
|
||||
query: string,
|
||||
tracker?: TokenTracker
|
||||
): Promise<{ response: SearchResponse }> {
|
||||
if (!query.trim()) {
|
||||
throw new Error('Query cannot be empty');
|
||||
}
|
||||
|
||||
try {
|
||||
const { data } = await axios.get<SearchResponse>(
|
||||
`https://s.jina.ai/?q=${encodeURIComponent(query)}`,
|
||||
{
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': `Bearer ${JINA_API_KEY}`,
|
||||
'X-Respond-With': 'no-content',
|
||||
},
|
||||
timeout: 30000,
|
||||
responseType: 'json'
|
||||
}
|
||||
);
|
||||
|
||||
if (!data.data || !Array.isArray(data.data)) {
|
||||
throw new Error('Invalid response format');
|
||||
}
|
||||
|
||||
const options = {
|
||||
hostname: 's.jina.ai',
|
||||
port: 443,
|
||||
path: `/?q=${encodeURIComponent(query)}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': `Bearer ${JINA_API_KEY}`,
|
||||
'X-Respond-With': 'no-content',
|
||||
const totalTokens = data.data.reduce(
|
||||
(sum, item) => sum + (item.usage?.tokens || 0),
|
||||
0
|
||||
);
|
||||
|
||||
console.log('Total URLs:', data.data.length);
|
||||
|
||||
const tokenTracker = tracker || new TokenTracker();
|
||||
tokenTracker.trackUsage('search', {
|
||||
totalTokens,
|
||||
promptTokens: query.length,
|
||||
completionTokens: totalTokens
|
||||
});
|
||||
|
||||
return { response: data };
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (error.response) {
|
||||
const status = error.response.status;
|
||||
const errorData = error.response.data as any;
|
||||
|
||||
if (status === 402) {
|
||||
throw new Error(errorData?.readableMessage || 'Insufficient balance');
|
||||
}
|
||||
throw new Error(errorData?.readableMessage || `HTTP Error ${status}`);
|
||||
} else if (error.request) {
|
||||
throw new Error('No response received from server');
|
||||
} else {
|
||||
throw new Error(`Request failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const req = https.request(options, (res) => {
|
||||
let responseData = '';
|
||||
|
||||
res.on('data', (chunk) => responseData += chunk);
|
||||
|
||||
res.on('end', () => {
|
||||
// Check HTTP status code first
|
||||
if (res.statusCode && res.statusCode >= 400) {
|
||||
try {
|
||||
// Try to parse error message from response if available
|
||||
const errorResponse = JSON.parse(responseData);
|
||||
if (res.statusCode === 402) {
|
||||
reject(new Error(errorResponse.readableMessage || 'Insufficient balance'));
|
||||
return;
|
||||
}
|
||||
reject(new Error(errorResponse.readableMessage || `HTTP Error ${res.statusCode}`));
|
||||
} catch {
|
||||
// If parsing fails, just return the status code
|
||||
reject(new Error(`HTTP Error ${res.statusCode}`));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Only parse JSON for successful responses
|
||||
let response: SearchResponse;
|
||||
try {
|
||||
response = JSON.parse(responseData) as SearchResponse;
|
||||
} catch (error: unknown) {
|
||||
reject(new Error(`Failed to parse response: ${error instanceof Error ? error.message : 'Unknown error'}`));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.data || !Array.isArray(response.data)) {
|
||||
reject(new Error('Invalid response format'));
|
||||
return;
|
||||
}
|
||||
|
||||
const totalTokens = response.data.reduce((sum, item) => sum + (item.usage?.tokens || 0), 0);
|
||||
console.log('Total URLs:', response.data.length);
|
||||
|
||||
const tokenTracker = tracker || new TokenTracker();
|
||||
tokenTracker.trackUsage('search', {
|
||||
totalTokens,
|
||||
promptTokens: query.length,
|
||||
completionTokens: totalTokens
|
||||
});
|
||||
|
||||
resolve({ response });
|
||||
});
|
||||
});
|
||||
|
||||
// Add timeout handling
|
||||
req.setTimeout(30000, () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timed out'));
|
||||
});
|
||||
|
||||
req.on('error', (error) => {
|
||||
reject(new Error(`Request failed: ${error.message}`));
|
||||
});
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -1,105 +1,86 @@
|
||||
import https from 'https';
|
||||
import {TokenTracker} from "../utils/token-tracker";
|
||||
import {ReadResponse} from '../types';
|
||||
import {JINA_API_KEY} from "../config";
|
||||
import axios from 'axios';
|
||||
import { TokenTracker } from "../utils/token-tracker";
|
||||
import { ReadResponse } from '../types';
|
||||
import { JINA_API_KEY } from "../config";
|
||||
|
||||
export function readUrl(url: string, withAllLinks?: boolean, tracker?: TokenTracker): Promise<{ response: ReadResponse }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!url.trim()) {
|
||||
reject(new Error('URL cannot be empty'));
|
||||
return;
|
||||
}
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
reject(new Error('Invalid URL, only http and https URLs are supported'));
|
||||
return;
|
||||
export async function readUrl(
|
||||
url: string,
|
||||
withAllLinks?: boolean,
|
||||
tracker?: TokenTracker
|
||||
): Promise<{ response: ReadResponse }> {
|
||||
if (!url.trim()) {
|
||||
throw new Error('URL cannot be empty');
|
||||
}
|
||||
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
throw new Error('Invalid URL, only http and https URLs are supported');
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': `Bearer ${JINA_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
'X-Retain-Images': 'none',
|
||||
'X-Md-Link-Style': 'discarded',
|
||||
};
|
||||
|
||||
if (withAllLinks) {
|
||||
headers['X-With-Links-Summary'] = 'all';
|
||||
}
|
||||
|
||||
try {
|
||||
// Use axios which handles encoding properly
|
||||
const { data } = await axios.post<ReadResponse>(
|
||||
'https://r.jina.ai/',
|
||||
{ url },
|
||||
{
|
||||
headers,
|
||||
timeout: 60000,
|
||||
responseType: 'json'
|
||||
}
|
||||
);
|
||||
|
||||
if (!data.data) {
|
||||
throw new Error('Invalid response data');
|
||||
}
|
||||
|
||||
const data = JSON.stringify({url});
|
||||
const headers: Record<string, any> = {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': `Bearer ${JINA_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
'X-Retain-Images': 'none',
|
||||
'X-Md-Link-Style': 'discarded',
|
||||
};
|
||||
if (withAllLinks) {
|
||||
headers['X-With-Links-Summary'] = 'all'
|
||||
console.log('Read:', {
|
||||
title: data.data.title,
|
||||
url: data.data.url,
|
||||
tokens: data.data.usage?.tokens || 0
|
||||
});
|
||||
|
||||
const tokens = data.data.usage?.tokens || 0;
|
||||
const tokenTracker = tracker || new TokenTracker();
|
||||
tokenTracker.trackUsage('read', {
|
||||
totalTokens: tokens,
|
||||
promptTokens: url.length,
|
||||
completionTokens: tokens
|
||||
});
|
||||
|
||||
return { response: data };
|
||||
} catch (error) {
|
||||
// Handle axios errors with better type safety
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (error.response) {
|
||||
// The request was made and the server responded with a status code
|
||||
// that falls out of the range of 2xx
|
||||
const status = error.response.status;
|
||||
const errorData = error.response.data as any;
|
||||
|
||||
if (status === 402) {
|
||||
throw new Error(errorData?.readableMessage || 'Insufficient balance');
|
||||
}
|
||||
throw new Error(errorData?.readableMessage || `HTTP Error ${status}`);
|
||||
} else if (error.request) {
|
||||
// The request was made but no response was received
|
||||
throw new Error('No response received from server');
|
||||
} else {
|
||||
// Something happened in setting up the request
|
||||
throw new Error(`Request failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const options = {
|
||||
hostname: 'r.jina.ai',
|
||||
port: 443,
|
||||
path: '/',
|
||||
method: 'POST',
|
||||
headers
|
||||
};
|
||||
|
||||
const req = https.request(options, (res) => {
|
||||
let responseData = '';
|
||||
|
||||
res.on('data', (chunk) => responseData += chunk);
|
||||
|
||||
res.on('end', () => {
|
||||
// Check HTTP status code first
|
||||
if (res.statusCode && res.statusCode >= 400) {
|
||||
try {
|
||||
// Try to parse error message from response if available
|
||||
const errorResponse = JSON.parse(responseData);
|
||||
if (res.statusCode === 402) {
|
||||
reject(new Error(errorResponse.readableMessage || 'Insufficient balance'));
|
||||
return;
|
||||
}
|
||||
reject(new Error(errorResponse.readableMessage || `HTTP Error ${res.statusCode}`));
|
||||
} catch (error: unknown) {
|
||||
// If parsing fails, just return the status code
|
||||
reject(new Error(`HTTP Error ${res.statusCode}`));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Only parse JSON for successful responses
|
||||
let response: ReadResponse;
|
||||
try {
|
||||
response = JSON.parse(responseData) as ReadResponse;
|
||||
} catch (error: unknown) {
|
||||
reject(new Error(`Failed to parse response: ${error instanceof Error ? error.message : 'Unknown error'}`));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.data) {
|
||||
reject(new Error('Invalid response data'));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Read:', {
|
||||
title: response.data.title,
|
||||
url: response.data.url,
|
||||
tokens: response.data.usage?.tokens || 0
|
||||
});
|
||||
|
||||
const tokens = response.data.usage?.tokens || 0;
|
||||
const tokenTracker = tracker || new TokenTracker();
|
||||
tokenTracker.trackUsage('read', {
|
||||
totalTokens: tokens,
|
||||
promptTokens: url.length,
|
||||
completionTokens: tokens
|
||||
});
|
||||
|
||||
resolve({response});
|
||||
});
|
||||
});
|
||||
|
||||
// Add timeout handling
|
||||
req.setTimeout(60000, () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timed out'));
|
||||
});
|
||||
|
||||
req.on('error', (error: Error) => {
|
||||
reject(new Error(`Request failed: ${error.message}`));
|
||||
});
|
||||
|
||||
req.write(data);
|
||||
req.end();
|
||||
});
|
||||
// For non-axios errors
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user