github/node-DeepResearch
1
0
Fork 0
mirror of https://github.com/jina-ai/node-DeepResearch.git synced 2026-03-22 07:29:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add content length token tracking for URL content

Browse Source 
Co-Authored-By: sha.zhou@jina.ai <sha.zhou@jina.ai>
This commit is contained in:
Devin AI
2025-02-11 10:31:47 +00:00
parent 5808ae9540
commit 6fbbb5301e
1 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/tools/read.ts
View File

@@ -70,11 +70,20 @@ export function readUrl(url: string, tracker?: TokenTracker): Promise<{ response
tokens: response.data.usage?.tokens || 0 tokens: response.data.usage?.tokens || 0
}); });
const tokens = response.data.usage?.tokens || 0; const apiTokens = response.data.usage?.tokens || 0;
const tokenTracker = tracker || new TokenTracker();
tokenTracker.trackUsage('read', tokens); if (tracker) {
// Track API response tokens
tracker.trackUsage('read_api', apiTokens);
// Track content length tokens using the same estimation method
if (response.data.content) {
const contentTokens = Math.ceil(Buffer.byteLength(response.data.content, 'utf-8') / 4);
tracker.trackUsage('read_content', contentTokens);
}
}
resolve({ response, tokens }); resolve({ response, tokens: apiTokens });
}); });
}); });
@@ -95,4 +104,4 @@ export function readUrl(url: string, tracker?: TokenTracker): Promise<{ response
export function removeAllLineBreaks(text: string) { export function removeAllLineBreaks(text: string) {
return text.replace(/(\r\n|\n|\r)/gm, " "); return text.replace(/(\r\n|\n|\r)/gm, " ");
} }