From 6fbbb5301e255c73544cfafbed39679f080b7956 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:31:47 +0000 Subject: [PATCH] fix: add content length token tracking for URL content Co-Authored-By: sha.zhou@jina.ai --- src/tools/read.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/tools/read.ts b/src/tools/read.ts index 5ccaaf4..374dbad 100644 --- a/src/tools/read.ts +++ b/src/tools/read.ts @@ -70,11 +70,20 @@ export function readUrl(url: string, tracker?: TokenTracker): Promise<{ response tokens: response.data.usage?.tokens || 0 }); - const tokens = response.data.usage?.tokens || 0; - const tokenTracker = tracker || new TokenTracker(); - tokenTracker.trackUsage('read', tokens); + const apiTokens = response.data.usage?.tokens || 0; + + if (tracker) { + // Track API response tokens + tracker.trackUsage('read_api', apiTokens); + + // Track content length tokens using the same estimation method + if (response.data.content) { + const contentTokens = Math.ceil(Buffer.byteLength(response.data.content, 'utf-8') / 4); + tracker.trackUsage('read_content', contentTokens); + } + } - resolve({ response, tokens }); + resolve({ response, tokens: apiTokens }); }); }); @@ -95,4 +104,4 @@ export function readUrl(url: string, tracker?: TokenTracker): Promise<{ response export function removeAllLineBreaks(text: string) { return text.replace(/(\r\n|\n|\r)/gm, " "); -} \ No newline at end of file +}