diff --git a/.eslintrc.js b/.eslintrc.js index 68212fd..9a720bb 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -14,7 +14,7 @@ module.exports = { sourceType: 'module' }, rules: { - 'no-console': ['error', { allow: ['log', 'error'] }], + 'no-console': ['error', { allow: ['log', 'error', 'warn'] }], '@typescript-eslint/no-var-requires': 'off', '@typescript-eslint/no-explicit-any': 'off' }, diff --git a/src/agent.ts b/src/agent.ts index ff68b02..e9af66c 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -46,7 +46,6 @@ import { finalizeAnswer } from "./tools/finalizer"; import { buildImageReferences, buildReferences } from "./tools/build-ref"; import { logInfo, logError, logDebug, logWarning } from './logging'; import { researchPlan } from './tools/research-planner'; -import { reduceAnswers } from './tools/reducer'; import { AxiosError } from 'axios'; import { dedupImagesWithEmbeddings, filterImages } from './utils/image-tools'; import { serpCluster } from './tools/serp-cluster'; @@ -304,10 +303,11 @@ async function executeSearchQueries( logDebug('Search query:', { query }); switch (searchProvider || SEARCH_PROVIDER) { case 'jina': - case 'arxiv': + case 'arxiv': { const num = meta ? undefined : 30; results = (await search(query, searchProvider, num, meta, context.tokenTracker)).response.results || []; break; + } case 'duck': results = (await duckSearch(query.q, { safeSearch: SafeSearchType.STRICT })).results; break; diff --git a/src/app.ts b/src/app.ts index df75666..3d588de 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,7 +15,7 @@ import { ActionTracker } from "./utils/action-tracker"; import { ObjectGeneratorSafe } from "./utils/safe-generator"; import { jsonSchema } from "ai"; // or another converter library import { normalizeHostName } from "./utils/url-tools"; -import { logInfo, logError, logDebug, logWarning } from './logging'; +import { logInfo, logError, logDebug } from './logging'; import { body, validationResult } from 'express-validator'; const app = express(); diff --git a/src/cli.ts b/src/cli.ts index c0238f8..3e3d634 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,7 +2,7 @@ import { Command } from 'commander'; import { getResponse } from './agent'; import { version } from '../package.json'; -import { logInfo, logError, logDebug, logWarning } from './logging'; +import { logInfo, logError } from './logging'; const program = new Command(); diff --git a/src/config.ts b/src/config.ts index b42c5f2..4cc1431 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,7 @@ import { ProxyAgent, setGlobalDispatcher } from 'undici'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { createOpenAI, OpenAIProviderSettings } from '@ai-sdk/openai'; import configJson from '../config.json'; -import { logInfo, logError, logDebug, logWarning } from './logging'; +import { logInfo, logError } from './logging'; // Load environment variables dotenv.config(); diff --git a/src/evals/batch-evals.ts b/src/evals/batch-evals.ts index 306d011..9e924b2 100644 --- a/src/evals/batch-evals.ts +++ b/src/evals/batch-evals.ts @@ -7,7 +7,7 @@ import { GEMINI_API_KEY } from '../config'; import { z } from 'zod'; import { AnswerAction, TrackerContext } from "../types"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logError } from '../logging'; const execAsync = promisify(exec); diff --git a/src/tools/brave-search.ts b/src/tools/brave-search.ts index a2b0f28..b51c2f6 100644 --- a/src/tools/brave-search.ts +++ b/src/tools/brave-search.ts @@ -1,6 +1,6 @@ import { BRAVE_API_KEY } from "../config"; import axiosClient from "../utils/axios-client"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo } from '../logging'; import { BraveSearchResponse } from '../types'; diff --git a/src/tools/broken-ch-fixer.ts b/src/tools/broken-ch-fixer.ts index db6ba65..261f7af 100644 --- a/src/tools/broken-ch-fixer.ts +++ b/src/tools/broken-ch-fixer.ts @@ -2,7 +2,7 @@ import { generateText } from "ai"; import { getModel } from "../config"; import { TrackerContext } from "../types"; import { detectBrokenUnicodeViaFileIO } from "../utils/text-tools"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logError, logDebug, logWarning } from '../logging'; /** diff --git a/src/tools/code-sandbox.ts b/src/tools/code-sandbox.ts index df6849b..920d0ac 100644 --- a/src/tools/code-sandbox.ts +++ b/src/tools/code-sandbox.ts @@ -1,7 +1,7 @@ import { ObjectGeneratorSafe } from "../utils/safe-generator"; import { CodeGenResponse, PromptPair, TrackerContext } from "../types"; import { Schemas } from "../utils/schemas"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logDebug, logWarning } from '../logging'; interface SandboxResult { diff --git a/src/tools/cosine.ts b/src/tools/cosine.ts index 7c1313f..46c633d 100644 --- a/src/tools/cosine.ts +++ b/src/tools/cosine.ts @@ -1,4 +1,4 @@ -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logWarning } from '../logging'; export function cosineSimilarity(vecA: number[], vecB: number[]): number { if (vecA.length !== vecB.length) { diff --git a/src/tools/embeddings.ts b/src/tools/embeddings.ts index 47e0a94..e593a66 100644 --- a/src/tools/embeddings.ts +++ b/src/tools/embeddings.ts @@ -149,7 +149,7 @@ async function getBatchEmbeddingsWithRetry( } const receivedIndices = new Set(response.data.data.map(item => item.index)); - const dimensionSize = response.data.data[0]?.embedding?.length || options.dimensions || 1024; + // const dimensionSize = response.data.data[0]?.embedding?.length || options.dimensions || 1024; // Process successful embeddings const successfulEmbeddings: number[][] = []; diff --git a/src/tools/error-analyzer.ts b/src/tools/error-analyzer.ts index bbf9d75..4a6dab9 100644 --- a/src/tools/error-analyzer.ts +++ b/src/tools/error-analyzer.ts @@ -1,7 +1,7 @@ import { ErrorAnalysisResponse, PromptPair, TrackerContext } from '../types'; import { ObjectGeneratorSafe } from "../utils/safe-generator"; import { Schemas } from "../utils/schemas"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logError } from '../logging'; function getPrompt(diaryContext: string[]): PromptPair { diff --git a/src/tools/evaluator.ts b/src/tools/evaluator.ts index 6443f4d..3f873f5 100644 --- a/src/tools/evaluator.ts +++ b/src/tools/evaluator.ts @@ -3,7 +3,7 @@ import { AnswerAction, EvaluationResponse, EvaluationType, KnowledgeItem, Prompt import { ObjectGeneratorSafe } from "../utils/safe-generator"; import { Schemas } from "../utils/schemas"; import { getKnowledgeStr } from "../utils/text-tools"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logError } from '../logging'; const TOOL_NAME = 'evaluator'; diff --git a/src/tools/jina-dedup.ts b/src/tools/jina-dedup.ts index 8cc9fd6..6e779fa 100644 --- a/src/tools/jina-dedup.ts +++ b/src/tools/jina-dedup.ts @@ -1,4 +1,4 @@ -import { logDebug, logError, logInfo } from "../logging"; +import { logError, logInfo } from "../logging"; import { TokenTracker } from "../utils/token-tracker"; import { cosineSimilarity } from "./cosine"; import { getEmbeddings } from "./embeddings"; diff --git a/src/tools/jina-rerank.ts b/src/tools/jina-rerank.ts index c4dad26..a491d93 100644 --- a/src/tools/jina-rerank.ts +++ b/src/tools/jina-rerank.ts @@ -1,7 +1,7 @@ import { TokenTracker } from "../utils/token-tracker"; import { JINA_API_KEY } from "../config"; import axiosClient from '../utils/axios-client'; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logError, logDebug } from '../logging'; const JINA_API_URL = 'https://api.jina.ai/v1/rerank'; diff --git a/src/tools/query-rewriter.ts b/src/tools/query-rewriter.ts index a9da757..8b9976e 100644 --- a/src/tools/query-rewriter.ts +++ b/src/tools/query-rewriter.ts @@ -1,7 +1,7 @@ import { PromptPair, SearchAction, SERPQuery, TrackerContext } from '../types'; import { ObjectGeneratorSafe } from "../utils/safe-generator"; import { Schemas } from "../utils/schemas"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logError } from '../logging'; function getPrompt(query: string, think: string, context: string): PromptPair { diff --git a/src/tools/read.ts b/src/tools/read.ts index 50a733a..dc10369 100644 --- a/src/tools/read.ts +++ b/src/tools/read.ts @@ -63,6 +63,7 @@ export async function readUrl( return { response: data }; } catch (error: any) { + logDebug(`Read error: ${error.message}`); throw error; } } \ No newline at end of file diff --git a/src/tools/segment.ts b/src/tools/segment.ts index 770a035..cf3bf3d 100644 --- a/src/tools/segment.ts +++ b/src/tools/segment.ts @@ -22,19 +22,21 @@ export function chunkText(text: string, options: ChunkOptions = {}): { chunks = text.split(/(?<=[.!?。!?])/).filter(chunk => chunk.trim().length > 0); break; - case 'characters': + case 'characters': { const chunkSize = Number(options.value) || 1000; for (let i = 0; i < text.length; i += chunkSize) { chunks.push(text.slice(i, i + chunkSize)); } break; + } - case 'regex': + case 'regex': { if (!options.value || typeof options.value !== 'string') { throw new Error('Regex pattern is required for regex chunking'); } chunks = text.split(new RegExp(options.value)).filter(chunk => chunk.trim().length > 0); break; + } default: throw new Error('Invalid chunking type'); diff --git a/src/tools/serper-search.ts b/src/tools/serper-search.ts index f451c2b..ce3717e 100644 --- a/src/tools/serper-search.ts +++ b/src/tools/serper-search.ts @@ -1,6 +1,5 @@ import { SERPER_API_KEY } from "../config"; import axiosClient from "../utils/axios-client"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; import { SerperSearchResponse, SERPQuery } from '../types'; diff --git a/src/utils/axios-client.ts b/src/utils/axios-client.ts index 45a1e80..16a8381 100644 --- a/src/utils/axios-client.ts +++ b/src/utils/axios-client.ts @@ -1,5 +1,5 @@ import axios, { AxiosRequestConfig } from 'axios'; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logError } from '../logging'; // Default timeout in milliseconds const DEFAULT_TIMEOUT = 30000; diff --git a/src/utils/image-tools.ts b/src/utils/image-tools.ts index 0b4e763..e242e11 100644 --- a/src/utils/image-tools.ts +++ b/src/utils/image-tools.ts @@ -2,7 +2,7 @@ import { getEmbeddings } from '../tools/embeddings'; import { TokenTracker } from './token-tracker'; import { ImageObject, ImageReference } from '../types'; import { cosineSimilarity } from '../tools/cosine'; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo, logError, logWarning } from '../logging'; import sharp from 'sharp'; export const downloadFile = async (uri: string) => { diff --git a/src/utils/text-tools.ts b/src/utils/text-tools.ts index 9a1679e..1e42449 100644 --- a/src/utils/text-tools.ts +++ b/src/utils/text-tools.ts @@ -2,7 +2,7 @@ import { AnswerAction, KnowledgeItem, Reference } from "../types"; import i18nJSON from './i18n.json'; import { JSDOM } from 'jsdom'; import fs from "fs/promises"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logError } from '../logging'; export function buildMdFromAnswer(answer: AnswerAction): string { @@ -916,7 +916,7 @@ export function extractNgrams( // Second pass: calculate PMI and filter const results: NgramResult[] = Array.from(ngramFreq.entries()) - .filter(([ngram, freq]) => freq >= minFreq) + .filter(([, freq]) => freq >= minFreq) .map(([ngram, freq]) => { const pmi = isCJKText(ngram) ? 0 : calculatePMI(ngram, freq, wordFreq, totalNgrams); return { ngram, freq, pmi }; diff --git a/src/utils/token-tracker.ts b/src/utils/token-tracker.ts index 892115f..ef4d35f 100644 --- a/src/utils/token-tracker.ts +++ b/src/utils/token-tracker.ts @@ -2,7 +2,7 @@ import { EventEmitter } from 'events'; import { TokenUsage } from '../types'; import { LanguageModelUsage } from "ai"; -import { logInfo, logError, logDebug, logWarning } from '../logging'; +import { logInfo } from '../logging'; export class TokenTracker extends EventEmitter { private usages: TokenUsage[] = []; diff --git a/src/utils/url-tools.ts b/src/utils/url-tools.ts index 70c60ba..3a37c71 100644 --- a/src/utils/url-tools.ts +++ b/src/utils/url-tools.ts @@ -562,7 +562,7 @@ export async function processURLs( // Process images if (withImages && data.images) { const imageEntries = Object.entries(data.images || {}); - imageEntries.forEach(async ([alt, url]) => { + imageEntries.forEach(async ([, url]) => { const imageObject = await processImage(url, context.tokenTracker); if (imageObject && !imageObjects.find(i => i.url === imageObject.url)) { imageObjects.push(imageObject);