github/node-DeepResearch
1
0
Fork 0
mirror of https://github.com/jina-ai/node-DeepResearch.git synced 2025-12-26 06:28:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: late chunking

Browse Source
This commit is contained in:
Han Xiao 2025-03-12 14:21:52 +08:00
parent e10419ee78
commit 5f780435ac
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/tools/jina-latechunk.ts
View File

@ -6,15 +6,15 @@ import {Schemas} from "../utils/schemas";
export async function cherryPick(question: string, longContext: string, options: any = {}, trackers: TrackerContext, schemaGen: Schemas) {
const {
snippetLength = 2000,
numSnippets = 2,
snippetLength = 3000,
numSnippets = Math.max(2, Math.min(5, Math.floor(longContext.length / snippetLength))),
chunkSize = 200,
maxTokensPerRequest = 8192, // Maximum tokens per embedding request
// Rough estimate of tokens per character (can be adjusted based on your text)
tokensPerCharacter = 0.5
} = options;
if (longContext.length < snippetLength * numSnippets) {
if (longContext.length < snippetLength * 2) {
// If the context is shorter than the snippet length, return the whole context
return longContext;
}