github/node-DeepResearch
1
0
Fork 0
mirror of https://github.com/jina-ai/node-DeepResearch.git synced 2026-03-22 07:29:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

revert: no spam filter

Browse Source
This commit is contained in:
Han Xiao
2025-03-19 15:31:42 +08:00
parent d824957d29
commit d5cb62f7ea
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/utils/url-tools.ts
View File

@@ -488,12 +488,12 @@ export async function processURLs(
// check if content is likely a blocked msg from paywall, bot detection, etc. // check if content is likely a blocked msg from paywall, bot detection, etc.
// only check for <5000 char length content as most blocking msg is short // only check for <5000 char length content as most blocking msg is short
// const spamDetectLength = 1000; const spamDetectLength = 300;
// const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content); const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content);
// if (!isGoodContent) { if (!isGoodContent) {
// console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength)); console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength));
// throw new Error(`Blocked content ${url}`); throw new Error(`Blocked content ${url}`);
// } }
// Add to knowledge base // Add to knowledge base
allKnowledge.push({ allKnowledge.push({