github/node-DeepResearch
1
0
Fork 0
mirror of https://github.com/jina-ai/node-DeepResearch.git synced 2025-12-26 06:28:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

revert: no spam filter

Browse Source
This commit is contained in:
Han Xiao 2025-03-19 15:31:42 +08:00
parent d824957d29
commit d5cb62f7ea
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/utils/url-tools.ts
View File

@ -488,12 +488,12 @@ export async function processURLs(
// check if content is likely a blocked msg from paywall, bot detection, etc.
// only check for <5000 char length content as most blocking msg is short
// const spamDetectLength = 1000;
// const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content);
// if (!isGoodContent) {
// console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength));
// throw new Error(`Blocked content ${url}`);
// }
const spamDetectLength = 300;
const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content);
if (!isGoodContent) {
console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength));
throw new Error(`Blocked content ${url}`);
}
// Add to knowledge base
allKnowledge.push({