From d5cb62f7eabacfe467415c0576fd937828490b51 Mon Sep 17 00:00:00 2001
From: Han Xiao <han.xiao@jina.ai>
Date: Wed, 19 Mar 2025 15:31:42 +0800
Subject: [PATCH] revert: no spam filter

---
 src/utils/url-tools.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/utils/url-tools.ts b/src/utils/url-tools.ts
index e9f7ac6..80f5961 100644
--- a/src/utils/url-tools.ts
+++ b/src/utils/url-tools.ts
@@ -488,12 +488,12 @@ export async function processURLs(
 
         // check if content is likely a blocked msg from paywall, bot detection, etc.
         // only check for <5000 char length content as most blocking msg is short
-        // const spamDetectLength = 1000;
-        // const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content);
-        // if (!isGoodContent) {
-        //   console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength));
-        //   throw new Error(`Blocked content ${url}`);
-        // }
+        const spamDetectLength = 300;
+        const isGoodContent = data.content.length > spamDetectLength || await classifyText(data.content);
+        if (!isGoodContent) {
+          console.error(`Blocked content ${data.content.length}:`, url, data.content.slice(0, spamDetectLength));
+          throw new Error(`Blocked content ${url}`);
+        }
 
         // Add to knowledge base
         allKnowledge.push({