From 92f1a15f8c587f53036834695139eacb24feebe8 Mon Sep 17 00:00:00 2001
From: Han Xiao <han.xiao@jina.ai>
Date: Wed, 19 Mar 2025 15:20:27 +0800
Subject: [PATCH] feat: filter out blocked content

---
 src/utils/url-tools.ts | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/utils/url-tools.ts b/src/utils/url-tools.ts
index 52f75ab..9e68113 100644
--- a/src/utils/url-tools.ts
+++ b/src/utils/url-tools.ts
@@ -11,7 +11,8 @@ export function normalizeUrl(urlString: string, debug = false, options = {
   removeAnchors: true,
   removeSessionIDs: true,
   removeUTMParams: true,
-  removeTrackingParams: true
+  removeTrackingParams: true,
+  removeXAnalytics: true  // New option to control x.com /analytics removal
 }) {
   try {
     urlString = urlString.replace(/\s+/g, '').trim();
@@ -28,6 +29,20 @@ export function normalizeUrl(urlString: string, debug = false, options = {
       throw new Error('Example URL');
     }
 
+    // Handle x.com and twitter.com URLs with /analytics
+    if (options.removeXAnalytics) {
+      // Match with or without query parameters and fragments
+      const xComPattern = /^(https?:\/\/(www\.)?(x\.com|twitter\.com)\/([^/]+)\/status\/(\d+))\/analytics(\/)?(\?.*)?(#.*)?$/i;
+      const xMatch = urlString.match(xComPattern);
+      if (xMatch) {
+        // Preserve query parameters and fragments if present
+        let cleanUrl = xMatch[1]; // Base URL without /analytics
+        if (xMatch[7]) cleanUrl += xMatch[7]; // Add query parameters if present
+        if (xMatch[8]) cleanUrl += xMatch[8]; // Add fragment if present
+        urlString = cleanUrl;
+      }
+    }
+
     const url = new URL(urlString);
     if (url.protocol !== 'http:' && url.protocol !== 'https:') {
       throw new Error('Unsupported protocol');