github/node-DeepResearch
1
0
Fork 0
mirror of https://github.com/jina-ai/node-DeepResearch.git synced 2025-12-26 06:28:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: filter out blocked content

Browse Source
This commit is contained in:
Han Xiao 2025-03-19 15:20:27 +08:00
parent ef5820729d
commit 92f1a15f8c
1 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/utils/url-tools.ts
View File

@ -11,7 +11,8 @@ export function normalizeUrl(urlString: string, debug = false, options = {
removeAnchors: true,
removeSessionIDs: true,
removeUTMParams: true,
removeTrackingParams: true
removeTrackingParams: true,
removeXAnalytics: true // New option to control x.com /analytics removal
}) {
try {
urlString = urlString.replace(/\s+/g, '').trim();
@ -28,6 +29,20 @@ export function normalizeUrl(urlString: string, debug = false, options = {
throw new Error('Example URL');
}
// Handle x.com and twitter.com URLs with /analytics
if (options.removeXAnalytics) {
// Match with or without query parameters and fragments
const xComPattern = /^(https?:\/\/(www\.)?(x\.com|twitter\.com)\/([^/]+)\/status\/(\d+))\/analytics(\/)?(\?.*)?(#.*)?$/i;
const xMatch = urlString.match(xComPattern);
if (xMatch) {
// Preserve query parameters and fragments if present
let cleanUrl = xMatch[1]; // Base URL without /analytics
if (xMatch[7]) cleanUrl += xMatch[7]; // Add query parameters if present
if (xMatch[8]) cleanUrl += xMatch[8]; // Add fragment if present
urlString = cleanUrl;
}
}
const url = new URL(urlString);
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Unsupported protocol');