github/GitNexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gitnexus config and central ignores implemented. Gitnexus config has reference error, central ignores not tested yet

Browse Source
This commit is contained in:
abhigyantrumio
2025-09-23 05:21:55 +05:30
parent 35b73f1b33
commit 3170fea794
15 changed files with 1076 additions and 664 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2 -31
View File
@@ -1,39 +1,10 @@
# ========================================
# ENGINE CONFIGURATION
# PROCESSING CONFIGURATION
# ========================================
# Default engine to use ('legacy' or 'nextgen')
ENGINE_DEFAULT=nextgen
# Enable/disable engines
ENGINE_LEGACY_ENABLED=true
ENGINE_NEXTGEN_ENABLED=true
# Allow fallback between engines if one fails
ENGINE_ALLOW_FALLBACK=true
# Enable performance monitoring
ENGINE_PERFORMANCE_MONITORING=true
# Auto engine selection based on repository size (experimental)
ENGINE_AUTO_SELECTION=false
# Time threshold before falling back to another engine (ms)
ENGINE_FALLBACK_THRESHOLD_MS=10000
# ========================================
# LEGACY ENGINE CONFIGURATION
# ========================================
# Memory limits for Legacy engine
ENGINE_LEGACY_MEMORY_LIMIT_MB=512
ENGINE_LEGACY_GC_INTERVAL_MS=30000
# Processing settings for Legacy engine
ENGINE_LEGACY_BATCH_SIZE=10
ENGINE_LEGACY_TIMEOUT_MS=30000
ENGINE_LEGACY_USE_WORKERS=true
PROCESSING_PERFORMANCE_MONITORING=true
# ========================================
# PARALLEL PROCESSING CONFIGURATION
gitnexus.config.ts
+307
View File
@@ -0,0 +1,307 @@
/**
* GitNexus Configuration
*
* Centralized configuration file for all GitNexus settings.
* This replaces scattered .env variables and hardcoded values.
*
* Environment variables can still override these values for deployment.
*/
export interface GitNexusConfig {
// ========================================
// PROCESSING CONFIGURATION
// ========================================
processing: {
mode: 'parallel' | 'single';
parallel: {
maxWorkers: number;
batchSize: number;
workerTimeoutMs: number;
};
memory: {
maxMB: number;
cleanupThresholdMB: number;
gcIntervalMs: number;
maxFileSizeMB: number;
maxFilesInMemory: number;
};
fileExtensions: string[];
performanceMonitoring: boolean;
};
// ========================================
// KUZU DB CONFIGURATION
// ========================================
kuzu: {
enabled: boolean;
persistence: boolean;
dualWrite: boolean;
fallbackToJson: boolean;
performance: {
enableCache: boolean;
cacheSize: number;
queryTimeout: number;
};
};
// ========================================
// AI & QUERY CONFIGURATION
// ========================================
ai: {
cypher: {
defaultLimit: number;
maxLimit: number;
timeoutMs: number;
enableValidation: boolean;
enableLimiting: boolean; // Enable/disable automatic LIMIT addition
enableTruncation: boolean; // Enable/disable response truncation
};
llm: {
defaultProvider: 'openai' | 'azure' | 'anthropic' | 'gemini';
providers: {
openai?: {
apiKey?: string;
model: string;
maxTokens: number;
temperature: number;
};
azure?: {
apiKey?: string;
endpoint?: string;
deployment?: string;
maxTokens: number;
temperature: number;
};
anthropic?: {
apiKey?: string;
model: string;
maxTokens: number;
temperature: number;
};
gemini?: {
apiKey?: string;
model: string;
maxTokens: number;
temperature: number;
};
};
};
};
// ========================================
// IGNORE PATTERNS (CENTRALIZED!)
// ========================================
ignore: {
enabled: boolean;
patterns: string[];
suffixes: string[];
fileExtensions: string[];
customPatterns: string[];
};
// ========================================
// LOGGING & DEBUGGING
// ========================================
logging: {
level: 'debug' | 'info' | 'warn' | 'error';
enableMetrics: boolean;
enablePerformance: boolean;
maxEntries: number;
monitoringIntervalMs: number;
};
// ========================================
// GITHUB INTEGRATION
// ========================================
github: {
token?: string;
apiUrl: string;
rateLimit: {
maxRequests: number;
windowMs: number;
};
retry: {
maxRetries: number;
backoffMs: number;
};
};
// ========================================
// ENVIRONMENT & DEPLOYMENT
// ========================================
environment: 'development' | 'staging' | 'production';
}
/**
* Default GitNexus Configuration
*
* These are the default values. Environment variables can override them.
*/
const config: GitNexusConfig = {
// ========================================
// PROCESSING CONFIGURATION
// ========================================
processing: {
mode: (process.env.VITE_PARSING_MODE as 'parallel' | 'single') ?? 'parallel',
parallel: {
maxWorkers: parseInt(process.env.PARALLEL_MAX_WORKERS ?? '4'),
batchSize: parseInt(process.env.PARALLEL_BATCH_SIZE ?? '20'),
workerTimeoutMs: parseInt(process.env.PARALLEL_WORKER_TIMEOUT_MS ?? '60000')
},
memory: {
maxMB: parseInt(process.env.MEMORY_MAX_MB ?? '512'),
cleanupThresholdMB: parseInt(process.env.MEMORY_CLEANUP_THRESHOLD_MB ?? '400'),
gcIntervalMs: parseInt(process.env.MEMORY_GC_INTERVAL_MS ?? '30000'),
maxFileSizeMB: parseInt(process.env.MEMORY_MAX_FILE_SIZE_MB ?? '10'),
maxFilesInMemory: parseInt(process.env.MEMORY_MAX_FILES ?? '1000')
},
fileExtensions: process.env.PROCESSING_FILE_EXTENSIONS?.split(',') ?? [
'.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp',
'.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart',
'.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.cfg', '.properties'
],
performanceMonitoring: process.env.PROCESSING_PERFORMANCE_MONITORING !== 'false'
},
// ========================================
// KUZU DB CONFIGURATION
// ========================================
kuzu: {
enabled: process.env.VITE_KUZU_ENABLED === 'true',
persistence: process.env.KUZU_PERSISTENCE !== 'false',
dualWrite: process.env.KUZU_DUAL_WRITE !== 'false',
fallbackToJson: process.env.KUZU_FALLBACK_JSON !== 'false',
performance: {
enableCache: process.env.KUZU_ENABLE_CACHE !== 'false',
cacheSize: parseInt(process.env.KUZU_CACHE_SIZE ?? '1000'),
queryTimeout: parseInt(process.env.KUZU_QUERY_TIMEOUT ?? '30000')
}
},
// ========================================
// AI & QUERY CONFIGURATION
// ========================================
ai: {
cypher: {
defaultLimit: parseInt(process.env.AI_CYPHER_DEFAULT_LIMIT ?? '20'),
maxLimit: parseInt(process.env.AI_CYPHER_MAX_LIMIT ?? '100'),
timeoutMs: parseInt(process.env.AI_CYPHER_TIMEOUT_MS ?? '30000'),
enableValidation: process.env.AI_CYPHER_ENABLE_VALIDATION !== 'false',
enableLimiting: process.env.AI_CYPHER_ENABLE_LIMITING !== 'false',
enableTruncation: process.env.AI_CYPHER_ENABLE_TRUNCATION !== 'false'
},
llm: {
defaultProvider: (process.env.LLM_DEFAULT_PROVIDER as 'openai' | 'azure' | 'anthropic' | 'gemini') ?? 'openai',
providers: {
openai: process.env.OPENAI_API_KEY ? {
apiKey: process.env.OPENAI_API_KEY,
model: process.env.OPENAI_MODEL ?? 'gpt-4o-mini',
maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS ?? '4000'),
temperature: parseFloat(process.env.OPENAI_TEMPERATURE ?? '0.1')
} : undefined,
azure: process.env.AZURE_API_KEY ? {
apiKey: process.env.AZURE_API_KEY,
endpoint: process.env.AZURE_ENDPOINT,
deployment: process.env.AZURE_DEPLOYMENT,
maxTokens: parseInt(process.env.AZURE_MAX_TOKENS ?? '4000'),
temperature: parseFloat(process.env.AZURE_TEMPERATURE ?? '0.1')
} : undefined,
anthropic: process.env.ANTHROPIC_API_KEY ? {
apiKey: process.env.ANTHROPIC_API_KEY,
model: process.env.ANTHROPIC_MODEL ?? 'claude-3-sonnet-20240229',
maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS ?? '4000'),
temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE ?? '0.1')
} : undefined,
gemini: process.env.GEMINI_API_KEY ? {
apiKey: process.env.GEMINI_API_KEY,
model: process.env.GEMINI_MODEL ?? 'gemini-pro',
maxTokens: parseInt(process.env.GEMINI_MAX_TOKENS ?? '4000'),
temperature: parseFloat(process.env.GEMINI_TEMPERATURE ?? '0.1')
} : undefined
}
}
},
// ========================================
// IGNORE PATTERNS (CENTRALIZED!)
// ========================================
ignore: {
enabled: process.env.IGNORE_ENABLED !== 'false',
patterns: process.env.IGNORE_PATTERNS?.split(',') ?? [
// Version Control
'.git', '.svn', '.hg',
// Package Managers & Dependencies
'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps',
// Python Virtual Environments & Cache
'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__',
'.pytest_cache', '.mypy_cache', '.tox',
// Build & Distribution Directories
'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build',
// Static Assets and Public Directories
'public', 'assets', 'static',
// IDE & Editor Directories
'.vs', '.vscode', '.idea', '.eclipse', '.settings',
// Temporary & Log Directories
'tmp', '.tmp', 'temp', 'logs', 'log',
// Coverage & Testing
'coverage', '.coverage', 'htmlcov', '.nyc_output',
// OS & System
'.DS_Store', 'Thumbs.db',
// Documentation Build Output
'_site', '.docusaurus',
// Cache Directories
'.cache', '.parcel-cache', '.next', '.nuxt'
],
suffixes: process.env.IGNORE_SUFFIXES?.split(',') ?? ['.tmp', '~', '.bak', '.swp', '.swo'],
fileExtensions: process.env.IGNORE_FILE_EXTENSIONS?.split(',') ?? [
// Compiled/Binary
'.pyc', '.pyo', '.pyd', '.so', '.dll', '.exe', '.jar', '.war', '.ear',
// Archives
'.zip', '.tar', '.rar', '.7z', '.gz',
// Media
'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.ico', '.mp4', '.avi', '.mp3', '.wav',
// Documents
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
// Fonts
'.woff', '.woff2', '.ttf', '.eot', '.otf',
// Minified/Generated
'.min.js', '.min.css', '.map'
],
customPatterns: process.env.IGNORE_CUSTOM_PATTERNS?.split(',') ?? []
},
// ========================================
// LOGGING & DEBUGGING
// ========================================
logging: {
level: (process.env.LOG_LEVEL as 'debug' | 'info' | 'warn' | 'error') ?? 'info',
enableMetrics: process.env.LOG_ENABLE_METRICS !== 'false',
enablePerformance: process.env.LOG_ENABLE_PERFORMANCE !== 'false',
maxEntries: parseInt(process.env.LOG_MAX_ENTRIES ?? '1000'),
monitoringIntervalMs: parseInt(process.env.LOG_MONITORING_INTERVAL_MS ?? '30000')
},
// ========================================
// GITHUB INTEGRATION
// ========================================
github: {
token: process.env.GITHUB_TOKEN,
apiUrl: process.env.GITHUB_API_URL ?? 'https://api.github.com',
rateLimit: {
maxRequests: parseInt(process.env.GITHUB_RATE_LIMIT_MAX ?? '60'),
windowMs: parseInt(process.env.GITHUB_RATE_LIMIT_WINDOW_MS ?? '60000')
},
retry: {
maxRetries: parseInt(process.env.GITHUB_RETRY_MAX ?? '3'),
backoffMs: parseInt(process.env.GITHUB_RETRY_BACKOFF_MS ?? '1000')
}
},
// ========================================
// ENVIRONMENT & DEPLOYMENT
// ========================================
environment: (process.env.NODE_ENV as 'development' | 'staging' | 'production') ?? 'development'
};
export default config;
scripts/migrate-config.ts
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env tsx
/**
* Configuration Migration Script
*
* Helps migrate from .env variables to the new gitnexus.config.ts system.
* This script analyzes your current .env file and suggests config updates.
*/
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
interface EnvMapping {
envKey: string;
configPath: string;
transform?: (value: string) => any;
}
const ENV_MAPPINGS: EnvMapping[] = [
// Engine Configuration
{ envKey: 'ENGINE_DEFAULT', configPath: 'engine.default' },
{ envKey: 'ENGINE_LEGACY_ENABLED', configPath: 'engine.legacy.enabled', transform: (v) => v === 'true' },
{ envKey: 'ENGINE_LEGACY_MEMORY_LIMIT_MB', configPath: 'engine.legacy.memoryLimitMB', transform: parseInt },
{ envKey: 'ENGINE_LEGACY_BATCH_SIZE', configPath: 'engine.legacy.batchSize', transform: parseInt },
// Processing Configuration
{ envKey: 'VITE_PARSING_MODE', configPath: 'processing.mode' },
{ envKey: 'PARALLEL_MAX_WORKERS', configPath: 'processing.parallel.maxWorkers', transform: parseInt },
{ envKey: 'PARALLEL_BATCH_SIZE', configPath: 'processing.parallel.batchSize', transform: parseInt },
{ envKey: 'MEMORY_MAX_MB', configPath: 'processing.memory.maxMB', transform: parseInt },
// KuzuDB Configuration
{ envKey: 'VITE_KUZU_ENABLED', configPath: 'kuzu.enabled', transform: (v) => v === 'true' },
// Logging Configuration
{ envKey: 'LOG_LEVEL', configPath: 'logging.level' },
{ envKey: 'LOG_ENABLE_METRICS', configPath: 'logging.enableMetrics', transform: (v) => v === 'true' },
// GitHub Configuration
{ envKey: 'GITHUB_TOKEN', configPath: 'github.token' },
{ envKey: 'GITHUB_API_URL', configPath: 'github.apiUrl' }
];
function parseEnvFile(filePath: string): Record<string, string> {
if (!existsSync(filePath)) {
console.log(`❌ .env file not found at ${filePath}`);
return {};
}
const content = readFileSync(filePath, 'utf-8');
const env: Record<string, string> = {};
content.split('\n').forEach(line => {
line = line.trim();
if (line && !line.startsWith('#')) {
const [key, ...valueParts] = line.split('=');
if (key && valueParts.length > 0) {
env[key.trim()] = valueParts.join('=').trim();
}
}
});
return env;
}
function generateConfigSuggestions(env: Record<string, string>): string {
const suggestions: string[] = [];
const foundMappings: EnvMapping[] = [];
// Find matching environment variables
ENV_MAPPINGS.forEach(mapping => {
if (env[mapping.envKey]) {
foundMappings.push(mapping);
}
});
if (foundMappings.length === 0) {
return 'No matching environment variables found for migration.';
}
suggestions.push('// Suggested updates for gitnexus.config.ts based on your .env file:\n');
foundMappings.forEach(mapping => {
const envValue = env[mapping.envKey];
const transformedValue = mapping.transform ? mapping.transform(envValue) : `'${envValue}'`;
suggestions.push(`// ${mapping.envKey}=${envValue}`);
suggestions.push(`${mapping.configPath}: ${transformedValue},\n`);
});
return suggestions.join('\n');
}
function analyzeCurrentConfig(): void {
console.log('🔍 GitNexus Configuration Migration Analysis\n');
// Check for .env file
const envPath = join(process.cwd(), '.env');
const env = parseEnvFile(envPath);
if (Object.keys(env).length === 0) {
console.log('No .env file found or it\'s empty.');
return;
}
console.log(`📋 Found ${Object.keys(env).length} environment variables in .env\n`);
// Generate suggestions
const suggestions = generateConfigSuggestions(env);
console.log('💡 Configuration Suggestions:');
console.log('=' .repeat(50));
console.log(suggestions);
console.log('=' .repeat(50));
// Check if gitnexus.config.ts exists
const configPath = join(process.cwd(), 'gitnexus.config.ts');
if (existsSync(configPath)) {
console.log('\n✅ gitnexus.config.ts already exists');
console.log('You can update it with the suggestions above.');
} else {
console.log('\n⚠️ gitnexus.config.ts not found');
console.log('Please create it first, then apply the suggestions above.');
}
// Show next steps
console.log('\n📝 Next Steps:');
console.log('1. Update gitnexus.config.ts with the suggested values');
console.log('2. Test your application to ensure everything works');
console.log('3. Consider removing the corresponding .env variables');
console.log('4. Update your deployment scripts to use the new config system');
}
// Run the analysis
if (require.main === module) {
analyzeCurrentConfig();
}
src/ai/react-agent.ts
+14 -8
View File
@@ -4,6 +4,7 @@ import type { LLMService, LLMConfig } from './llm-service.ts';
import type { CypherGenerator } from './cypher-generator.ts';
import type { KnowledgeGraph } from '../core/graph/types.ts';
import type { LocalStorageChatHistory } from '../lib/chat-history.ts';
import { configLoader } from '../config/config-loader.ts';
export interface ReActContext {
graph: KnowledgeGraph;
@@ -380,21 +381,26 @@ CRITICAL: The action field must be exactly one of these four values: query_graph
maxRetries: 3
});
// Add LIMIT if not present to prevent JSON truncation issues
// Get config once for efficiency
const config = await configLoader.loadConfig();
// Add LIMIT if not present and limiting is enabled
let finalCypher = cypherQuery.cypher;
if (!finalCypher.toLowerCase().includes('limit')) {
finalCypher += ' LIMIT 20';
if (config.ai.cypher.enableLimiting && !finalCypher.toLowerCase().includes('limit')) {
const defaultLimit = config.ai.cypher.defaultLimit;
finalCypher += ` LIMIT ${defaultLimit}`;
}
// Execute the query
const results = await this.executeGraphQuery(finalCypher);
// Truncate large responses to prevent JSON parsing issues
if (results.rows && results.rows.length > 20) {
results.rows = results.rows.slice(0, 20);
results.rowCount = 20;
// Truncate large responses if truncation is enabled
if (config.ai.cypher.enableTruncation && results.rows && results.rows.length > config.ai.cypher.maxLimit) {
const maxLimit = config.ai.cypher.maxLimit;
results.rows = results.rows.slice(0, maxLimit);
results.rowCount = maxLimit;
results.truncated = true;
results.summary += ' (showing first 20 results)';
results.summary += ` (showing first ${maxLimit} results)`;
}
output = JSON.stringify(results, null, 2);
src/config/config-loader.ts
+279
View File
@@ -0,0 +1,279 @@
/**
* Configuration Loader
*
* Loads and validates configuration from the root gitnexus.config.ts file
* with Zod schema validation and type safety.
*/
import { z } from 'zod';
// Zod schemas for validation
const ProcessingConfigSchema = z.object({
mode: z.enum(['parallel', 'single']),
parallel: z.object({
maxWorkers: z.number().min(1).max(16),
batchSize: z.number().min(1).max(100),
workerTimeoutMs: z.number().min(10000).max(300000)
}),
memory: z.object({
maxMB: z.number().min(100).max(4096),
cleanupThresholdMB: z.number().min(50).max(2048),
gcIntervalMs: z.number().min(5000).max(60000),
maxFileSizeMB: z.number().min(1).max(100),
maxFilesInMemory: z.number().min(100).max(10000)
}),
fileExtensions: z.array(z.string()),
performanceMonitoring: z.boolean()
});
const KuzuConfigSchema = z.object({
enabled: z.boolean(),
persistence: z.boolean(),
dualWrite: z.boolean(),
fallbackToJson: z.boolean(),
performance: z.object({
enableCache: z.boolean(),
cacheSize: z.number().min(100).max(10000),
queryTimeout: z.number().min(1000).max(60000)
})
});
const AIConfigSchema = z.object({
cypher: z.object({
defaultLimit: z.number().min(1).max(1000),
maxLimit: z.number().min(10).max(1000),
timeoutMs: z.number().min(1000).max(60000),
enableValidation: z.boolean(),
enableLimiting: z.boolean(),
enableTruncation: z.boolean()
}),
llm: z.object({
defaultProvider: z.enum(['openai', 'azure', 'anthropic', 'gemini']),
providers: z.object({
openai: z.object({
apiKey: z.string().optional(),
model: z.string(),
maxTokens: z.number().min(100).max(10000),
temperature: z.number().min(0).max(2)
}).optional(),
azure: z.object({
apiKey: z.string().optional(),
endpoint: z.string().optional(),
deployment: z.string().optional(),
maxTokens: z.number().min(100).max(10000),
temperature: z.number().min(0).max(2)
}).optional(),
anthropic: z.object({
apiKey: z.string().optional(),
model: z.string(),
maxTokens: z.number().min(100).max(10000),
temperature: z.number().min(0).max(2)
}).optional(),
gemini: z.object({
apiKey: z.string().optional(),
model: z.string(),
maxTokens: z.number().min(100).max(10000),
temperature: z.number().min(0).max(2)
}).optional()
})
})
});
const IgnoreConfigSchema = z.object({
enabled: z.boolean(),
patterns: z.array(z.string()),
suffixes: z.array(z.string()),
fileExtensions: z.array(z.string()),
customPatterns: z.array(z.string())
});
const LoggingConfigSchema = z.object({
level: z.enum(['debug', 'info', 'warn', 'error']),
enableMetrics: z.boolean(),
enablePerformance: z.boolean(),
maxEntries: z.number().min(100).max(10000),
monitoringIntervalMs: z.number().min(5000).max(60000)
});
const GitHubConfigSchema = z.object({
token: z.string().optional(),
apiUrl: z.string().url(),
rateLimit: z.object({
maxRequests: z.number().min(1).max(5000),
windowMs: z.number().min(1000).max(3600000)
}),
retry: z.object({
maxRetries: z.number().min(0).max(10),
backoffMs: z.number().min(100).max(10000)
})
});
const GitNexusConfigSchema = z.object({
processing: ProcessingConfigSchema,
kuzu: KuzuConfigSchema,
ai: AIConfigSchema,
ignore: IgnoreConfigSchema,
logging: LoggingConfigSchema,
github: GitHubConfigSchema,
environment: z.enum(['development', 'staging', 'production'])
});
export type ValidatedGitNexusConfig = z.infer<typeof GitNexusConfigSchema>;
/**
* Configuration Loader Service
*/
export class ConfigLoader {
private static instance: ConfigLoader;
private config: ValidatedGitNexusConfig | null = null;
private validationErrors: string[] = [];
private constructor() {}
public static getInstance(): ConfigLoader {
if (!ConfigLoader.instance) {
ConfigLoader.instance = new ConfigLoader();
}
return ConfigLoader.instance;
}
/**
* Load and validate configuration from gitnexus.config.ts
*/
public async loadConfig(): Promise<ValidatedGitNexusConfig> {
if (this.config) {
return this.config;
}
try {
// Import the config file
const configModule = await import('../../gitnexus.config.ts');
const rawConfig = configModule.default;
// Validate with Zod
const result = GitNexusConfigSchema.safeParse(rawConfig);
if (!result.success) {
this.validationErrors = result.error.errors.map(e =>
`${e.path.join('.')}: ${e.message}`
);
console.error('❌ Configuration validation errors:', this.validationErrors);
// Return a minimal valid config as fallback
this.config = this.getMinimalConfig();
} else {
this.config = result.data;
console.log('✅ Configuration loaded and validated successfully');
}
return this.config;
} catch (error) {
console.error('❌ Failed to load gitnexus.config.ts:', error);
this.config = this.getMinimalConfig();
return this.config;
}
}
/**
* Get minimal fallback configuration
*/
private getMinimalConfig(): ValidatedGitNexusConfig {
return {
processing: {
mode: 'parallel',
parallel: {
maxWorkers: 4,
batchSize: 20,
workerTimeoutMs: 60000
},
memory: {
maxMB: 512,
cleanupThresholdMB: 400,
gcIntervalMs: 30000,
maxFileSizeMB: 10,
maxFilesInMemory: 1000
},
fileExtensions: ['.js', '.ts', '.jsx', '.tsx', '.py'],
performanceMonitoring: true
},
kuzu: {
enabled: true,
persistence: true,
dualWrite: true,
fallbackToJson: true,
performance: {
enableCache: true,
cacheSize: 1000,
queryTimeout: 30000
}
},
ai: {
cypher: {
defaultLimit: 20,
maxLimit: 100,
timeoutMs: 30000,
enableValidation: true,
enableLimiting: true,
enableTruncation: true
},
llm: {
defaultProvider: 'openai',
providers: {}
}
},
ignore: {
enabled: true,
patterns: ['node_modules', '.git', 'build', 'dist'],
suffixes: ['.tmp', '~'],
fileExtensions: ['.pyc', '.zip', '.jpg'],
customPatterns: []
},
logging: {
level: 'info',
enableMetrics: true,
enablePerformance: true,
maxEntries: 1000,
monitoringIntervalMs: 30000
},
github: {
apiUrl: 'https://api.github.com',
rateLimit: {
maxRequests: 60,
windowMs: 60000
},
retry: {
maxRetries: 3,
backoffMs: 1000
}
},
environment: 'development'
};
}
/**
* Get current configuration
*/
public getConfig(): ValidatedGitNexusConfig | null {
return this.config;
}
/**
* Get validation errors
*/
public getValidationErrors(): string[] {
return [...this.validationErrors];
}
/**
* Reload configuration (useful for development)
*/
public async reloadConfig(): Promise<ValidatedGitNexusConfig> {
this.config = null;
this.validationErrors = [];
return this.loadConfig();
}
}
// Export singleton instance
export const configLoader = ConfigLoader.getInstance();
src/config/config.ts
+26 -3
View File
@@ -1,9 +1,16 @@
/**
* Centralized Configuration System
* Provides type-safe configuration with environment validation and defaults
* Legacy Configuration System (DEPRECATED)
*
* This file is being replaced by the new centralized config system:
* - Root config: gitnexus.config.ts
* - Config loader: src/config/config-loader.ts
* - Ignore service: src/config/ignore-service.ts
*
* TODO: Migrate remaining usage to the new system
*/
import { z } from 'zod';
import { configLoader, type ValidatedGitNexusConfig } from './config-loader.ts';
// Configuration schemas with validation
const MemoryConfigSchema = z.object({
@@ -99,16 +106,32 @@ export type ProcessingConfig = z.infer<typeof ProcessingConfigSchema>;
export type LoggingConfig = z.infer<typeof LoggingConfigSchema>;
/**
* Configuration service with environment validation
* Configuration service with environment validation (LEGACY)
*
* This is a compatibility layer that bridges to the new centralized config system.
* New code should use configLoader directly.
*/
export class ConfigService {
private static instance: ConfigService;
private config: AppConfig;
private validationErrors: string[] = [];
private newConfig: ValidatedGitNexusConfig | null = null;
private constructor() {
this.config = this.loadConfiguration();
this.validateEnvironment();
this.loadNewConfig();
}
/**
* Load the new centralized configuration
*/
private async loadNewConfig(): Promise<void> {
try {
this.newConfig = await configLoader.loadConfig();
} catch (error) {
console.warn('Failed to load new config system, using legacy config:', error);
}
}
public static getInstance(): ConfigService {
src/config/ignore-service.ts
+284
View File
@@ -0,0 +1,284 @@
/**
* Centralized Ignore Service
*
* Provides robust file and directory filtering based on centralized configuration.
* Implements the same logic as the Python example with directory-component matching.
*/
import { configLoader, type ValidatedGitNexusConfig } from './config-loader.ts';
export class IgnoreService {
private static instance: IgnoreService;
private config: ValidatedGitNexusConfig | null = null;
private patternsSet: Set<string> = new Set();
private suffixesSet: Set<string> = new Set();
private extensionsSet: Set<string> = new Set();
private customRegexes: RegExp[] = [];
private constructor() {}
public static getInstance(): IgnoreService {
if (!IgnoreService.instance) {
IgnoreService.instance = new IgnoreService();
}
return IgnoreService.instance;
}
/**
* Initialize the ignore service with configuration
*/
public async initialize(): Promise<void> {
this.config = await configLoader.loadConfig();
this.updatePatterns();
}
/**
* Update internal pattern sets from configuration
*/
private updatePatterns(): void {
if (!this.config) return;
const { ignore } = this.config;
// Convert patterns to lowercase Set for O(1) lookup
this.patternsSet = new Set(ignore.patterns.map(p => p.toLowerCase()));
this.suffixesSet = new Set(ignore.suffixes.map(s => s.toLowerCase()));
this.extensionsSet = new Set(ignore.fileExtensions.map(e => e.toLowerCase()));
// Compile custom regex patterns
this.customRegexes = ignore.customPatterns
.map(pattern => {
try {
return new RegExp(pattern, 'i');
} catch (error) {
console.warn(`Invalid regex pattern: ${pattern}`, error);
return null;
}
})
.filter((regex): regex is RegExp => regex !== null);
console.log(`🔧 IgnoreService initialized with ${this.patternsSet.size} patterns, ${this.suffixesSet.size} suffixes, ${this.extensionsSet.size} extensions`);
}
/**
* Check if a file path should be ignored
*
* Uses directory-component matching like the Python example:
* - Splits path into components
* - Checks each component against ignore patterns
* - Prevents false positives (e.g., "my_node_modules_notes.txt")
*/
public shouldIgnorePath(filePath: string): boolean {
if (!this.config?.ignore.enabled) {
return false;
}
// Normalize path separators and remove leading/trailing slashes
const normalizedPath = filePath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
// Split path into components for directory-part matching
const pathComponents = normalizedPath.split('/').filter(Boolean);
// Check each component against ignore patterns (case-insensitive)
for (const component of pathComponents) {
const lowerComponent = component.toLowerCase();
if (this.patternsSet.has(lowerComponent)) {
return true;
}
}
// Check file suffixes
const lowerPath = normalizedPath.toLowerCase();
for (const suffix of this.suffixesSet) {
if (lowerPath.endsWith(suffix)) {
return true;
}
}
// Check file extensions
const fileName = pathComponents[pathComponents.length - 1] || '';
const lowerFileName = fileName.toLowerCase();
for (const ext of this.extensionsSet) {
if (lowerFileName.endsWith(ext)) {
return true;
}
}
// Check custom regex patterns
for (const regex of this.customRegexes) {
if (regex.test(normalizedPath)) {
return true;
}
}
return false;
}
/**
* Filter an array of paths, removing ignored ones
*
* Optimized for batch processing with early returns
*/
public filterPaths(paths: string[]): string[] {
if (!this.config?.ignore.enabled) {
return paths;
}
const startTime = performance.now();
const filtered = paths.filter(path => !this.shouldIgnorePath(path));
const endTime = performance.now();
const filteredCount = paths.length - filtered.length;
if (filteredCount > 0) {
console.log(`🚫 IgnoreService filtered ${filteredCount}/${paths.length} paths in ${(endTime - startTime).toFixed(2)}ms`);
}
return filtered;
}
/**
* Check if a directory should be ignored
*
* Specialized method for directory filtering during traversal
*/
public shouldIgnoreDirectory(dirPath: string): boolean {
if (!this.config?.ignore.enabled) {
return false;
}
// For directories, we only check the directory name itself
const dirName = dirPath.split('/').pop()?.toLowerCase() || '';
return this.patternsSet.has(dirName);
}
/**
* Get current ignore statistics
*/
public getStats(): {
enabled: boolean;
patterns: number;
suffixes: number;
extensions: number;
customPatterns: number;
} {
return {
enabled: this.config?.ignore.enabled ?? false,
patterns: this.patternsSet.size,
suffixes: this.suffixesSet.size,
extensions: this.extensionsSet.size,
customPatterns: this.customRegexes.length
};
}
/**
* Update ignore patterns at runtime
*/
public updateIgnorePatterns(updates: Partial<ValidatedGitNexusConfig['ignore']>): void {
if (!this.config) return;
// Update configuration
this.config.ignore = { ...this.config.ignore, ...updates };
// Rebuild pattern sets
this.updatePatterns();
console.log('🔄 IgnoreService patterns updated');
}
/**
* Add custom patterns at runtime
*/
public addCustomPatterns(patterns: string[], type: 'patterns' | 'suffixes' | 'extensions' | 'regex' = 'patterns'): void {
if (!this.config) return;
switch (type) {
case 'patterns':
this.config.ignore.patterns.push(...patterns);
break;
case 'suffixes':
this.config.ignore.suffixes.push(...patterns);
break;
case 'extensions':
this.config.ignore.fileExtensions.push(...patterns);
break;
case 'regex':
this.config.ignore.customPatterns.push(...patterns);
break;
}
this.updatePatterns();
console.log(`➕ Added ${patterns.length} ${type} to IgnoreService`);
}
/**
* Test a path against ignore patterns (for debugging)
*/
public testPath(filePath: string): {
ignored: boolean;
reason?: string;
matchedPattern?: string;
} {
if (!this.config?.ignore.enabled) {
return { ignored: false };
}
const normalizedPath = filePath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
const pathComponents = normalizedPath.split('/').filter(Boolean);
// Check directory components
for (const component of pathComponents) {
const lowerComponent = component.toLowerCase();
if (this.patternsSet.has(lowerComponent)) {
return {
ignored: true,
reason: 'directory pattern',
matchedPattern: component
};
}
}
// Check suffixes
const lowerPath = normalizedPath.toLowerCase();
for (const suffix of this.suffixesSet) {
if (lowerPath.endsWith(suffix)) {
return {
ignored: true,
reason: 'file suffix',
matchedPattern: suffix
};
}
}
// Check extensions
const fileName = pathComponents[pathComponents.length - 1] || '';
const lowerFileName = fileName.toLowerCase();
for (const ext of this.extensionsSet) {
if (lowerFileName.endsWith(ext)) {
return {
ignored: true,
reason: 'file extension',
matchedPattern: ext
};
}
}
// Check regex patterns
for (let i = 0; i < this.customRegexes.length; i++) {
const regex = this.customRegexes[i];
if (regex.test(normalizedPath)) {
return {
ignored: true,
reason: 'custom regex',
matchedPattern: this.config.ignore.customPatterns[i]
};
}
}
return { ignored: false };
}
}
// Export singleton instance
export const ignoreService = IgnoreService.getInstance();
src/config/language-config.ts
+2 -36
View File
@@ -161,42 +161,8 @@ const TYPESCRIPT_BUILTINS = new Set([
'InstanceType', 'ThisParameterType', 'OmitThisParameter', 'ThisType'
]);
// Centralized ignore patterns
export const IGNORE_PATTERNS = new Set([
// Version Control
'.git', '.svn', '.hg',
// Package Managers & Dependencies
'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps',
// Python Virtual Environments & Cache
'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__',
'.pytest_cache', '.mypy_cache', '.tox',
// Build & Distribution Directories
'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build',
// Static Assets and Public Directories
'public', 'assets', 'static',
// IDE & Editor Directories
'.vs', '.vscode', '.idea', '.eclipse', '.settings',
// Temporary & Log Directories
'tmp', '.tmp', 'temp', 'logs', 'log',
// Coverage & Testing
'coverage', '.coverage', 'htmlcov', '.nyc_output',
// OS & System
'.DS_Store', 'Thumbs.db',
// Documentation Build Output
'_site', '.docusaurus',
// Cache Directories
'.cache', '.parcel-cache', '.next', '.nuxt'
]);
// Note: Ignore patterns have been moved to the centralized IgnoreService
// See src/config/ignore-service.ts and gitnexus.config.ts
// Language configurations
export const LANGUAGE_CONFIGS: Record<string, LanguageConfig> = {
src/core/ingestion/parallel-parsing-processor.ts
+3 -48
View File
@@ -6,7 +6,7 @@ import {
DuplicateDetector,
pathUtils
} from '../../lib/shared-utils.js';
import { IGNORE_PATTERNS } from '../../config/language-config.js';
import { ignoreService } from '../../config/ignore-service.js';
import { WebWorkerPool, WebWorkerPoolUtils } from '../../lib/web-worker-pool.js';
import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js';
import { generateDeterministicId } from '../../lib/utils.ts';
@@ -478,54 +478,9 @@ export interface ParallelParsingResult {
filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext)));
}
// Apply ignore patterns (be more selective to avoid over-filtering) - EXACT MATCH TO SINGLE-THREADED
// Apply centralized ignore patterns
const beforeIgnoreFilter = filtered.length;
filtered = filtered.filter(path => {
// More precise ignore pattern matching
for (const pattern of IGNORE_PATTERNS) {
if (typeof pattern === 'string') {
// Only ignore if the pattern is a complete directory component
if (path.includes(`/${pattern}/`) ||
path.startsWith(`${pattern}/`) ||
path.endsWith(`/${pattern}`) ||
path === pattern) {
return false;
}
}
}
// Additional filtering for files that shouldn't be in KG
const fileName = path.split('/').pop()?.toLowerCase() || '';
// Skip documentation and readme files
if (fileName.includes('readme') ||
fileName.includes('license') ||
fileName.includes('changelog') ||
fileName.includes('contributing') ||
fileName.includes('authors') ||
fileName.includes('maintainers')) {
return false;
}
// Skip git and version control files
if (fileName.startsWith('.git') ||
fileName.includes('.gitignore') ||
fileName.includes('.gitattributes')) {
return false;
}
// Skip common non-source files
if (fileName.includes('dockerfile') ||
fileName.includes('docker-compose') ||
fileName.endsWith('.md') ||
fileName.endsWith('.txt') ||
fileName.endsWith('.log') ||
fileName.endsWith('.lock')) {
return false;
}
return true;
});
filtered = ignoreService.filterPaths(filtered);
// Apply content filter (only exclude truly empty files)
const beforeContentFilter = filtered.length;
src/core/ingestion/parsing-processor.ts
+3 -48
View File
@@ -7,7 +7,7 @@ import {
DuplicateDetector,
BatchProcessor
} from '../../lib/shared-utils.js';
import { IGNORE_PATTERNS } from '../../config/language-config.js';
import { ignoreService } from '../../config/ignore-service.js';
import Parser from 'web-tree-sitter';
import { TYPESCRIPT_QUERIES, JAVASCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries';
import { initTreeSitter, loadTypeScriptParser, loadPythonParser, loadJavaScriptParser } from '../tree-sitter/parser-loader.js';
@@ -148,54 +148,9 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext)));
}
// Apply ignore patterns (be more selective to avoid over-filtering)
// Apply centralized ignore patterns
const beforeIgnoreFilter = filtered.length;
filtered = filtered.filter(path => {
// More precise ignore pattern matching
for (const pattern of IGNORE_PATTERNS) {
if (typeof pattern === 'string') {
// Only ignore if the pattern is a complete directory component
if (path.includes(`/${pattern}/`) ||
path.startsWith(`${pattern}/`) ||
path.endsWith(`/${pattern}`) ||
path === pattern) {
return false;
}
}
}
// Additional filtering for files that shouldn't be in KG
const fileName = path.split('/').pop()?.toLowerCase() || '';
// Skip documentation and readme files
if (fileName.includes('readme') ||
fileName.includes('license') ||
fileName.includes('changelog') ||
fileName.includes('contributing') ||
fileName.includes('authors') ||
fileName.includes('maintainers')) {
return false;
}
// Skip git and version control files
if (fileName.startsWith('.git') ||
fileName.includes('.gitignore') ||
fileName.includes('.gitattributes')) {
return false;
}
// Skip common non-source files
if (fileName.includes('dockerfile') ||
fileName.includes('docker-compose') ||
fileName.endsWith('.md') ||
fileName.endsWith('.txt') ||
fileName.endsWith('.log') ||
fileName.endsWith('.lock')) {
return false;
}
return true;
});
filtered = ignoreService.filterPaths(filtered);
// Apply content filter (only exclude truly empty files)
const beforeContentFilter = filtered.length;
src/core/ingestion/structure-processor.ts
+5 -102
View File
@@ -1,6 +1,7 @@
import type { KnowledgeGraph } from '../graph/graph.ts';
import type { GraphNode, GraphRelationship } from '../graph/types.ts';
import { generateDeterministicId } from '../../lib/utils.ts';
import { ignoreService } from '../../config/ignore-service.js';
export interface StructureInput {
projectRoot: string;
@@ -16,42 +17,6 @@ export class StructureProcessor {
relationshipsProcessed: 0
};
// Import ignore patterns from ParsingProcessor
private static readonly IGNORE_PATTERNS = new Set([
// Version Control
'.git', '.svn', '.hg',
// Package Managers & Dependencies
'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps',
// Python Virtual Environments & Cache
'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__',
'.pytest_cache', '.mypy_cache', '.tox',
// Build & Distribution Directories
'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build',
// Static Assets and Public Directories
'public', 'assets', 'static',
// IDE & Editor Directories
'.vs', '.vscode', '.idea', '.eclipse', '.settings',
// Temporary & Log Directories
'tmp', '.tmp', 'temp', 'logs', 'log',
// Coverage & Testing
'coverage', '.coverage', 'htmlcov', '.nyc_output',
// OS & System
'.DS_Store', 'Thumbs.db',
// Documentation Build Output
'_site', '.docusaurus',
// Cache Directories
'.cache', '.parcel-cache', '.next', '.nuxt'
]);
/**
* Process complete repository structure directly from discovered paths
@@ -328,80 +293,18 @@ export class StructureProcessor {
/**
* Check if a directory should be hidden from the KG visualization
* This matches the ignore patterns used in ParsingProcessor
* Uses centralized ignore service
*/
private shouldHideDirectory(dirPath: string): boolean {
const pathSegments = dirPath.split('/');
// Check if any segment of the path matches an ignore pattern
const hasIgnoredSegment = pathSegments.some(segment =>
StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase())
);
if (hasIgnoredSegment) {
return true;
}
// Additional pattern matching
const lowerPath = dirPath.toLowerCase();
// Hide Python egg-info directories
if (lowerPath.includes('.egg-info')) {
return true;
}
// Hide site-packages directories
if (lowerPath.includes('site-packages')) {
return true;
}
// Hide most hidden directories (except important ones like .github)
for (const segment of pathSegments) {
if (segment.startsWith('.') && segment !== '.github') {
return true;
}
}
return false;
return ignoreService.shouldIgnoreDirectory(dirPath);
}
/**
* Check if a file should be hidden from the KG visualization
* This matches the ignore patterns used in ParsingProcessor
* Uses centralized ignore service
*/
private shouldHideFile(filePath: string): boolean {
const pathSegments = filePath.split('/');
// Check if any segment of the path matches an ignore pattern
const hasIgnoredSegment = pathSegments.some(segment =>
StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase())
);
if (hasIgnoredSegment) {
return true;
}
// Additional pattern matching
const lowerPath = filePath.toLowerCase();
// Hide Python egg-info directories
if (lowerPath.includes('.egg-info')) {
return true;
}
// Hide site-packages directories
if (lowerPath.includes('site-packages')) {
return true;
}
// Hide most hidden directories (except important ones like .github)
for (const segment of pathSegments) {
if (segment.startsWith('.') && segment !== '.github') {
return true;
}
}
return false;
return ignoreService.shouldIgnorePath(filePath);
}
/**
src/lib/preload.ts
+6
View File
@@ -10,5 +10,11 @@ import '../core/ingestion/parsing-processor';
import '../core/ingestion/call-processor';
import '../core/ingestion/structure-processor';
import '../core/tree-sitter/parser-loader';
import { ignoreService } from '../config/ignore-service';
// Initialize ignore service
ignoreService.initialize().catch(error => {
console.warn('Failed to initialize ignore service:', error);
});
console.log('Dependencies preloaded to prevent runtime optimization');
src/services/github.ts
+4 -189
View File
@@ -1,4 +1,5 @@
import axios, { type AxiosInstance, type AxiosResponse } from 'axios';
import { ignoreService } from '../config/ignore-service.js';
interface GitHubFile {
name: string;
@@ -253,202 +254,16 @@ export class GitHubService {
private shouldSkipDirectory(path: string): boolean {
if (!path) return true; // Skip if path is undefined/null
const skipDirs = [
// Git version control
'.git',
// JavaScript dependencies (common in full-stack projects)
'node_modules',
// Python bytecode cache
'__pycache__',
// Python virtual environments
'venv',
'env',
'.venv',
'envs',
'virtualenv',
// Build, distribution, and temporary directories
'build',
'dist',
'logs',
'tmp',
'.tmp',
// Static assets and public files
'public',
'assets',
'static',
// Additional common directories to skip
'coverage',
'.coverage',
'htmlcov',
'vendor',
'deps',
'_build',
'.gradle',
'bin',
'obj',
'.vs',
'.vscode',
'.idea',
'temp'
];
// Check each directory component in the path
const pathParts = path.split('/');
for (const part of pathParts) {
const dirName = part.toLowerCase();
// Check for exact matches
if (skipDirs.includes(dirName) || dirName.startsWith('.')) {
return true;
}
// Check for .egg-info directories
if (dirName.endsWith('.egg-info')) {
return true;
}
}
// Check for virtual environment patterns anywhere in the path
const fullPathLower = path.toLowerCase();
const venvPatterns = [
'/.venv/',
'/venv/',
'/env/',
'/.env/',
'/envs/',
'/virtualenv/',
'/site-packages/',
'/lib/python',
'/lib64/python',
'/scripts/',
'/bin/python'
];
if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) {
return true;
}
return false;
return ignoreService.shouldIgnoreDirectory(path);
}
private shouldIncludeFile(path: string): boolean {
if (!path) return false; // Skip if path is undefined/null
const fileName = path.split('/').pop() || '';
// Skip hidden files except specific config files
if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) {
return false;
}
// Skip Python-specific file patterns
const skipPatterns = [
// Python compiled bytecode
/\.pyc$/,
/\.pyo$/,
// Python extension modules (binary)
/\.pyd$/,
/\.so$/,
// Python packages
/\.egg$/,
/\.whl$/,
// Lock files
/\.lock$/,
/poetry\.lock$/,
/Pipfile\.lock$/,
// Editor swap files
/\..*\.swp$/,
/\..*\.swo$/,
// OS metadata files
/^Thumbs\.db$/,
/^\.DS_Store$/,
// General binary and archive files
/\.zip$/,
/\.tar$/,
/\.rar$/,
/\.7z$/,
/\.gz$/,
// Media files
/\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i,
/\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i,
/\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i,
// Document files
/\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i,
// Other binary files
/\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i,
// Font files
/\.(woff|woff2|ttf|eot|otf)$/i,
// Minified files and source maps
/\.min\.(js|css)$/,
/\.map$/,
// Log and temporary files
/\.log$/,
/\.tmp$/,
/\.cache$/,
/\.pid$/,
/\.seed$/
];
if (skipPatterns.some(pattern => pattern.test(fileName))) {
return false;
}
// Include common source and important config files
const extension = '.' + (fileName.split('.').pop() || '').toLowerCase();
const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']);
if (includeSourceExts.has(extension)) {
return true;
}
const importantConfigFiles = new Set([
// Python
'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml',
'__init__.py',
// JS/TS ecosystem
'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
'tsconfig.json', 'tsconfig.base.json',
'vite.config.ts', 'vite.config.js',
'.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json',
// Docs/licenses
'readme.md', 'license', 'changelog.md', 'manifest.in'
]);
if (importantConfigFiles.has(fileName.toLowerCase())) {
return true;
}
return false;
return !ignoreService.shouldIgnorePath(path);
}
private shouldSkipFileForContent(path: string): boolean {
const pathLower = path.toLowerCase();
// Skip .git files and directories (these can be massive and not useful for code analysis)
if (pathLower.includes('/.git/') || pathLower.startsWith('.git/')) {
return true;
}
// Skip binary files that shouldn't have content
const binaryExtensions = [
// Images
'.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp',
// Fonts
'.woff', '.woff2', '.ttf', '.eot', '.otf',
// Documents
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
// Archives
'.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma',
// Executables & Libraries
'.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm',
// Package formats
'.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa',
// Media
'.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v',
'.aac', '.ogg', '.flac', '.m4a', '.wma'
];
return binaryExtensions.some(ext => pathLower.endsWith(ext));
return ignoreService.shouldIgnorePath(path);
}
public async getAllPathsRecursively(owner: string, repo: string, path: string = ''): Promise<string[]> {
src/services/zip.ts
+5 -199
View File
@@ -1,4 +1,5 @@
import JSZip from 'jszip';
import { ignoreService } from '../config/ignore-service.js';
interface JSZipObjectWithData extends JSZip.JSZipObject {
_data?: {
@@ -378,216 +379,21 @@ export class ZipService {
}
private shouldSkipPath(filePath: string): boolean {
// Skip directories that shouldn't be processed
if (this.shouldSkipDirectory(filePath)) {
return true;
}
// Skip files that shouldn't be processed
if (!this.shouldIncludeFile(filePath)) {
return true;
}
return false;
return ignoreService.shouldIgnorePath(filePath);
}
private shouldSkipDirectory(path: string): boolean {
if (!path) return true; // Skip if path is undefined/null
const skipDirs = [
// Git version control
'.git',
// JavaScript dependencies (common in full-stack projects)
'node_modules',
// Python bytecode cache
'__pycache__',
// Python virtual environments
'venv',
'env',
'.venv',
'envs',
'virtualenv',
// Build, distribution, and temporary directories
'build',
'dist',
'logs',
'tmp',
'.tmp',
// Static assets and public files
'public',
'assets',
'static',
// Additional common directories to skip
'coverage',
'.coverage',
'htmlcov',
'vendor',
'deps',
'_build',
'.gradle',
'bin',
'obj',
'.vs',
'.vscode',
'.idea',
'temp'
];
// Check each directory component in the path
const pathParts = path.split('/');
for (const part of pathParts) {
const dirName = part.toLowerCase();
// Check for exact matches
if (skipDirs.includes(dirName) || dirName.startsWith('.')) {
return true;
}
// Check for .egg-info directories
if (dirName.endsWith('.egg-info')) {
return true;
}
}
// Check for virtual environment patterns anywhere in the path
const fullPathLower = path.toLowerCase();
const venvPatterns = [
'/.venv/',
'/venv/',
'/env/',
'/.env/',
'/envs/',
'/virtualenv/',
'/site-packages/',
'/lib/python',
'/lib64/python',
'/scripts/',
'/bin/python'
];
if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) {
return true;
}
return false;
return ignoreService.shouldIgnoreDirectory(path);
}
private shouldIncludeFile(path: string): boolean {
if (!path) return false; // Skip if path is undefined/null
const fileName = path.split('/').pop() || '';
// Skip hidden files except specific config files
if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) {
return false;
}
// Skip common compiled/binary patterns
const skipPatterns = [
// Python compiled bytecode
/\.pyc$/,
/\.pyo$/,
// Python extension modules (binary)
/\.pyd$/,
/\.so$/,
// Python packages
/\.egg$/,
/\.whl$/,
// Lock files
/\.lock$/,
/poetry\.lock$/,
/Pipfile\.lock$/,
// Editor swap files
/\..*\.swp$/,
/\..*\.swo$/,
// OS metadata files
/^Thumbs\.db$/,
/^\.DS_Store$/,
// General binary and archive files
/\.zip$/,
/\.tar$/,
/\.rar$/,
/\.7z$/,
/\.gz$/,
// Media files
/\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i,
/\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i,
/\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i,
// Document files
/\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i,
// Other binary files
/\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i,
// Font files
/\.(woff|woff2|ttf|eot|otf)$/i,
// Minified files and source maps
/\.min\.(js|css)$/,
/\.map$/,
// Log and temporary files
/\.log$/,
/\.tmp$/,
/\.cache$/,
/\.pid$/,
/\.seed$/
];
if (skipPatterns.some(pattern => pattern.test(fileName))) {
return false;
}
// Include common source and important config files
const extension = '.' + (fileName.split('.').pop() || '').toLowerCase();
const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']);
if (includeSourceExts.has(extension)) {
return true;
}
const importantConfigFiles = new Set([
// Python
'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml',
'__init__.py',
// JS/TS ecosystem
'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml',
'tsconfig.json', 'tsconfig.base.json',
'vite.config.ts', 'vite.config.js',
'.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json',
// Docs/licenses
'readme.md', 'license', 'changelog.md', 'manifest.in'
]);
if (importantConfigFiles.has(fileName.toLowerCase())) {
return true;
}
return false;
return !ignoreService.shouldIgnorePath(path);
}
private shouldSkipFileForContent(path: string): boolean {
const pathLower = path.toLowerCase();
// Skip .git files and directories (these can be massive and not useful for code analysis)
if (pathLower.includes('/.git/') || pathLower.startsWith('.git/')) {
return true;
}
// Skip binary files that shouldn't have content
const binaryExtensions = [
// Images
'.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp',
// Fonts
'.woff', '.woff2', '.ttf', '.eot', '.otf',
// Documents
'.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
// Archives
'.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma',
// Executables & Libraries
'.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm',
// Package formats
'.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa',
// Media
'.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v',
'.aac', '.ogg', '.flac', '.m4a', '.wma'
];
return binaryExtensions.some(ext => pathLower.endsWith(ext));
return ignoreService.shouldIgnorePath(path);
}
public getDefaultTextExtensions(): string[] {
test-ignore-service.js
+1
View File
@@ -0,0 +1 @@