Merge pull request #1 from alexboone29/main

feat: Added Support for Self-Hosted Firecrawl

README.md

@@ -90,6 +90,9 @@ npm install
 
 ```bash
 FIRECRAWL_KEY="your_firecrawl_key"
+# If you want to use your self-hosted Firecrawl, add the following below:
+# FIRECRAWL_BASE_URL="http://localhost:3002"
+
 OPENAI_KEY="your_openai_key"
 ```
 

src/deep-research.ts

@@ -15,10 +15,29 @@ type ResearchResult = {
 // increase this if you have higher API rate limits
 const ConcurrencyLimit = 2;
 
+// Initialize Firecrawl with optional API key and optional base url
+
 const firecrawl = new FirecrawlApp({
-  apiKey: process.env.FIRECRAWL_KEY!,
+  apiKey: process.env.FIRECRAWL_KEY ?? "",
+  apiUrl: process.env.FIRECRAWL_BASE_URL 
 });
 
+// Default options for API consistency
+const defaultOptions = {
+  search: {
+    timeout: 15000,
+    scrapeOptions: { formats: ['markdown'] }
+  },
+  extract: {
+    timeout: 60000,
+    waitForResults: true
+  },
+  scrape: {
+    timeout: 30000,
+    format: 'markdown'
+  }
+};
+
 // take in user query, return a list of SERP queries
 async function generateSerpQueries({
   query,
@@ -156,6 +175,7 @@ export async function deepResearch({
       limit(async () => {
         try {
           const result = await firecrawl.search(serpQuery.query, {
+            ...defaultOptions.search,
             timeout: 15000,
             scrapeOptions: { formats: ['markdown'] },
           });