From 8f16edac6563f97ed06978c0e7f4d76fcac47f23 Mon Sep 17 00:00:00 2001
From: alexboone <accounts@neutralbase.com>
Date: Mon, 3 Feb 2025 20:05:48 -0800
Subject: [PATCH] added support for local firecrawl endpoint

---
 .gitignore           |  1 +
 README.md            |  3 +++
 src/deep-research.ts | 22 +++++++++++++++++++++-
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 96fab4f..1ec9db4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,4 @@ yarn-error.log*
 # Misc
 .DS_Store
 *.pem
+report.md
diff --git a/README.md b/README.md
index 7e398b7..977594f 100644
--- a/README.md
+++ b/README.md
@@ -90,6 +90,9 @@ npm install
 
 ```bash
 FIRECRAWL_KEY="your_firecrawl_key"
+# If you want to use your localhost firecrawl, add the following below:
+FIRECRAWL_BASE_URL="http://localhost:3002"
+
 OPENAI_KEY="your_openai_key"
 ```
 
diff --git a/src/deep-research.ts b/src/deep-research.ts
index 1f1a591..152bb72 100644
--- a/src/deep-research.ts
+++ b/src/deep-research.ts
@@ -15,10 +15,29 @@ type ResearchResult = {
 // increase this if you have higher API rate limits
 const ConcurrencyLimit = 2;
 
+// Initialize Firecrawl with API key if available, otherwise use local
+const isLocal = !process.env.FIRECRAWL_KEY || process.env.FIRECRAWL_KEY === "your_firecrawl_key";
 const firecrawl = new FirecrawlApp({
-  apiKey: process.env.FIRECRAWL_KEY!,
+  apiKey: isLocal ? 'local-development-token' : process.env.FIRECRAWL_KEY!,
+  apiUrl: isLocal ? (process.env.FIRECRAWL_BASE_URL || 'http://localhost:3002') : undefined
 });
 
+// Default options for API consistency
+const defaultOptions = {
+  search: {
+    timeout: 15000,
+    scrapeOptions: { formats: ['markdown'] }
+  },
+  extract: {
+    timeout: 60000,
+    waitForResults: true
+  },
+  scrape: {
+    timeout: 30000,
+    format: 'markdown'
+  }
+};
+
 // take in user query, return a list of SERP queries
 async function generateSerpQueries({
   query,
@@ -156,6 +175,7 @@ export async function deepResearch({
       limit(async () => {
         try {
           const result = await firecrawl.search(serpQuery.query, {
+            ...defaultOptions.search,
             timeout: 15000,
             scrapeOptions: { formats: ['markdown'] },
           });