From 459f381b9f21bf31d6f356b8e17c4b44c31ecf6c Mon Sep 17 00:00:00 2001
From: David Zhang <dzz0615@gmail.com>
Date: Sun, 23 Mar 2025 23:06:10 -0700
Subject: [PATCH] added concurrency limit env var

---
 .env.example         |  1 +
 README.md            |  4 ++--
 src/deep-research.ts | 22 +++++++---------------
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/.env.example b/.env.example
index 6a336db..455d415 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,7 @@
 FIRECRAWL_KEY="YOUR_KEY"
 # If you want to use your self-hosted Firecrawl, add the following below:
 # FIRECRAWL_BASE_URL="http://localhost:3002"
+# FIRECRAWL_CONCURRENCY="2"
 
 OPENAI_KEY="YOUR_KEY"
 CONTEXT_SIZE="128000"
diff --git a/README.md b/README.md
index cc0bdbb..92a3e02 100644
--- a/README.md
+++ b/README.md
@@ -152,9 +152,9 @@ The final report will be saved as `report.md` or `answer.md` in your working dir
 
 ### Concurrency
 
-If you have a paid version of Firecrawl or a local version, feel free to increase the `ConcurrencyLimit` in `deep-research.ts` so it runs a lot faster.
+If you have a paid version of Firecrawl or a local version, feel free to increase the `ConcurrencyLimit` by setting the `CONCURRENCY_LIMIT` environment variable so it runs faster.
 
-If you have a free version, you may sometimes run into rate limit errors, you can reduce the limit (but it will run a lot slower).
+If you have a free version, you may sometimes run into rate limit errors, you can reduce the limit to 1 (but it will run a lot slower).
 
 ### DeepSeek R1
 
diff --git a/src/deep-research.ts b/src/deep-research.ts
index 924bf96..7d074a1 100644
--- a/src/deep-research.ts
+++ b/src/deep-research.ts
@@ -27,7 +27,7 @@ type ResearchResult = {
 };
 
 // increase this if you have higher API rate limits
-const ConcurrencyLimit = 2;
+const ConcurrencyLimit = Number(process.env.FIRECRAWL_CONCURRENCY) || 2;
 
 // Initialize Firecrawl with optional API key and optional base url
 
@@ -89,8 +89,8 @@ async function processSerpResult({
   numLearnings?: number;
   numFollowUpQuestions?: number;
 }) {
-  const contents = compact(result.data.map(item => item.markdown)).map(
-    content => trimPrompt(content, 25_000),
+  const contents = compact(result.data.map(item => item.markdown)).map(content =>
+    trimPrompt(content, 25_000),
   );
   log(`Ran ${query}, found ${contents.length} contents`);
 
@@ -104,9 +104,7 @@ async function processSerpResult({
         .join('\n')}</contents>`,
     ),
     schema: z.object({
-      learnings: z
-        .array(z.string())
-        .describe(`List of learnings, max of ${numLearnings}`),
+      learnings: z.array(z.string()).describe(`List of learnings, max of ${numLearnings}`),
       followUpQuestions: z
         .array(z.string())
         .describe(
@@ -139,9 +137,7 @@ export async function writeFinalReport({
       `Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as as detailed as possible, aim for 3 or more pages, include ALL the learnings from research:\n\n<prompt>${prompt}</prompt>\n\nHere are all the learnings from previous research:\n\n<learnings>\n${learningsString}\n</learnings>`,
     ),
     schema: z.object({
-      reportMarkdown: z
-        .string()
-        .describe('Final report on the topic in Markdown'),
+      reportMarkdown: z.string().describe('Final report on the topic in Markdown'),
     }),
   });
 
@@ -170,9 +166,7 @@ export async function writeFinalAnswer({
     schema: z.object({
       exactAnswer: z
         .string()
-        .describe(
-          'The final answer, make it short and concise, just the answer, no other text',
-        ),
+        .describe('The final answer, make it short and concise, just the answer, no other text'),
     }),
   });
 
@@ -245,9 +239,7 @@ export async function deepResearch({
           const allUrls = [...visitedUrls, ...newUrls];
 
           if (newDepth > 0) {
-            log(
-              `Researching deeper, breadth: ${newBreadth}, depth: ${newDepth}`,
-            );
+            log(`Researching deeper, breadth: ${newBreadth}, depth: ${newDepth}`);
 
             reportProgress({
               currentDepth: newDepth,