Add integration test framework with mock llm (#1301)

* Add integration test framework with mock llm

* Fix MonologueAgent and PlannerAgent tests

* Remove adhoc logging

* Use existing logs

* Fix SWEAgent and PlannerAgent

* Check-in test log files

* conftest: look up under test name folder only

* Add docstring to conftest

* Finish dev doc

* Avoid non-determinism

* Remove dependency on llm embedding model

* Init embedding model only for MonologueAgent

* Add adhoc fix for sandbox discrepancy

* Test ssh and exec sandboxes

* CI: fix missing sandbox type

* conftest: Remove hack

* Reword comment for TODO

.github/workflows/run-integration-tests.yml

@@ -0,0 +1,70 @@
+name: Run Integration Tests
+
+on: [push, pull_request]
+
+jobs:
+  on-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: SWEAgent-py311-ssh
+            python-version: "3.11"
+            agent: "SWEAgent"
+            embedding-model: "none"
+            sandbox: "ssh"
+          - name: PlannerAgent-py311-ssh
+            python-version: "3.11"
+            agent: "PlannerAgent"
+            embedding-model: "none"
+            sandbox: "ssh"
+          - name: MonologueAgent-py311-ssh
+            python-version: "3.11"
+            agent: "MonologueAgent"
+            embedding-model: "local"
+            sandbox: "ssh"
+          - name: CodeActAgent-py311-ssh
+            python-version: "3.11"
+            agent: "CodeActAgent"
+            embedding-model: "none"
+            sandbox: "ssh"
+          - name: SWEAgent-py311-exec
+            python-version: "3.11"
+            agent: "SWEAgent"
+            embedding-model: "none"
+            sandbox: "exec"
+          - name: PlannerAgent-py311-exec
+            python-version: "3.11"
+            agent: "PlannerAgent"
+            embedding-model: "none"
+            sandbox: "exec"
+          - name: MonologueAgent-py311-exec
+            python-version: "3.11"
+            agent: "MonologueAgent"
+            embedding-model: "local"
+            sandbox: "exec"
+          - name: CodeActAgent-py311-exec
+            python-version: "3.11"
+            agent: "CodeActAgent"
+            embedding-model: "none"
+            sandbox: "exec"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python3 -
+      - name: Build Environment
+        run: make build
+      - name: Run Integration Tests
+        env:
+          SANDBOX_TYPE: ${{ matrix.sandbox }}
+          AGENT: ${{ matrix.agent }}
+          MAX_ITERATIONS: 10
+          LLM_EMBEDDING_MODEL: ${{ matrix.embedding-model }}
+        run: |
+          rm -rf workspace
+          mkdir workspace
+          WORKSPACE_BASE="$GITHUB_WORKSPACE/workspace" WORKSPACE_MOUNT_PATH="$GITHUB_WORKSPACE/workspace" poetry run pytest -s ./tests/integration

.github/workflows/run-tests.yml

@@ -1,20 +0,0 @@
-name: Run Tests
-
-on: [push]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.11'
-      - name: Set up environment
-        run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-          poetry install --without evaluation
-      - name: Run tests
-        run: |
-          poetry run pytest ./tests

.github/workflows/run-unit-tests.yml

@@ -1,4 +1,4 @@
-name: Build & Run Tests
+name: Run Unit Tests
 
 on: [push, pull_request]
 
@@ -26,7 +26,7 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Tests
-        run: poetry run pytest ./tests
+        run: poetry run pytest ./tests/unit
   on-linux:
     runs-on: ubuntu-latest
     strategy:
@@ -44,4 +44,4 @@ jobs:
       - name: Build Environment
         run: make build
       - name: Run Tests
-        run: poetry run pytest ./tests
+        run: poetry run pytest ./tests/unit