diff --git a/.github/workflows/dummy-agent-test.yml b/.github/workflows/dummy-agent-test.yml
index 392876db22..9ac8c590d2 100644
--- a/.github/workflows/dummy-agent-test.yml
+++ b/.github/workflows/dummy-agent-test.yml
@@ -1,4 +1,4 @@
-name: Run e2e test with dummy agent
+name: Run E2E test with dummy agent
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -30,4 +30,13 @@ jobs:
           wget https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json -P /tmp/llama_index/models--BAAI--bge-small-en-v1.5/snapshots/5c38ec7c405ec4b44b94cc5a9bb96e735b38267a/1_Pooling/
       - name: Run tests
         run: |
+          set -e
           poetry run python opendevin/core/main.py -t "do a flip" -m ollama/not-a-model -d ./workspace/ -c DummyAgent
+      - name: Check exit code
+        run: |
+          if [ $? -ne 0 ]; then
+            echo "Test failed"
+            exit 1
+          else
+            echo "Test passed"
+          fi