diff --git a/.github/ISSUE_TEMPLATE/bug_template.yml b/.github/ISSUE_TEMPLATE/bug_template.yml
index 6089f1c2ff..b8f25e47db 100644
--- a/.github/ISSUE_TEMPLATE/bug_template.yml
+++ b/.github/ISSUE_TEMPLATE/bug_template.yml
@@ -66,4 +66,4 @@ body:
     id: additional-context
     attributes:
       label: Logs, Errors, Screenshots, and Additional Context
-      description: Please add any additional context about the problem here.
+      description: If you set DEBUG = 1 in config or env, LLM logs will be stored in the `logs/llm` folder. Please add any additional context about the problem here.
diff --git a/.gitignore b/.gitignore
index e123abaee8..2bce73cacd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -202,3 +202,5 @@ cache
 
 # configuration
 config.toml
+
+test_results*
diff --git a/opendevin/core/config.py b/opendevin/core/config.py
index 32c7b7df80..f78735da58 100644
--- a/opendevin/core/config.py
+++ b/opendevin/core/config.py
@@ -33,7 +33,7 @@ DEFAULT_CONFIG: dict = {
     ConfigType.CACHE_DIR: '/tmp/cache',  # '/tmp/cache' is the default cache directory
     ConfigType.LLM_MODEL: 'gpt-3.5-turbo-1106',
     ConfigType.SANDBOX_CONTAINER_IMAGE: DEFAULT_CONTAINER_IMAGE,
-    ConfigType.RUN_AS_DEVIN: 'true',
+    ConfigType.RUN_AS_DEVIN: True,
     ConfigType.LLM_EMBEDDING_MODEL: 'local',
     ConfigType.LLM_EMBEDDING_BASE_URL: None,
     ConfigType.LLM_EMBEDDING_DEPLOYMENT_NAME: None,
@@ -56,13 +56,14 @@ DEFAULT_CONFIG: dict = {
     ConfigType.AGENT: 'CodeActAgent',
     ConfigType.E2B_API_KEY: '',
     ConfigType.SANDBOX_TYPE: 'ssh',  # Can be 'ssh', 'exec', or 'e2b'
-    ConfigType.USE_HOST_NETWORK: 'false',
+    ConfigType.USE_HOST_NETWORK: False,
     ConfigType.SSH_HOSTNAME: 'localhost',
-    ConfigType.DISABLE_COLOR: 'false',
+    ConfigType.DISABLE_COLOR: False,
     ConfigType.SANDBOX_USER_ID: os.getuid() if hasattr(os, 'getuid') else None,
     ConfigType.SANDBOX_TIMEOUT: 120,
     ConfigType.GITHUB_TOKEN: None,
     ConfigType.SANDBOX_USER_ID: None,
+    ConfigType.DEBUG: False,
 }
 
 config_str = ''
@@ -71,6 +72,15 @@ if os.path.exists('config.toml'):
         config_str = f.read().decode('utf-8')
 
 
+def str_to_bool(value):
+    if isinstance(value, bool):
+        return value
+    return value.lower() in [
+        'true',
+        '1',
+    ]
+
+
 def int_value(value, default, config_key):
     # FIXME use a library
     try:
@@ -89,6 +99,7 @@ for k, v in config.items():
         config[k] = os.environ[k]
     elif k in tomlConfig:
         config[k] = tomlConfig[k]
+
     if k in [
         ConfigType.LLM_NUM_RETRIES,
         ConfigType.LLM_RETRY_MIN_WAIT,
@@ -96,6 +107,14 @@ for k, v in config.items():
     ]:
         config[k] = int_value(config[k], v, config_key=k)
 
+    if k in [
+        ConfigType.RUN_AS_DEVIN,
+        ConfigType.USE_HOST_NETWORK,
+        ConfigType.AGENT_MEMORY_ENABLED,
+        ConfigType.DISABLE_COLOR,
+        ConfigType.DEBUG,
+    ]:
+        config[k] = str_to_bool(config[k])
 # In local there is no sandbox, the workspace will have the same pwd as the host
 if config[ConfigType.SANDBOX_TYPE] == 'local':
     config[ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX] = config[
@@ -179,7 +198,7 @@ def finalize_config():
     if config.get(ConfigType.LLM_EMBEDDING_BASE_URL) is None:
         config[ConfigType.LLM_EMBEDDING_BASE_URL] = config.get(ConfigType.LLM_BASE_URL)
 
-    USE_HOST_NETWORK = config[ConfigType.USE_HOST_NETWORK].lower() != 'false'
+    USE_HOST_NETWORK = config[ConfigType.USE_HOST_NETWORK]
     if USE_HOST_NETWORK and platform.system() == 'Darwin':
         logger.warning(
             'Please upgrade to Docker Desktop 4.29.0 or later to use host network mode on macOS. '
diff --git a/opendevin/core/logger.py b/opendevin/core/logger.py
index 4329ed781a..ecca191f81 100644
--- a/opendevin/core/logger.py
+++ b/opendevin/core/logger.py
@@ -10,7 +10,7 @@ from termcolor import colored
 from opendevin.core import config
 from opendevin.core.schema.config import ConfigType
 
-DISABLE_COLOR_PRINTING = config.get(ConfigType.DISABLE_COLOR).lower() == 'true'
+DISABLE_COLOR_PRINTING = config.get(ConfigType.DISABLE_COLOR)
 
 ColorType = Literal[
     'red',
@@ -91,7 +91,8 @@ def get_file_handler():
     timestamp = datetime.now().strftime('%Y-%m-%d')
     file_name = f'opendevin_{timestamp}.log'
     file_handler = logging.FileHandler(os.path.join(log_dir, file_name))
-    file_handler.setLevel(logging.DEBUG)
+    if config.get(ConfigType.DEBUG):
+        file_handler.setLevel(logging.DEBUG)
     file_handler.setFormatter(file_formatter)
     return file_handler
 
@@ -196,10 +197,12 @@ def get_llm_response_file_handler():
 
 llm_prompt_logger = logging.getLogger('prompt')
 llm_prompt_logger.propagate = False
-llm_prompt_logger.setLevel(logging.DEBUG)
+if config.get(ConfigType.DEBUG):
+    llm_prompt_logger.setLevel(logging.DEBUG)
 llm_prompt_logger.addHandler(get_llm_prompt_file_handler())
 
 llm_response_logger = logging.getLogger('response')
 llm_response_logger.propagate = False
-llm_response_logger.setLevel(logging.DEBUG)
+if config.get(ConfigType.DEBUG):
+    llm_response_logger.setLevel(logging.DEBUG)
 llm_response_logger.addHandler(get_llm_response_file_handler())
diff --git a/opendevin/core/schema/config.py b/opendevin/core/schema/config.py
index bc290392c3..f409b9940a 100644
--- a/opendevin/core/schema/config.py
+++ b/opendevin/core/schema/config.py
@@ -41,3 +41,4 @@ class ConfigType(str, Enum):
     SSH_HOSTNAME = 'SSH_HOSTNAME'
     DISABLE_COLOR = 'DISABLE_COLOR'
     GITHUB_TOKEN = 'GITHUB_TOKEN'
+    DEBUG = 'DEBUG'
diff --git a/opendevin/runtime/docker/exec_box.py b/opendevin/runtime/docker/exec_box.py
index cf45d4f896..c2676a0503 100644
--- a/opendevin/runtime/docker/exec_box.py
+++ b/opendevin/runtime/docker/exec_box.py
@@ -27,7 +27,7 @@ SANDBOX_WORKSPACE_DIR = config.get(ConfigType.WORKSPACE_MOUNT_PATH_IN_SANDBOX)
 
 # FIXME: On some containers, the devin user doesn't have enough permission, e.g. to install packages
 # How do we make this more flexible?
-RUN_AS_DEVIN = config.get(ConfigType.RUN_AS_DEVIN).lower() != 'false'
+RUN_AS_DEVIN = config.get(ConfigType.RUN_AS_DEVIN)
 USER_ID = 1000
 if SANDBOX_USER_ID := config.get(ConfigType.SANDBOX_USER_ID):
     USER_ID = int(SANDBOX_USER_ID)
diff --git a/opendevin/runtime/docker/ssh_box.py b/opendevin/runtime/docker/ssh_box.py
index 69861b7ef1..0e1ffa006f 100644
--- a/opendevin/runtime/docker/ssh_box.py
+++ b/opendevin/runtime/docker/ssh_box.py
@@ -38,7 +38,7 @@ USE_HOST_NETWORK = config.get(ConfigType.USE_HOST_NETWORK)
 
 # FIXME: On some containers, the devin user doesn't have enough permission, e.g. to install packages
 # How do we make this more flexible?
-RUN_AS_DEVIN = config.get(ConfigType.RUN_AS_DEVIN).lower() != 'false'
+RUN_AS_DEVIN = config.get(ConfigType.RUN_AS_DEVIN)
 USER_ID = 1000
 if SANDBOX_USER_ID := config.get(ConfigType.SANDBOX_USER_ID):
     USER_ID = int(SANDBOX_USER_ID)
diff --git a/opendevin/runtime/plugins/jupyter/execute_server b/opendevin/runtime/plugins/jupyter/execute_server
index 134ea58f3e..5b6f589e0c 100755
--- a/opendevin/runtime/plugins/jupyter/execute_server
+++ b/opendevin/runtime/plugins/jupyter/execute_server
@@ -175,7 +175,7 @@ class JupyterKernel:
                 if parent_msg_id != msg_id:
                     continue
 
-                if os.environ.get('DEBUG', False):
+                if os.environ.get('DEBUG'):
                     logging.info(
                         f"MSG TYPE: {msg_type.upper()} DONE:{execution_done}\nCONTENT: {msg['content']}"
                     )
@@ -221,7 +221,7 @@ class JupyterKernel:
         # Remove ANSI
         ret = strip_ansi(ret)
 
-        if os.environ.get('DEBUG', False):
+        if os.environ.get('DEBUG'):
             logging.info(f'OUTPUT:\n{ret}')
         return ret