diff --git a/docs/modules/usage/llms/llms.md b/docs/modules/usage/llms/llms.md index 08b1b1fff6..c7e23059dc 100644 --- a/docs/modules/usage/llms/llms.md +++ b/docs/modules/usage/llms/llms.md @@ -15,13 +15,14 @@ OpenDevin will issue many prompts to the LLM you configure. Most of these LLMs c The `LLM_MODEL` environment variable controls which model is used in programmatic interactions. But when using the OpenDevin UI, you'll need to choose your model in the settings window. -The following environment variables might be necessary for some LLMs: +The following environment variables might be necessary for some LLMs/providers: - `LLM_API_KEY` - `LLM_BASE_URL` - `LLM_EMBEDDING_MODEL` - `LLM_EMBEDDING_DEPLOYMENT_NAME` - `LLM_API_VERSION` +- `LLM_DROP_PARAMS` We have a few guides for running OpenDevin with specific model providers: diff --git a/opendevin/core/config.py b/opendevin/core/config.py index fbe4d8191f..1360d99884 100644 --- a/opendevin/core/config.py +++ b/opendevin/core/config.py @@ -49,6 +49,7 @@ class LLMConfig: input_cost_per_token: The cost per input token. This will available in logs for the user to check. output_cost_per_token: The cost per output token. This will available in logs for the user to check. ollama_base_url: The base URL for the OLLAMA API. + drop_params: Drop any unmapped (unsupported) params without causing an exception. """ model: str = 'gpt-4o' @@ -75,6 +76,7 @@ class LLMConfig: input_cost_per_token: float | None = None output_cost_per_token: float | None = None ollama_base_url: str | None = None + drop_params: bool | None = None def defaults_to_dict(self) -> dict: """Serialize fields to a dict for the frontend, including type hints, defaults, and whether it's optional.""" diff --git a/opendevin/core/schema/config.py b/opendevin/core/schema/config.py index 4b8c05c4ac..b10ebe7ad0 100644 --- a/opendevin/core/schema/config.py +++ b/opendevin/core/schema/config.py @@ -4,6 +4,7 @@ from enum import Enum class ConfigType(str, Enum): # For frontend LLM_CUSTOM_LLM_PROVIDER = 'LLM_CUSTOM_LLM_PROVIDER' + LLM_DROP_PARAMS = 'LLM_DROP_PARAMS' LLM_MAX_INPUT_TOKENS = 'LLM_MAX_INPUT_TOKENS' LLM_MAX_OUTPUT_TOKENS = 'LLM_MAX_OUTPUT_TOKENS' LLM_TOP_P = 'LLM_TOP_P' diff --git a/opendevin/llm/llm.py b/opendevin/llm/llm.py index aff76421f3..5d4e8b55ab 100644 --- a/opendevin/llm/llm.py +++ b/opendevin/llm/llm.py @@ -52,7 +52,6 @@ class LLM: Args: config: The LLM configuration """ - self.config = copy.deepcopy(config) self.metrics = metrics if metrics is not None else Metrics() self.cost_metric_supported = True @@ -60,10 +59,12 @@ class LLM: # litellm actually uses base Exception here for unknown model self.model_info = None try: - if not config.model.startswith('openrouter'): - self.model_info = litellm.get_model_info(config.model.split(':')[0]) + if self.config.model.startswith('openrouter'): + self.model_info = litellm.get_model_info(self.config.model) else: - self.model_info = litellm.get_model_info(config.model) + self.model_info = litellm.get_model_info( + self.config.model.split(':')[0] + ) # noinspection PyBroadException except Exception: logger.warning(f'Could not get model info for {config.model}') @@ -91,6 +92,9 @@ class LLM: # Max output tokens for gpt3.5, so this is a safe fallback for any potentially viable model self.config.max_output_tokens = 1024 + if self.config.drop_params: + litellm.drop_params = self.config.drop_params + self._completion = partial( litellm_completion, model=self.config.model,