diff --git a/config.template.toml b/config.template.toml index e40bfc4d4d..1b6cd11c9a 100644 --- a/config.template.toml +++ b/config.template.toml @@ -200,6 +200,23 @@ model = "gpt-4o" # https://github.com/All-Hands-AI/OpenHands/pull/4711 #native_tool_calling = None +# Safety settings for models that support them (e.g., Mistral AI, Gemini) +# Example for Mistral AI: +# safety_settings = [ +# { "category" = "hate", "threshold" = "low" }, +# { "category" = "harassment", "threshold" = "low" }, +# { "category" = "sexual", "threshold" = "low" }, +# { "category" = "dangerous", "threshold" = "low" } +# ] +# +# Example for Gemini: +# safety_settings = [ +# { "category" = "HARM_CATEGORY_HARASSMENT", "threshold" = "BLOCK_NONE" }, +# { "category" = "HARM_CATEGORY_HATE_SPEECH", "threshold" = "BLOCK_NONE" }, +# { "category" = "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold" = "BLOCK_NONE" }, +# { "category" = "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold" = "BLOCK_NONE" } +# ] +#safety_settings = [] [llm.gpt4o-mini] diff --git a/docs/usage/llms/llms.mdx b/docs/usage/llms/llms.mdx index 8b7d6b1859..8dea4beffb 100644 --- a/docs/usage/llms/llms.mdx +++ b/docs/usage/llms/llms.mdx @@ -73,6 +73,15 @@ We have a few guides for running OpenHands with specific model providers: - [OpenAI](/usage/llms/openai-llms) - [OpenRouter](/usage/llms/openrouter) +## Model Customization + +LLM providers have specific settings that can be customized to optimize their performance with OpenHands, such as: + +- **Custom Tokenizers**: For specialized models, you can add a suitable tokenizer +- **Native Tool Calling**: Toggle native function/tool calling capabilities + +For detailed information about model customization, see [LLM Configuration Options](configuration-options#llm-customization). + ### API retries and rate limits LLM providers typically have rate limits, sometimes very low, and may require retries. OpenHands will automatically diff --git a/openhands/core/config/llm_config.py b/openhands/core/config/llm_config.py index 6a3fe10190..c39986746a 100644 --- a/openhands/core/config/llm_config.py +++ b/openhands/core/config/llm_config.py @@ -45,6 +45,7 @@ class LLMConfig(BaseModel): native_tool_calling: Whether to use native tool calling if supported by the model. Can be True, False, or not set. reasoning_effort: The effort to put into reasoning. This is a string that can be one of 'low', 'medium', 'high', or 'none'. Exclusive for o1 models. seed: The seed to use for the LLM. + safety_settings: Safety settings for models that support them (like Mistral AI and Gemini). """ model: str = Field(default='claude-sonnet-4-20250514') @@ -86,6 +87,10 @@ class LLMConfig(BaseModel): native_tool_calling: bool | None = Field(default=None) reasoning_effort: str | None = Field(default='high') seed: int | None = Field(default=None) + safety_settings: list[dict[str, str]] | None = Field( + default=None, + description='Safety settings for models that support them (like Mistral AI and Gemini)', + ) model_config = {'extra': 'forbid'} diff --git a/openhands/llm/llm.py b/openhands/llm/llm.py index 91e0c79f15..1fddb51218 100644 --- a/openhands/llm/llm.py +++ b/openhands/llm/llm.py @@ -182,6 +182,12 @@ class LLM(RetryMixin, DebugMixin): kwargs['max_tokens'] = self.config.max_output_tokens kwargs.pop('max_completion_tokens') + # Add safety settings for models that support them + if 'mistral' in self.config.model.lower() and self.config.safety_settings: + kwargs['safety_settings'] = self.config.safety_settings + elif 'gemini' in self.config.model.lower() and self.config.safety_settings: + kwargs['safety_settings'] = self.config.safety_settings + self._completion = partial( litellm_completion, model=self.config.model,