Spaces:
Build error
Build error
| from __future__ import annotations | |
| import os | |
| from typing import Any | |
| from pydantic import BaseModel, Field, SecretStr, ValidationError | |
| from openhands.core.logger import LOG_DIR | |
| from openhands.core.logger import openhands_logger as logger | |
| class LLMConfig(BaseModel): | |
| """Configuration for the LLM model. | |
| Attributes: | |
| model: The model to use. | |
| api_key: The API key to use. | |
| base_url: The base URL for the API. This is necessary for local LLMs. | |
| api_version: The version of the API. | |
| aws_access_key_id: The AWS access key ID. | |
| aws_secret_access_key: The AWS secret access key. | |
| aws_region_name: The AWS region name. | |
| num_retries: The number of retries to attempt. | |
| retry_multiplier: The multiplier for the exponential backoff. | |
| retry_min_wait: The minimum time to wait between retries, in seconds. This is exponential backoff minimum. For models with very low limits, this can be set to 15-20. | |
| retry_max_wait: The maximum time to wait between retries, in seconds. This is exponential backoff maximum. | |
| timeout: The timeout for the API. | |
| max_message_chars: The approximate max number of characters in the content of an event included in the prompt to the LLM. Larger observations are truncated. | |
| temperature: The temperature for the API. | |
| top_p: The top p for the API. | |
| top_k: The top k for the API. | |
| custom_llm_provider: The custom LLM provider to use. This is undocumented in openhands, and normally not used. It is documented on the litellm side. | |
| max_input_tokens: The maximum number of input tokens. Note that this is currently unused, and the value at runtime is actually the total tokens in OpenAI (e.g. 128,000 tokens for GPT-4). | |
| max_output_tokens: The maximum number of output tokens. This is sent to the LLM. | |
| input_cost_per_token: The cost per input token. This will available in logs for the user to check. | |
| output_cost_per_token: The cost per output token. This will available in logs for the user to check. | |
| ollama_base_url: The base URL for the OLLAMA API. | |
| drop_params: Drop any unmapped (unsupported) params without causing an exception. | |
| modify_params: Modify params allows litellm to do transformations like adding a default message, when a message is empty. | |
| disable_vision: If model is vision capable, this option allows to disable image processing (useful for cost reduction). | |
| caching_prompt: Use the prompt caching feature if provided by the LLM and supported by the provider. | |
| log_completions: Whether to log LLM completions to the state. | |
| log_completions_folder: The folder to log LLM completions to. Required if log_completions is True. | |
| custom_tokenizer: A custom tokenizer to use for token counting. | |
| native_tool_calling: Whether to use native tool calling if supported by the model. Can be True, False, or not set. | |
| reasoning_effort: The effort to put into reasoning. This is a string that can be one of 'low', 'medium', 'high', or 'none'. Exclusive for o1 models. | |
| seed: The seed to use for the LLM. | |
| """ | |
| model: str = Field(default='claude-sonnet-4-20250514') | |
| api_key: SecretStr | None = Field(default=None) | |
| base_url: str | None = Field(default=None) | |
| api_version: str | None = Field(default=None) | |
| aws_access_key_id: SecretStr | None = Field(default=None) | |
| aws_secret_access_key: SecretStr | None = Field(default=None) | |
| aws_region_name: str | None = Field(default=None) | |
| openrouter_site_url: str = Field(default='https://docs.all-hands.dev/') | |
| openrouter_app_name: str = Field(default='OpenHands') | |
| # total wait time: 5 + 10 + 20 + 30 = 65 seconds | |
| num_retries: int = Field(default=4) | |
| retry_multiplier: float = Field(default=2) | |
| retry_min_wait: int = Field(default=5) | |
| retry_max_wait: int = Field(default=30) | |
| timeout: int | None = Field(default=None) | |
| max_message_chars: int = Field( | |
| default=30_000 | |
| ) # maximum number of characters in an observation's content when sent to the llm | |
| temperature: float = Field(default=0.0) | |
| top_p: float = Field(default=1.0) | |
| top_k: float | None = Field(default=None) | |
| custom_llm_provider: str | None = Field(default=None) | |
| max_input_tokens: int | None = Field(default=None) | |
| max_output_tokens: int | None = Field(default=None) | |
| input_cost_per_token: float | None = Field(default=None) | |
| output_cost_per_token: float | None = Field(default=None) | |
| ollama_base_url: str | None = Field(default=None) | |
| # This setting can be sent in each call to litellm | |
| drop_params: bool = Field(default=True) | |
| # Note: this setting is actually global, unlike drop_params | |
| modify_params: bool = Field(default=True) | |
| disable_vision: bool | None = Field(default=None) | |
| caching_prompt: bool = Field(default=True) | |
| log_completions: bool = Field(default=False) | |
| log_completions_folder: str = Field(default=os.path.join(LOG_DIR, 'completions')) | |
| custom_tokenizer: str | None = Field(default=None) | |
| native_tool_calling: bool | None = Field(default=None) | |
| reasoning_effort: str | None = Field(default='high') | |
| seed: int | None = Field(default=None) | |
| model_config = {'extra': 'forbid'} | |
| def from_toml_section(cls, data: dict) -> dict[str, LLMConfig]: | |
| """ | |
| Create a mapping of LLMConfig instances from a toml dictionary representing the [llm] section. | |
| The default configuration is built from all non-dict keys in data. | |
| Then, each key with a dict value (e.g. [llm.random_name]) is treated as a custom LLM configuration, | |
| and its values override the default configuration. | |
| Example: | |
| Apply generic LLM config with custom LLM overrides, e.g. | |
| [llm] | |
| model=... | |
| num_retries = 5 | |
| [llm.claude] | |
| model="claude-3-5-sonnet" | |
| results in num_retries APPLIED to claude-3-5-sonnet. | |
| Returns: | |
| dict[str, LLMConfig]: A mapping where the key "llm" corresponds to the default configuration | |
| and additional keys represent custom configurations. | |
| """ | |
| # Initialize the result mapping | |
| llm_mapping: dict[str, LLMConfig] = {} | |
| # Extract base config data (non-dict values) | |
| base_data = {} | |
| custom_sections: dict[str, dict] = {} | |
| for key, value in data.items(): | |
| if isinstance(value, dict): | |
| custom_sections[key] = value | |
| else: | |
| base_data[key] = value | |
| # Try to create the base config | |
| try: | |
| base_config = cls.model_validate(base_data) | |
| llm_mapping['llm'] = base_config | |
| except ValidationError: | |
| logger.warning( | |
| 'Cannot parse [llm] config from toml. Continuing with defaults.' | |
| ) | |
| # If base config fails, create a default one | |
| base_config = cls() | |
| # Still add it to the mapping | |
| llm_mapping['llm'] = base_config | |
| # Process each custom section independently | |
| for name, overrides in custom_sections.items(): | |
| try: | |
| # Merge base config with overrides | |
| merged = {**base_config.model_dump(), **overrides} | |
| custom_config = cls.model_validate(merged) | |
| llm_mapping[name] = custom_config | |
| except ValidationError: | |
| logger.warning( | |
| f'Cannot parse [{name}] config from toml. This section will be skipped.' | |
| ) | |
| # Skip this custom section but continue with others | |
| continue | |
| return llm_mapping | |
| def model_post_init(self, __context: Any) -> None: | |
| """Post-initialization hook to assign OpenRouter-related variables to environment variables. | |
| This ensures that these values are accessible to litellm at runtime. | |
| """ | |
| super().model_post_init(__context) | |
| # Assign OpenRouter-specific variables to environment variables | |
| if self.openrouter_site_url: | |
| os.environ['OR_SITE_URL'] = self.openrouter_site_url | |
| if self.openrouter_app_name: | |
| os.environ['OR_APP_NAME'] = self.openrouter_app_name | |
| # Set an API version by default for Azure models | |
| # Required for newer models. | |
| # Azure issue: https://github.com/All-Hands-AI/OpenHands/issues/7755 | |
| if self.model.startswith('azure') and self.api_version is None: | |
| self.api_version = '2024-12-01-preview' | |