import json
from textwrap import dedent
from typing import Optional, List, Iterator, Dict, Any, Mapping, Union
from phi.llm.base import LLM
from phi.llm.message import Message
from phi.tools.function import FunctionCall
from phi.utils.log import logger
from phi.utils.timer import Timer
from phi.utils.tools import get_function_call_for_tool_call
try:
from ollama import Client as OllamaClient
except ImportError:
logger.error("`ollama` not installed")
raise
class Ollama(LLM):
name: str = "Ollama"
model: str = "openhermes"
host: Optional[str] = None
timeout: Optional[Any] = None
format: Optional[str] = None
options: Optional[Any] = None
keep_alive: Optional[Union[float, str]] = None
client_kwargs: Optional[Dict[str, Any]] = None
ollama_client: Optional[OllamaClient] = None
# Maximum number of function calls allowed across all iterations.
function_call_limit: int = 5
# Deactivate tool calls after 1 tool call
deactivate_tools_after_use: bool = False
# After a tool call is run, add the user message as a reminder to the LLM
add_user_message_after_tool_call: bool = True
@property
def client(self) -> OllamaClient:
if self.ollama_client:
return self.ollama_client
_ollama_params: Dict[str, Any] = {}
if self.host:
_ollama_params["host"] = self.host
if self.timeout:
_ollama_params["timeout"] = self.timeout
if self.client_kwargs:
_ollama_params.update(self.client_kwargs)
return OllamaClient(**_ollama_params)
@property
def api_kwargs(self) -> Dict[str, Any]:
kwargs: Dict[str, Any] = {}
if self.format is not None:
kwargs["format"] = self.format
elif self.response_format is not None:
if self.response_format.get("type") == "json_object":
kwargs["format"] = "json"
# elif self.functions is not None:
# kwargs["format"] = "json"
if self.options is not None:
kwargs["options"] = self.options
if self.keep_alive is not None:
kwargs["keep_alive"] = self.keep_alive
return kwargs
def to_dict(self) -> Dict[str, Any]:
_dict = super().to_dict()
if self.host:
_dict["host"] = self.host
if self.timeout:
_dict["timeout"] = self.timeout
if self.format:
_dict["format"] = self.format
if self.response_format:
_dict["response_format"] = self.response_format
return _dict
def to_llm_message(self, message: Message) -> Dict[str, Any]:
msg = {
"role": message.role,
"content": message.content,
}
if message.model_extra is not None and "images" in message.model_extra:
msg["images"] = message.model_extra.get("images")
return msg
def invoke(self, messages: List[Message]) -> Mapping[str, Any]:
return self.client.chat(
model=self.model,
messages=[self.to_llm_message(m) for m in messages],
**self.api_kwargs,
)
def invoke_stream(self, messages: List[Message]) -> Iterator[Mapping[str, Any]]:
yield from self.client.chat(
model=self.model,
messages=[self.to_llm_message(m) for m in messages],
stream=True,
**self.api_kwargs,
) # type: ignore
def deactivate_function_calls(self) -> None:
# Deactivate tool calls by turning off JSON mode after 1 tool call
# This is triggered when the function call limit is reached.
self.format = ""
def response(self, messages: List[Message]) -> str:
logger.debug("---------- Ollama Response Start ----------")
# -*- Log messages for debugging
for m in messages:
m.log()
response_timer = Timer()
response_timer.start()
response: Mapping[str, Any] = self.invoke(messages=messages)
response_timer.stop()
logger.debug(f"Time to generate response: {response_timer.elapsed:.4f}s")
# logger.debug(f"Ollama response type: {type(response)}")
# logger.debug(f"Ollama response: {response}")
# -*- Parse response
response_message: Mapping[str, Any] = response.get("message") # type: ignore
response_role = response_message.get("role")
response_content: Optional[str] = response_message.get("content")
# -*- Create assistant message
assistant_message = Message(
role=response_role or "assistant",
content=response_content,
)
# Check if the response is a tool call
try:
if response_content is not None:
_tool_call_content = response_content.strip()
if _tool_call_content.startswith("{") and _tool_call_content.endswith("}"):
_tool_call_content_json = json.loads(_tool_call_content)
if "tool_calls" in _tool_call_content_json:
assistant_tool_calls = _tool_call_content_json.get("tool_calls")
if isinstance(assistant_tool_calls, list):
# Build tool calls
tool_calls: List[Dict[str, Any]] = []
logger.debug(f"Building tool calls from {assistant_tool_calls}")
for tool_call in assistant_tool_calls:
tool_call_name = tool_call.get("name")
tool_call_args = tool_call.get("arguments")
_function_def = {"name": tool_call_name}
if tool_call_args is not None:
_function_def["arguments"] = json.dumps(tool_call_args)
tool_calls.append(
{
"type": "function",
"function": _function_def,
}
)
assistant_message.tool_calls = tool_calls
assistant_message.role = "assistant"
except Exception:
logger.warning(f"Could not parse tool calls from response: {response_content}")
pass
# -*- Update usage metrics
# Add response time to metrics
assistant_message.metrics["time"] = response_timer.elapsed
if "response_times" not in self.metrics:
self.metrics["response_times"] = []
self.metrics["response_times"].append(response_timer.elapsed)
# -*- Add assistant message to messages
messages.append(assistant_message)
assistant_message.log()
# -*- Parse and run function call
if assistant_message.tool_calls is not None and self.run_tools:
final_response = ""
function_calls_to_run: List[FunctionCall] = []
for tool_call in assistant_message.tool_calls:
_function_call = get_function_call_for_tool_call(tool_call, self.functions)
if _function_call is None:
messages.append(Message(role="user", content="Could not find function to call."))
continue
if _function_call.error is not None:
messages.append(Message(role="user", content=_function_call.error))
continue
function_calls_to_run.append(_function_call)
if self.show_tool_calls:
if len(function_calls_to_run) == 1:
final_response += f"\n - Running: {function_calls_to_run[0].get_call_str()}\n\n"
elif len(function_calls_to_run) > 1:
final_response += "\nRunning:"
for _f in function_calls_to_run:
final_response += f"\n - {_f.get_call_str()}"
final_response += "\n\n"
function_call_results = self.run_function_calls(function_calls_to_run, role="user")
if len(function_call_results) > 0:
messages.extend(function_call_results)
# Reconfigure messages so the LLM is reminded of the original task
if self.add_user_message_after_tool_call:
messages = self.add_original_user_message(messages)
# Deactivate tool calls by turning off JSON mode after 1 tool call
if self.deactivate_tools_after_use:
self.deactivate_function_calls()
# -*- Yield new response using results of tool calls
final_response += self.response(messages=messages)
return final_response
logger.debug("---------- Ollama Response End ----------")
# -*- Return content if no function calls are present
if assistant_message.content is not None:
return assistant_message.get_content_string()
return "Something went wrong, please try again."
def response_stream(self, messages: List[Message]) -> Iterator[str]:
logger.debug("---------- Ollama Response Start ----------")
# -*- Log messages for debugging
for m in messages:
m.log()
assistant_message_content = ""
response_is_tool_call = False
tool_call_bracket_count = 0
is_last_tool_call_bracket = False
completion_tokens = 0
time_to_first_token = None
response_timer = Timer()
response_timer.start()
for response in self.invoke_stream(messages=messages):
completion_tokens += 1
if completion_tokens == 1:
time_to_first_token = response_timer.elapsed
logger.debug(f"Time to first token: {time_to_first_token:.4f}s")
# -*- Parse response
# logger.info(f"Ollama partial response: {response}")
# logger.info(f"Ollama partial response type: {type(response)}")
response_message: Optional[dict] = response.get("message")
response_content = response_message.get("content") if response_message else None
# logger.info(f"Ollama partial response content: {response_content}")
# Add response content to assistant message
if response_content is not None:
assistant_message_content += response_content
# Strip out tool calls from the response
# If the response is a tool call, it will start with a {
if not response_is_tool_call and assistant_message_content.strip().startswith("{"):
response_is_tool_call = True
# If the response is a tool call, count the number of brackets
if response_is_tool_call and response_content is not None:
if "{" in response_content.strip():
# Add the number of opening brackets to the count
tool_call_bracket_count += response_content.strip().count("{")
# logger.debug(f"Tool call bracket count: {tool_call_bracket_count}")
if "}" in response_content.strip():
# Subtract the number of closing brackets from the count
tool_call_bracket_count -= response_content.strip().count("}")
# Check if the response is the last bracket
if tool_call_bracket_count == 0:
response_is_tool_call = False
is_last_tool_call_bracket = True
# logger.debug(f"Tool call bracket count: {tool_call_bracket_count}")
# -*- Yield content if not a tool call and content is not None
if not response_is_tool_call and response_content is not None:
if is_last_tool_call_bracket and response_content.strip().endswith("}"):
is_last_tool_call_bracket = False
continue
yield response_content
response_timer.stop()
logger.debug(f"Tokens generated: {completion_tokens}")
logger.debug(f"Time per output token: {response_timer.elapsed / completion_tokens:.4f}s")
logger.debug(f"Throughput: {completion_tokens / response_timer.elapsed:.4f} tokens/s")
logger.debug(f"Time to generate response: {response_timer.elapsed:.4f}s")
# -*- Create assistant message
assistant_message = Message(
role="assistant",
content=assistant_message_content,
)
# Check if the response is a tool call
try:
if response_is_tool_call and assistant_message_content != "":
_tool_call_content = assistant_message_content.strip()
if _tool_call_content.startswith("{") and _tool_call_content.endswith("}"):
_tool_call_content_json = json.loads(_tool_call_content)
if "tool_calls" in _tool_call_content_json:
assistant_tool_calls = _tool_call_content_json.get("tool_calls")
if isinstance(assistant_tool_calls, list):
# Build tool calls
tool_calls: List[Dict[str, Any]] = []
logger.debug(f"Building tool calls from {assistant_tool_calls}")
for tool_call in assistant_tool_calls:
tool_call_name = tool_call.get("name")
tool_call_args = tool_call.get("arguments")
_function_def = {"name": tool_call_name}
if tool_call_args is not None:
_function_def["arguments"] = json.dumps(tool_call_args)
tool_calls.append(
{
"type": "function",
"function": _function_def,
}
)
assistant_message.tool_calls = tool_calls
except Exception:
logger.warning(f"Could not parse tool calls from response: {assistant_message_content}")
pass
# -*- Update usage metrics
# Add response time to metrics
assistant_message.metrics["time"] = f"{response_timer.elapsed:.4f}"
assistant_message.metrics["time_to_first_token"] = f"{time_to_first_token:.4f}s"
assistant_message.metrics["time_per_output_token"] = f"{response_timer.elapsed / completion_tokens:.4f}s"
if "response_times" not in self.metrics:
self.metrics["response_times"] = []
self.metrics["response_times"].append(response_timer.elapsed)
if "time_to_first_token" not in self.metrics:
self.metrics["time_to_first_token"] = []
self.metrics["time_to_first_token"].append(f"{time_to_first_token:.4f}s")
if "tokens_per_second" not in self.metrics:
self.metrics["tokens_per_second"] = []
self.metrics["tokens_per_second"].append(f"{completion_tokens / response_timer.elapsed:.4f}")
# -*- Add assistant message to messages
messages.append(assistant_message)
assistant_message.log()
# -*- Parse and run function call
if assistant_message.tool_calls is not None and self.run_tools:
function_calls_to_run: List[FunctionCall] = []
for tool_call in assistant_message.tool_calls:
_function_call = get_function_call_for_tool_call(tool_call, self.functions)
if _function_call is None:
messages.append(Message(role="user", content="Could not find function to call."))
continue
if _function_call.error is not None:
messages.append(Message(role="user", content=_function_call.error))
continue
function_calls_to_run.append(_function_call)
if self.show_tool_calls:
if len(function_calls_to_run) == 1:
yield f"\n - Running: {function_calls_to_run[0].get_call_str()}\n\n"
elif len(function_calls_to_run) > 1:
yield "\nRunning:"
for _f in function_calls_to_run:
yield f"\n - {_f.get_call_str()}"
yield "\n\n"
function_call_results = self.run_function_calls(function_calls_to_run, role="user")
# Add results of the function calls to the messages
if len(function_call_results) > 0:
messages.extend(function_call_results)
# Reconfigure messages so the LLM is reminded of the original task
if self.add_user_message_after_tool_call:
messages = self.add_original_user_message(messages)
# Deactivate tool calls by turning off JSON mode after 1 tool call
if self.deactivate_tools_after_use:
self.deactivate_function_calls()
# -*- Yield new response using results of tool calls
yield from self.response_stream(messages=messages)
logger.debug("---------- Ollama Response End ----------")
def add_original_user_message(self, messages: List[Message]) -> List[Message]:
# Add the original user message to the messages to remind the LLM of the original task
original_user_message_content = None
for m in messages:
if m.role == "user":
original_user_message_content = m.content
break
if original_user_message_content is not None:
_content = (
"Using the results of the tools above, respond to the following message:"
f"\n\n\n{original_user_message_content}\n"
)
messages.append(Message(role="user", content=_content))
return messages
def get_instructions_to_generate_tool_calls(self) -> List[str]:
if self.functions is not None:
return [
"To respond to the users message, you can use one or more of the tools provided above.",
"If you decide to use a tool, you must respond in the JSON format matching the following schema:\n"
+ dedent(
"""\
{
"tool_calls": [{
"name": "",
"arguments": Optional[str]:
if self.functions is not None:
_tool_choice_prompt = "To respond to the users message, you have access to the following tools:"
for _f_name, _function in self.functions.items():
_function_definition = _function.get_definition_for_prompt()
if _function_definition:
_tool_choice_prompt += f"\n{_function_definition}"
_tool_choice_prompt += "\n\n"
return _tool_choice_prompt
return None
def get_system_prompt_from_llm(self) -> Optional[str]:
return self.get_tool_calls_definition()
def get_instructions_from_llm(self) -> Optional[List[str]]:
return self.get_instructions_to_generate_tool_calls()