Discord

import spaces import json import subprocess import time import gradio as gr from llama_cpp import Llama from llama_cpp_agent import LlamaCppAgent from llama_cpp_agent.providers import LlamaCppPythonProvider from llama_cpp_agent.chat_history import BasicChatHistory from llama_cpp_agent.chat_history.messages import Roles from llama_cpp_agent.llm_output_settings import LlmStructuredOutputSettings from huggingface_hub import hf_hub_download from web_search import WebSearchTool model_selected = "Mistral-7B-Instruct-v0.3-Q6_K.gguf" examples = [ ["latest news about Yann LeCun"], ["Latest news site:github.blog"], ["Where I can find best hotel in Galapagos, Ecuador intitle:hotel"], ["filetype:pdf intitle:python"] ] hf_hub_download( repo_id="bartowski/Mistral-7B-Instruct-v0.3-GGUF", filename="Mistral-7B-Instruct-v0.3-Q6_K.gguf", local_dir="./models" ) hf_hub_download( repo_id="bartowski/Meta-Llama-3-8B-Instruct-GGUF", filename="Meta-Llama-3-8B-Instruct-Q6_K.gguf", local_dir="./models" ) css = """ .message-row { justify-content: space-evenly !important; } .message-bubble-border { border-radius: 6px !important; } .dark.message-bubble-border { border-color: #1b0f0f !important; } .dark.user { background: #140b0b !important; } .dark.assistant.dark, .dark.pending.dark { background: #0c0505 !important; } """ PLACEHOLDER = """

llama-cpp-agent

DDG Agent allows users to interact with it using natural language, making it easier for them to find the information they need. Offers a convenient and secure way for users to access web-based information.

Mistral 7B Instruct v0.3 Meta Llama 3 8B Instruct

""" def get_context_by_model(model_name): model_context_limits = { "Mistral-7B-Instruct-v0.3-Q6_K.gguf": 32768, "Meta-Llama-3-8B-Instruct-Q6_K.gguf": 8192 } return model_context_limits.get(model_name, None) def get_messages_formatter_type(model_name): from llama_cpp_agent import MessagesFormatterType if "Meta" in model_name or "aya" in model_name: return MessagesFormatterType.LLAMA_3 elif "Mistral" in model_name: return MessagesFormatterType.MISTRAL elif "Einstein-v6-7B" in model_name or "dolphin" in model_name: return MessagesFormatterType.CHATML elif "Phi" in model_name: return MessagesFormatterType.PHI_3 else: return MessagesFormatterType.CHATML def write_message_to_user(): """ Let you write a message to the user. """ return "Please write the message to the user." @spaces.GPU(duration=120) def respond( message, history: list[tuple[str, str]], model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty, ): chat_template = get_messages_formatter_type(model) model_selected = model system_message += f" {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))}" llm = Llama( model_path=f"models/{model}", flash_attn=True, n_threads=40, n_gpu_layers=81, n_batch=1024, n_ctx=get_context_by_model(model), ) provider = LlamaCppPythonProvider(llm) agent = LlamaCppAgent( provider, system_prompt=f"{system_message}", predefined_messages_formatter_type=chat_template, debug_output=True ) search_tool = WebSearchTool(provider, chat_template, get_context_by_model(model)) settings = provider.get_provider_default_settings() settings.temperature = temperature settings.top_k = top_k settings.top_p = top_p settings.max_tokens = max_tokens settings.repeat_penalty = repeat_penalty settings.stream = True output_settings = LlmStructuredOutputSettings.from_functions( [search_tool.get_tool(), write_message_to_user]) messages = BasicChatHistory() for msn in history: user = { 'role': Roles.user, 'content': msn[0] } assistant = { 'role': Roles.assistant, 'content': msn[1] } messages.add_message(user) messages.add_message(assistant) result = agent.get_chat_response(message, llm_sampling_settings=settings, structured_output_settings=output_settings, chat_history=messages, print_output=False) while True: if result[0]["function"] == "write_message_to_user": break else: result = agent.get_chat_response(result[0]["return_value"], role=Roles.tool, chat_history=messages,structured_output_settings=output_settings, print_output=False) stream = agent.get_chat_response( result[0]["return_value"], role=Roles.tool, llm_sampling_settings=settings, chat_history=messages, returns_streaming_generator=True, print_output=False ) outputs = "" for output in stream: outputs += output yield outputs demo = gr.ChatInterface( respond, additional_inputs=[ gr.Dropdown([ 'Mistral-7B-Instruct-v0.3-Q6_K.gguf', 'Meta-Llama-3-8B-Instruct-Q6_K.gguf' ], value="Mistral-7B-Instruct-v0.3-Q6_K.gguf", label="Model" ), gr.Textbox(value="You are a helpful assistant. Use additional available information you have access to when giving a response. Always give detailed and long responses. Format your response, well structured in markdown format.", label="System message"), gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p", ), gr.Slider( minimum=0, maximum=100, value=40, step=1, label="Top-k", ), gr.Slider( minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty", ), ], theme=gr.themes.Soft( primary_hue="orange", secondary_hue="amber", neutral_hue="gray", font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( body_background_fill_dark="#0c0505", block_background_fill_dark="#0c0505", block_border_width="1px", block_title_background_fill_dark="#1b0f0f", input_background_fill_dark="#140b0b", button_secondary_background_fill_dark="#140b0b", border_color_primary_dark="#1b0f0f", background_fill_secondary_dark="#0c0505", color_accent_soft_dark="transparent" ), css=css, retry_btn="Retry", undo_btn="Undo", clear_btn="Clear", submit_btn="Send", examples = (examples), description="Llama-cpp-agent: Chat Web Search DDG Agent", chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER) ) if __name__ == "__main__": demo.launch()