app.py

import os
import time
import spaces
import json
import subprocess
from llama_cpp import Llama
from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
from llama_cpp_agent.providers import LlamaCppPythonProvider
from llama_cpp_agent.chat_history import BasicChatHistory
from llama_cpp_agent.chat_history.messages import Roles
import gradio as gr
from huggingface_hub import hf_hub_download

# Define model details
MODEL_REPO = "prithivMLmods/Sombrero-QwQ-32B-Elite10-Fixed-Q2_K-GGUF"
MODEL_FILENAME = "sombrero-qwq-32b-elite10-fixed-q2_k.gguf"
MODEL_DIR = "./models"
MODEL_PATH = os.path.join(MODEL_DIR, MODEL_FILENAME)

# Ensure the model directory exists
os.makedirs(MODEL_DIR, exist_ok=True)

# Download the model if not already present
if not os.path.exists(MODEL_PATH):
    print("Downloading the model... This may take some time.")
    try:
        hf_hub_download(
            repo_id=MODEL_REPO,
            filename=MODEL_FILENAME,
            local_dir=MODEL_DIR
        )
        print("Model downloaded successfully!")
    except Exception as e:
        print(f"Error downloading model: {e}")
        exit(1)

# Ensure model is fully downloaded before using
while not os.path.exists(MODEL_PATH):
    print("Waiting for model to be available...")
    time.sleep(5)

# Function to handle AI responses
@spaces.GPU(duration=120)
def respond(
    message,
    history: list[tuple[str, str]],
    model,
    system_message,
    max_tokens,
    temperature,
    top_p,
    top_k,
    repeat_penalty,
):
    chat_template = MessagesFormatterType.GEMMA_2

    llm = Llama(
        model_path=MODEL_PATH,
        flash_attn=True,
        n_gpu_layers=81,
        n_batch=1024,
        n_ctx=8192,
    )
    provider = LlamaCppPythonProvider(llm)

    agent = LlamaCppAgent(
        provider,
        system_prompt=f"{system_message}",
        predefined_messages_formatter_type=chat_template,
        debug_output=True
    )

    settings = provider.get_provider_default_settings()
    settings.temperature = temperature
    settings.top_k = top_k
    settings.top_p = top_p
    settings.max_tokens = max_tokens
    settings.repeat_penalty = repeat_penalty
    settings.stream = True

    messages = BasicChatHistory()

    for msn in history:
        user = {'role': Roles.user, 'content': msn[0]}
        assistant = {'role': Roles.assistant, 'content': msn[1]}
        messages.add_message(user)
        messages.add_message(assistant)

    stream = agent.get_chat_response(
        message,
        llm_sampling_settings=settings,
        chat_history=messages,
        returns_streaming_generator=True,
        print_output=False
    )

    outputs = ""
    for output in stream:
        outputs += output
        yield outputs

# Function to create Gradio interface
def create_interface(model_name, description):
    return gr.ChatInterface(
        respond,
        additional_inputs=[
            gr.Textbox(value=model_name, label="Model", interactive=False),
            gr.Textbox(
                value="You are a helpful assistant. You should think step-by-step. You should answer all questions by prioritizing the principle of openness",
                label="System message"
            ),
            gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
            gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
            gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
            gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k"),
            gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty"),
        ],
        retry_btn="Retry",
        undo_btn="Undo",
        clear_btn="Clear",
        submit_btn="Send",
        title=f"{model_name}",
        description=description,
        chatbot=gr.Chatbot(scale=1, likeable=False, show_copy_button=True)
    )

# Set interface description
description = """<p align="center">Viper-Coder-32B-Elite13-GGUF</p>"""
interface = create_interface(MODEL_REPO, description)

# Create Gradio Blocks app
demo = gr.Blocks()

with demo:
    interface.render()

if __name__ == "__main__":
    demo.launch(share=True)