app.py

import gradio as gr
import os
import requests
from llama_cpp import Llama

# 确保模型文件夹存在
model_dir = "/home/user/models"
os.makedirs(model_dir, exist_ok=True)

# GGUF 模型文件名
model_name = "unsloth.Q8_0.gguf"
model_path = os.path.join(model_dir, model_name)

# 如果模型文件不存在，则从 Hugging Face 下载
hf_model_url = "https://huggingface.co/YLX1965/medical-model/resolve/main/unsloth.Q8_0.gguf"

if not os.path.exists(model_path):
    print(f"Downloading model from {hf_model_url}...")
    response = requests.get(hf_model_url, stream=True)
    with open(model_path, "wb") as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)
    print("Download complete.")

# 加载 GGUF 模型
llm = Llama(model_path=model_path)

# 定义聊天函数
def chat(prompt):
    output = llm(prompt, max_tokens=200)
    return output["choices"][0]["text"]

# 创建 Gradio 界面
interface = gr.Interface(fn=chat, inputs="text", outputs="text",
                         title="Medical Chatbot",
                         description="使用 GGUF 量化模型进行医疗文本生成")

interface.launch()