Aspik101's picture
Update app.py
dbb95d1
import gradio as gr
import random
import time
from ctransformers import AutoModelForCausalLM
from dl_hf_model import dl_hf_model
params = {
"max_new_tokens":512,
"stop":["<end>" ,"<|endoftext|>","["],
"temperature":0.7,
"top_p":0.8,
"stream":True,
"batch_size": 8}
#url = "https://huggingface.co/Aspik101/trurl-2-7b-GGML/blob/main/trurl-2-7b.ggmlv3.q8_0.bin"
#model_loc, file_size = dl_hf_model(url)
llm = AutoModelForCausalLM.from_pretrained("Aspik101/trurl-2-13b-GGML", model_type="llama")
with gr.Blocks() as demo:
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.Button("Clear")
def user(user_message, history):
return "", history + [[user_message, None]]
def parse_history(hist):
history_ = ""
for q, a in hist:
history_ += f"<user>: {q } \n"
if a:
history_ += f"<assistant>: {a} \n"
return history_
def bot(history):
print("history: ",history)
prompt = f"Jesteś AI assystentem. Odpowiadaj po polsku. {parse_history(history)}. <assistant>:"
print("prompt: ",prompt)
stream = llm(prompt, **params)
history[-1][1] = ""
answer_save = ""
for character in stream:
history[-1][1] += character
answer_save += character
time.sleep(0.005)
yield history
print("answer_save: ",answer_save)
msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
bot, chatbot, chatbot
)
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue()
demo.launch()