File size: 1,655 Bytes
9455096
 
 
 
97a1535
 
9455096
 
 
dac0248
9455096
 
 
 
 
dac0248
 
cd25fb2
 
9455096
aa95e28
9455096
 
 
 
 
 
 
 
dac0248
 
 
 
 
 
 
 
9455096
dac0248
 
 
 
9455096
7837c89
9455096
 
7837c89
9455096
 
 
dac0248
9455096
 
 
 
577e046
9455096
dbb95d1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import gradio as gr
import random
import time
from ctransformers import AutoModelForCausalLM
from dl_hf_model import dl_hf_model


params = {
        "max_new_tokens":512,
        "stop":["<end>" ,"<|endoftext|>","["],
        "temperature":0.7,
        "top_p":0.8,
        "stream":True,
        "batch_size": 8}


#url = "https://huggingface.co/Aspik101/trurl-2-7b-GGML/blob/main/trurl-2-7b.ggmlv3.q8_0.bin"
#model_loc, file_size = dl_hf_model(url)
llm = AutoModelForCausalLM.from_pretrained("Aspik101/trurl-2-13b-GGML", model_type="llama")


with gr.Blocks() as demo:
    chatbot = gr.Chatbot()
    msg = gr.Textbox()
    clear = gr.Button("Clear")

    def user(user_message, history):
        return "", history + [[user_message, None]]

    def parse_history(hist):
        history_ = ""
        for q, a in hist:
            history_ += f"<user>: {q } \n"
            if a:
                history_ += f"<assistant>: {a} \n"
        return history_

    def bot(history):
        print("history: ",history)
        prompt =  f"Jesteś AI assystentem. Odpowiadaj po polsku. {parse_history(history)}. <assistant>:"
        print("prompt: ",prompt)
        stream = llm(prompt, **params)
        history[-1][1] = ""
        answer_save = ""
        for character in stream:
            history[-1][1] += character
            answer_save += character
            time.sleep(0.005)
            yield history

        print("answer_save: ",answer_save)
    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
        bot, chatbot, chatbot
    )
    clear.click(lambda: None, None, chatbot, queue=False)
     
demo.queue()
demo.launch()