app.py

import gradio as gr
import random
import time
from ctransformers import AutoModelForCausalLM
import datetime


params = {
        "max_new_tokens":512,
        "stop":["<end>" ,"<|endoftext|>"],
        "temperature":0.7,
        "top_p":0.8,
        "stream":True,
        "batch_size": 8}


def save_log(task, to_save):
  with open(log_file_path, "a") as log_file:
    current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    log_file.write(f"[{current_time}] - {task}: {to_save}\n")
        

llm = AutoModelForCausalLM.from_pretrained("Aspik101/Llama-2-7b-chat-hf-pl-lora_GGML", model_type="llama")

with gr.Blocks(allow_flagging="auto") as demo:
    chatbot = gr.Chatbot()
    msg = gr.Textbox()
    clear = gr.Button("Clear")

    def user(user_message, history):
        return "", history + [[user_message, None]]

    def bot(history):
        save_log("question", history)
        stream = llm(prompt =  f"Jesteś AI assystentem. Odpowiadaj po polsku. <user>: {history}. <assistant>:", **params)
        history[-1][1] = ""
        answer_save = ""
        for character in stream:
            history[-1][1] += character
            answer_save += character
            time.sleep(0.005)
            yield history

        save_log("answer", answer_save)
    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
        bot, chatbot, chatbot
    )
    clear.click(lambda: None, None, chatbot, queue=False)
    
demo.queue()
demo.launch()