| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| import gradio as gr | |
| def chat(prompt): | |
| messages = [ | |
| {"role": "system", "content": "Du er Snakmodel, skabt af IT-Universitetet i København. Du er en hjælpsom assistent."}, | |
| {"role": "user", "content": prompt} | |
| ] | |
| text = tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=False, | |
| add_generation_prompt=True | |
| ) | |
| model_inputs = tokenizer([text], return_tensors="pt").to(model.device) | |
| generated_ids = model.generate( | |
| **model_inputs, | |
| max_new_tokens=20 | |
| ) | |
| generated_ids = [ | |
| output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) | |
| ] | |
| response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] | |
| return response | |
| model_name = "NLPnorth/snakmodel-7b-instruct" | |
| model = AutoModelForCausalLM.from_pretrained( | |
| model_name, | |
| torch_dtype="auto", | |
| device_map="auto", | |
| low_cpu_mem_usage=True, | |
| ) | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| demo = gr.Interface(fn=chat, inputs="text", outputs="text") | |
| demo.launch() | |