Spaces:
Runtime error
Runtime error
File size: 1,689 Bytes
315691e 3b3c5cf 2fcb420 315691e 2fcb420 3b3c5cf 2fcb420 3b3c5cf 315691e 2fcb420 315691e 2fcb420 315691e 2fcb420 315691e 2fcb420 315691e 2fcb420 315691e 2fcb420 315691e 2fcb420 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import gradio as gr
# Model and tokenizer paths
model_name = "ahmedbasemdev/llama-3.2-3b-ChatBot"
# Load the model
print("Loading the model...")
model = AutoModelForCausalLM.from_pretrained(model_name)
# Apply dynamic quantization to reduce model size and improve CPU performance
print("Applying quantization...")
model = torch.quantization.quantize_dynamic(
model, # Model to quantize
{torch.nn.Linear}, # Layers to quantize (e.g., Linear layers)
dtype=torch.qint8, # Quantized data type
)
# Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained(model_name)
# Define the inference function
def single_inference(question):
messages = []
messages.append({"role": "user", "content": question})
# Tokenize the input
input_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
return_tensors="pt"
).to("cpu") # Ensure everything runs on CPU
# Generate a response
terminators = [
tokenizer.eos_token_id,
tokenizer.convert_tokens_to_ids("<|eot_id|>")
]
outputs = model.generate(
input_ids,
max_new_tokens=256,
eos_token_id=terminators,
do_sample=True,
temperature=0.2,
)
response = outputs[0][input_ids.shape[-1]:]
output = tokenizer.decode(response, skip_special_tokens=True)
return output
# Gradio interface
print("Setting up Gradio app...")
interface = gr.Interface(
fn=single_inference,
inputs="text",
outputs="text",
title="Chatbot",
description="Ask me anything!"
)
# Launch the Gradio app
interface.launch()
|