File size: 1,354 Bytes
2774bcf 1d05594 f975005 1d05594 cd8908c f975005 cd8908c 1d05594 cd8908c f975005 8fa6632 1d05594 f975005 cd8908c f975005 8fa6632 f975005 cd8908c f975005 2774bcf cd8908c 2774bcf f975005 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch
# Load model and tokenizer
model_name = "unsloth/Llama-3.2-1B-Instruct" # Use the non-quantized version
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype=torch.float32,
low_cpu_mem_usage=True,
device_map="cpu"
)
def generate_text(prompt, max_new_tokens, temperature):
inputs = tokenizer(prompt, return_tensors="pt")
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=int(max_new_tokens),
temperature=temperature,
num_return_sequences=1,
do_sample=True,
)
return tokenizer.decode(outputs[0], skip_special_tokens=True)
# Define the Gradio interface
iface = gr.Interface(
fn=generate_text,
inputs=[
gr.Textbox(lines=5, label="Enter your prompt"),
gr.Slider(50, 500, value=200, step=1, label="Maximum New Tokens"),
gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
],
outputs=gr.Textbox(label="Generated Text"),
title="Text Generation with Llama-3.2-1B-Instruct",
description="Enter a prompt to generate text using the Llama-3.2-1B-Instruct model."
)
# Launch the interface
iface.launch() |