File size: 1,244 Bytes
2850349 6299c1f 2850349 6299c1f 2850349 6299c1f 2850349 6299c1f 2850349 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 |
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
import gradio as gr
# Load model and tokenizer
model_name = "tomg-group-umd/huginn-0125"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name)
# Function to generate text
def generate_response(prompt, num_steps):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(device)
model.eval()
with torch.no_grad():
output = model.generate(input_ids, num_steps=num_steps, max_length=256)
response = tokenizer.decode(output[0], skip_special_tokens=True)
return response
# Gradio interface
iface = gr.Interface(
fn=generate_response,
inputs=[
gr.Textbox(lines=5, label="Input Prompt"),
gr.Slider(minimum=4, maximum=64, step=1, value=16, label="Computation Scale (num_steps)")
],
outputs="text",
title="Huginn-0125 Text Generation",
description="Generate text using the Huginn-0125 model with adjustable computation scale."
)
# Run app
if __name__ == "__main__":
iface.launch() |