Spaces:
Sleeping
Sleeping
File size: 1,221 Bytes
ed921e2 072b88d a1ba27b ed921e2 a1ba27b ed921e2 a1ba27b 072b88d a1ba27b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
import os
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
# Retrieve the Hugging Face token from environment variables
hf_token = os.getenv("HUGGINGFACE_TOKEN")
# Load the tokenizer and model with the token
model_id = "meta-llama/Llama-3.2-3B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=hf_token)
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto", use_auth_token=hf_token)
# Define the prediction function
def generate_text(prompt):
inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=131072)
with torch.no_grad():
outputs = model.generate(**inputs, max_length=131072)
return tokenizer.decode(outputs[0], skip_special_tokens=True)
# Create the Gradio interface
interface = gr.Interface(
fn=generate_text,
inputs=gr.Textbox(lines=10, label="Input Prompt"),
outputs=gr.Textbox(lines=10, label="Generated Text"),
title="Meta Llama 3.2 3B Instruct Model",
description="Generate text using the Meta Llama 3.2 3B Instruct model with a context length of up to 128,000 tokens."
)
if __name__ == "__main__":
interface.launch()
|