app.py

import os
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# Retrieve the Hugging Face token from environment variables
hf_token = os.getenv("HUGGINGFACE_TOKEN")

# Load the tokenizer and model with the token
model_id = "meta-llama/Llama-3.2-3B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=hf_token)
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto", use_auth_token=hf_token)

# Define the prediction function
def generate_text(prompt):
    inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=131072)
    with torch.no_grad():
        outputs = model.generate(**inputs, max_length=131072)
    return tokenizer.decode(outputs[0], skip_special_tokens=True)

# Create the Gradio interface
interface = gr.Interface(
    fn=generate_text,
    inputs=gr.Textbox(lines=10, label="Input Prompt"),
    outputs=gr.Textbox(lines=10, label="Generated Text"),
    title="Meta Llama 3.2 3B Instruct Model",
    description="Generate text using the Meta Llama 3.2 3B Instruct model with a context length of up to 128,000 tokens."
)

if __name__ == "__main__":
    interface.launch()