app.py

from transformers import pipeline, BitsAndBytesConfig
import gradio as gr
import torch

pipeline = pipeline("text-generation", model="google/gemma-2-2b", torch_dtype=torch.bfloat16, device_map="auto", model_kwargs={"quantization_config": BitsAndBytesConfig(load_in_8bit=True)})
gr.Interface.from_pipeline(pipeline).launch()