Update app.py

app.py

@@ -1,44 +1,25 @@
 import gradio
 
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-
-MODEL_NAME = "arnir0/Tiny-LLM"
-
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
-
-def generate_text(prompt, model, tokenizer, max_length=4096, temperature=0.8, top_k=50, top_p=0.95):
-    inputs = tokenizer.encode(prompt, return_tensors="pt")
-
-    outputs = model.generate(
-        inputs,
-        max_length=max_length,
-        temperature=temperature,
-        top_k=top_k,
-        top_p=top_p,
-        do_sample=True
-    )
-
-
-    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    return generated_text
+from transformers import pipeline
 
+generator = pipeline("text-generation", model="foryui/Llama-3.2-1B-Instruct-GRPO", device="cuda")
 
 def my_inference_function(text):
     prompt = f"Summary the context below\n\n{text}"
-    generated_text = generate_text(prompt, model, tokenizer)
+    output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+    output = output["generated_text"]
     
-    return generated_text[len(prompt):]
+    return output[len(prompt):]
 
 gradio_interface = gradio.Interface(
     fn=my_inference_function,
     inputs="text",
     outputs="text",
     examples=[
-    ["Jill"],
-    ["Sam"]
+    ["""Background
+One of the founding fathers of marine mammal bioacoustics, William Watkins, carried out pioneering work with William Schevill at the Woods Hole Oceanographic Institution for more than four decades, laying the groundwork for our field today. One of the lasting achievements of his career was the Watkins Marine Mammal Sound Database, a resource that contains approximately 2000 unique recordings of more than 60 species of marine mammals (Table 1). Recordings were made by Watkins and Schevill as well as many others, including G. C. Ray, D. Wartzok, D. and M. Caldwell, K. Norris, and T. Poulter. Most of these have been digitized, along with approximately 15,000 annotated digital sound clips.
+
+The Watkins database has enormous historical and scientific value. The recordings provide sounds professionally identified as produced by particular marine mammal species in defined geographic regions (Figure 1) during specific seasons, which can be used as reference datasets for marine mammal detections from the growing amounts of passive acoustic monitoring (PAM) data that are being collected worldwide. In addition, the archive contains recordings that span seven decades, from the 1940's to the 2000's, and includes the very first recordings of 51 species of marine mammals. These data provide a rich resource to efforts aimed at examining long-term changes in vocal production that may be related to changes in ambient noise levels, as well as serve as a voucher collection for many species. We have made this resource fully accessible online, as was Watkins' goal (see crediting information below). The final product enables investigators, educators, students, and the public worldwide to freely and easily access acoustic samples from identified species of marine mammals, and place these samples in a geographic and temporal context. The physical collection has been donated to the New Bedford Whaling Museum."""],
     ],
     title="REST API with Gradio and Huggingface Spaces",
     description="This is a demo of how to build an AI powered REST API with Gradio and Huggingface Spaces – for free! Based on [this article](https://www.tomsoderlund.com/ai/building-ai-powered-rest-api). See the **Use via API** link at the bottom of this page.",