updated

app.py

@@ -9,12 +9,12 @@ from transformers import pipeline
 
 token = os.getenv("HUGGINGFACE_API_TOKEN")
 pipe = pipeline("text-generation", "meta-llama/Meta-Llama-3-8B-Instruct", torch_dtype=torch.bfloat16, device_map="auto", token=token)
-response = pipe(chat, max_new_tokens=512)
 
 pipe.to('cuda')
 
 @spaces.GPU
 def generate(prompt):
+    response = pipe(chat, max_new_tokens=512)
     r = response[0]['generated_text'][-1]['content']
     return r