Update app.py

app.py

@@ -4,6 +4,8 @@ import torch
 
 from transformers import AutoTokenizer, GenerationConfig
 from transformers import AutoModelForCausalLM
+from transformers import TextIteratorStreamer
+from threading import Thread
 import numpy as np
 
 temperature = 1.0
@@ -54,14 +56,26 @@ context = st.text_area('Contexto')
 max_length = st.number_input('Max generation length',value=52)
 
 
-if (model == None):
-    model = AutoModelForCausalLM.from_pretrained("lagy/carballo_finetuned")
-    tokenizer = AutoTokenizer.from_pretrained("lagy/carballo_finetuned")
-    model.eval()
+model = AutoModelForCausalLM.from_pretrained("lagy/carballo_finetuned")
+tokenizer = AutoTokenizer.from_pretrained("lagy/carballo_finetuned")
+model.eval()
+
+
 
 if st.button('Generate'):
-    #st.json(out)
     st.write("Generating...")
-    output = generate(model,tokenizer,instruction,context)
-    st.write(output)
+    thread = Thread(target=lambda: generate(model,instruction,context))
+    streamer = TextIteratorStreamer(tokenizer)
+    
+    thread.start()
+    print("Generating")
+    text = ""
+    for new_text in streamer:
+        text += new_text
+        st.write(text)
+    
+    #st.json(out)
+    #st.write("Generating...")
+    #output = generate(model,tokenizer,instruction,context)
+