Update app.py

app.py

@@ -1,35 +1,36 @@
 import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from optimum.intel import OVAutoModelForCasualLM
 
-from happytransformer import HappyGeneration
-
-happy_gen = HappyGeneration("GPTNEO", "DarwinAnim8or/Pythia-Greentext-1.4b")
-
-from happytransformer import GENSettings
+model_name = "DarwinAnim8or/Pythia-Greentext-1.4b"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = OVAutoModelForCausalLM.from_pretrained(model_name, export=True)
 
 def generate(text, length=100, penalty=3, temperature=0.8, topk=40):
-    args_top_k = GENSettings(no_repeat_ngram_size=penalty, do_sample=True, top_k=topk, temperature=temperature, max_length=length, early_stopping=True)
-    
-    inputText = "Write a greentext from 4chan.org. The story should be like a bullet-point list using > as the start of each line. Most greentexts are humorous or absurd in nature. Most greentexts have a twist near the end.\n" 
+    input_text = "Write a greentext from 4chan.org. The story should be like a bullet-point list using > as the start of each line. Most greentexts are humorous or absurd in nature. Most greentexts have a twist near the end.\n"
 
-    if not text[0] == ">":
-        inputText += ">" + text + "\n>"
+    if not text.startswith(">"):
+        input_text += ">" + text + "\n>"
     else:
-        inputText += text + "\n>"
-
-    print(inputText)
-    
-    result = happy_gen.generate_text(inputText, args=args_top_k)
-    generated_text = result.text #returns generated text only
+        input_text += text + "\n>"
 
-    #replace \n with actual newlines:
-    generated_text = generated_text.replace('\\n', '\n')
+    input_ids = tokenizer.encode(input_text, return_tensors="pt")
+    input_ids = input_ids[:, :-1]  # remove the last token, which is ">"
 
-    #clean up formatting:
-    generated_text = generated_text.replace('>', '\n>')
-    generated_text = generated_text.replace('\\"', "\"")
+    length = length + input_ids.size(1)  # adjust total length
 
-    generated_text = ">" + text + "\n>" + generated_text #include our prompt in our response (partially)
+    output = model.generate(
+        input_ids,
+        max_length=length,
+        temperature=temperature,
+        top_k=topk,
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id,
+        no_repeat_ngram_size=penalty,
+        early_stopping=True,
+    )
 
+    generated_text = tokenizer.decode(output[:, input_ids.size(1):][0], skip_special_tokens=True)
     return generated_text
 
 examples = [