Update app.py

app.py

@@ -102,26 +102,26 @@ def respond(
     
     messages= json_obj
 
-    # input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to('cuda') # .to(accelerator.device)
-    # input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda')
-    # print(f"Converted input_ids dtype: {input_ids.dtype}")
-    # input_str= str(input_ids2)
-    # print('input str = ', input_str)
+    input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to('cuda') # .to(accelerator.device)
+    input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda')
+    print(f"Converted input_ids dtype: {input_ids.dtype}")
+    input_str= str(input_ids2)
+    print('input str = ', input_str)
 
     
-    # gen_tokens = model.generate(
-    # input_ids, 
-    # max_new_tokens=max_tokens, 
-    # # do_sample=True, 
-    # temperature=temperature,
-    # )
-
-    # gen_text = tokenizer.decode(gen_tokens[0])
-    # print(gen_text)
-    # gen_text= gen_text.replace(input_str,'')
-    # gen_text= gen_text.replace('<|END_OF_TURN_TOKEN|>','')
+    gen_tokens = model.generate(
+    input_ids, 
+    max_new_tokens=max_tokens, 
+    # do_sample=True, 
+    temperature=temperature,
+    )
+
+    gen_text = tokenizer.decode(gen_tokens[0])
+    print(gen_text)
+    gen_text= gen_text.replace(input_str,'')
+    gen_text= gen_text.replace('<|END_OF_TURN_TOKEN|>','')
     
-    # yield gen_text
+    yield gen_text
    
   
 #     messages = [