Update app.py

app.py

@@ -17,17 +17,22 @@ model = PeftModel.from_pretrained(model, peft_model_id)
 
 
 
-def make_inference(conversations):
-    context = ""
-    for conversation in conversations:
-        context += f"{conversation}\n\n"
-    prompt = f"### Conversation:\n{context}"
-    batch = tokenizer(prompt, return_tensors="pt")
-    with torch.cuda.amp.autocast():
-        output_tokens = model.generate(**batch, max_new_tokens=50)
-    response = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
-    updated_conversation = f"{context}\n{response}"
-    return updated_conversation
+def make_inference(conversation):
+    conversation_history = conversation
+    response = ""
+    while True:
+        batch = tokenizer(
+            f"### Problem:\n{conversation_history}\n{response}",
+            return_tensors="pt",
+        )
+        with torch.cuda.amp.autocast():
+            output_tokens = model.generate(**batch, max_new_tokens=50)
+        new_response = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+        if new_response.strip() == "":
+            break
+        response = f"\n{new_response}"
+        conversation_history += response
+    return conversation_history
 
 
 if __name__ == "__main__":
@@ -52,5 +57,5 @@ gr.Interface(
     ],
     gr.outputs.Textbox(label="Updated Conversation"),
     title="tax-convos-demo",
-    description="Trying to create a crude chat bot for tax services.",
+    description="Ask any tax-related questions you may have.",
 ).launch()