Update app.py

app.py

@@ -6,7 +6,7 @@ import faiss
 from langchain_huggingface import HuggingFaceEmbeddings
 import numpy as np
 from pypdf import PdfReader
-
+from transformers import AutoTokenizer
 
 title = "Mistral-7B-Instruct-GGUF Run On CPU-Basic Free Hardware"
 
@@ -29,6 +29,10 @@ hf_hub_download(repo_id="TheBloke/Mistral-7B-Instruct-v0.1-GGUF", filename=model
 print("Start the model init process")
 model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
 
+model_name = "HuggingFaceH4/zephyr-7b-beta"
+#model_name = "gpt2"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
 
 
 # creating a pdf reader object
@@ -97,13 +101,13 @@ def respond(message, chat_history):
 
         context.append({'role':'user', 'content':f"{prompt}"})
 
-        #tokenized_chat = tokenizer.apply_chat_template(context, tokenize=True, add_generation_prompt=True, return_tensors="pt")
+        tokenized_chat = tokenizer.apply_chat_template(context, tokenize=True, add_generation_prompt=True, return_tensors="pt")
 
-        #outputs = model.generate(tokenized_chat, max_new_tokens=1000, temperature = 0.0)
+        outputs = model.generate(tokenized_chat, max_new_tokens=max_new_tokens, temperature = 0.0)
 
-        #bot_message = tokenizer.decode(outputs[0]).split("<|assistant|>")[-1].replace("</s>","")
+        bot_message = tokenizer.decode(outputs[0]).split("<|assistant|>")[-1].replace("</s>","")
 
-        bot_message = model.generate(prompt=prompt, temp=0.5, top_k = 40, top_p = 1, max_tokens = max_new_tokens, streaming=False)
+        #bot_message = model.generate(prompt=prompt, temp=0.5, top_k = 40, top_p = 1, max_tokens = max_new_tokens, streaming=False)
 
         context.append({'role':'assistant', 'content':f"{bot_message}"})