Update app.py

app.py

@@ -1,10 +1,11 @@
-import gradio as gr
 import torch
+import gradio as gr
 from peft import PeftModel
 from transformers import AutoModelForCausalLM, AutoTokenizer
 import transformers
 
-adapters_name = "pyakhurel/mistral-7b-mj-finetuned"
+
+adapters_name = "1littlecoder/mistral-7b-mj-finetuned"
 model_name = "bn22/Mistral-7B-Instruct-v0.1-sharded"
 device = "cuda"
 
@@ -22,7 +23,6 @@ model = AutoModelForCausalLM.from_pretrained(
     quantization_config=bnb_config,
     device_map='auto'
 )
-
 model = PeftModel.from_pretrained(model, adapters_name)
 
 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -30,9 +30,22 @@ tokenizer.bos_token_id = 1
 
 stop_token_ids = [0]
 
+print(f"Successfully loaded the model {model_name} into memory")
 
+def remove_substring(original_string, substring_to_remove):
+    # Replace the substring with an empty string
+    result_string = original_string.replace(substring_to_remove, '')
+    return result_string
 
+def list_to_string(input_list, delimiter=" "):
+    """
+    Convert a list to a string, joining elements with the specified delimiter.
 
+    :param input_list: The list to convert to a string.
+    :param delimiter: The separator to use between elements (default is a space).
+    :return: A string composed of list elements separated by the delimiter.
+    """
+    return delimiter.join(map(str, input_list))
 
 def format_prompt(message, history):
   prompt = "<s>"
@@ -61,19 +74,17 @@ def generate(
 
     formatted_prompt = format_prompt(prompt, history)
 
-    
     encoded = tokenizer(formatted_prompt, return_tensors="pt", add_special_tokens=False)
     model_input = encoded
     model.to(device)
-    generated_ids = model.generate(**model_input, max_new_tokens=1048, do_sample=True)
-    stream = tokenizer.batch_decode(generated_ids)
-    
-    output = ""
+    generated_ids = model.generate(**model_input, max_new_tokens=200, do_sample=True)
+
 
-    for response in stream:
-        output += response.token.text
-        yield output
-    return output
+    list_output = tokenizer.batch_decode(generated_ids)
+    string_output = list_to_string(list_output)
+    possible_output = remove_substring(string_output,formatted_prompt)
+   
+    return possible_output
 
 
 additional_inputs=[
@@ -117,9 +128,9 @@ additional_inputs=[
 
 css = """
   #mkd {
-    height: 500px; 
-    overflow: auto; 
-    border: 1px solid #ccc; 
+    height: 500px;
+    overflow: auto;
+    border: 1px solid #ccc;
   }
 """
 
@@ -133,4 +144,5 @@ with gr.Blocks(css=css) as demo:
         examples=[["What is the secret to life?"], ["Write me a recipe for pancakes."]]
     )
 
-demo.queue(concurrency_count=75, max_size=100).launch(debug=True)
+demo.queue(concurrency_count=75, max_size=100).launch(debug=True)
+