Upload app.py

app.py

@@ -1,22 +1,21 @@
 # -*- coding: utf-8 -*-
-"""gradio.ipynb
+"""Untitled14.ipynb
 
 Automatically generated by Colab.
 
 Original file is located at
-    https://colab.research.google.com/drive/1goHcmXF0Gc4_X9PN-zecV77j9KeI6Dmn
+    https://colab.research.google.com/drive/1qm9JqCY6CGVTqzvw3GEAI8BsI_-w1rwP
 """
 
 # !pip install -q -U gradio
 # !pip install -q -U torch transformers accelerate einops
 # !pip install -q peft
 
-
-
-
+examples = [["Which organelle carries out the synthesis and packaging of digestive enzymes?"],
+            ["What is the change in speed of a moving object per unit time?"] ,
+            ["What is the formula of carbon tetrafluoride?"]]
 
 import gradio as gr
-
 import torch
 from transformers import (
     AutoTokenizer,
@@ -24,14 +23,10 @@ from transformers import (
     TextIteratorStreamer,
     pipeline,
 )
-
-# The huggingface model id for Microsoft's phi-2 model
-# Download and load model and tokenizer
-tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)
-
-#Download safetensor of adapter of fine-tune Phi-2 model
 from peft import PeftModel, PeftConfig
 
+# Load the tokenizer and models
+tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)
 config = PeftConfig.from_pretrained("mudogruer/phi-2-SciQ")
 base_model = AutoModelForCausalLM.from_pretrained("microsoft/phi-2")
 model = PeftModel.from_pretrained(base_model, "mudogruer/phi-2-SciQ")
@@ -46,26 +41,28 @@ phi2 = pipeline(
     device_map="cpu",
 )
 
-examples = [["Which organelle carries out the synthesis and packaging of digestive enzymes?"],
-            ["What is the change in speed of a moving object per unit time?"] ,
-            ["What is the formula of carbon tetrafluoride?"]]
-
 def generate(message, max_new_tokens):
     instruction = "You are a helpful assistant to 'User'. You do not respond as 'User' or pretend to be 'User'. You only respond once as 'Assistant'."
     final_prompt = f"Instruction: {instruction}\nUser: {message}\nOutput:"
 
+    # Calculate the total length allowed for the model and adjust max_new_tokens
+    input_length = len(tokenizer.encode(final_prompt))
+    total_max_length = 512  # Adjust based on your model's max length capability
+    if input_length + max_new_tokens > total_max_length:
+        max_new_tokens = total_max_length - input_length  # Adjust to not exceed total max length
+
     # Generate text synchronously
     response = phi2(final_prompt, max_new_tokens=max_new_tokens)
     generated_text = response[0]['generated_text']
 
-    # Process to extract the last assistant's response
-    # Assuming the last line after 'Output:' is the response
+    # Extract the response
     last_response = generated_text.split('Output:')[-1].strip()
     return last_response
 
-# Update the Gradio interface setup
+# Gradio interface setup
 with gr.Blocks() as demo:
-    gr.Markdown("""### Phi-2 Scientific Question Chatbot(Fine-tuned from SciQ dataset)""")
-    tokens_slider = gr.Slider(8, 128, value=128, label="Maximum new tokens")
-    chatbot = gr.Interface(fn=generate, inputs=["text", tokens_slider], outputs="text", examples = examples)
-    demo.launch(share=True)
+    gr.Markdown("""### Phi-2 Scientific Question Chatbot (Fine-tuned from SciQ dataset)""")
+    tokens_slider = gr.Slider(8, 128, value=21, label="Maximum new tokens")
+    chatbot = gr.Interface(fn=generate, inputs=["text", tokens_slider], outputs="text", examples=examples)
+    demo.launch(share=True)
+