Update app.py

app.py

@@ -9,8 +9,6 @@ from peft import PeftModel
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 #client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-from llama_rope_scaled_monkey_patch import replace_llama_rope_with_scaled_rope
-replace_llama_rope_with_scaled_rope()
 base_model = "Neko-Institute-of-Science/LLaMA-65B-HF"
 lora_weights = "./"
 #lora_weights = LoraConfig(
@@ -32,6 +30,8 @@ lora_weights = "./"
 #)
 
 cache_dir = "/data"
+from llama_rope_scaled_monkey_patch import replace_llama_rope_with_scaled_rope
+replace_llama_rope_with_scaled_rope()
 model = transformers.AutoModelForCausalLM.from_pretrained(
             base_model,
             torch_dtype=torch.float16,
@@ -48,7 +48,6 @@ model = PeftModel.from_pretrained(
         )
 tokenizer =  AutoTokenizer.from_pretrained(base_model,use_fast=False,cache_dir=cache_dir)
 tokenizer.pad_token = tokenizer.unk_token
-model.eval()
 PROMPT_DICT = {
     "prompt_input": (
         "Below is an instruction that describes a task, paired with further context. "
@@ -148,4 +147,5 @@ demo = gr.ChatInterface(
 
 
 if __name__ == "__main__":
+    model.eval()
     demo.launch()