Update app.py

app.py

@@ -73,21 +73,20 @@ def loadModel():
     if model is None:
         from llama_rope_scaled_monkey_patch import replace_llama_rope_with_scaled_rope
         replace_llama_rope_with_scaled_rope()
-        t_model = transformers.AutoModelForCausalLM.from_pretrained(
+        model = transformers.AutoModelForCausalLM.from_pretrained(
                     base_model,
                     torch_dtype=torch.float16,
                     cache_dir=cache_dir,
                     device_map="auto",
                 )
         model = PeftModel.from_pretrained(
-                    t_model,
+                    model,
                     lora_weights,
                     device_map="auto",
                     cache_dir='',
                     torch_dtype=torch.float16,
                     is_trainable=False,
                 )
-        model.eval()
         tokenizer =  AutoTokenizer.from_pretrained(base_model,use_fast=False,cache_dir=cache_dir)
         tokenizer.pad_token = tokenizer.unk_token
         model = model.to("cuda")