Update app.py

app.py

@@ -18,14 +18,15 @@ from peft import AutoPeftModelForCausalLM
 from transformers import AutoTokenizer
 import torch
 
-attn_implementation = None
-if USE_FLASH_ATTENTION:
-  attn_implementation="flash_attention_2"
+# attn_implementation = None
+# USE_FLASH_ATTENTION = False
+# if USE_FLASH_ATTENTION:
+#   attn_implementation="flash_attention_2"
 
 
 model_id = "r1208/c4ai-command-r-v01-4bit_32r"
 
-model = AutoPeftModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, attn_implementation=attn_implementation,)
+model = AutoPeftModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 tokenizer_with_prefix_space = AutoTokenizer.from_pretrained(model_id, add_prefix_space=True)