Update app.py
Browse files
app.py
CHANGED
|
@@ -62,7 +62,7 @@ model = AutoModelForCausalLM.from_pretrained(model_id, token= token,
|
|
| 62 |
torch_dtype=torch.float16,
|
| 63 |
# load_in_8bit=True,
|
| 64 |
# # # torch_dtype=torch.fl,
|
| 65 |
-
|
| 66 |
# low_cpu_mem_usage=True,
|
| 67 |
# device_map='cuda',
|
| 68 |
# device_map=accelerator.device_map,
|
|
|
|
| 62 |
torch_dtype=torch.float16,
|
| 63 |
# load_in_8bit=True,
|
| 64 |
# # # torch_dtype=torch.fl,
|
| 65 |
+
attn_implementation="flash_attention_2",
|
| 66 |
# low_cpu_mem_usage=True,
|
| 67 |
# device_map='cuda',
|
| 68 |
# device_map=accelerator.device_map,
|