pyakhurel commited on
Commit
e5475e4
·
1 Parent(s): 6721da1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -10,7 +10,7 @@ model_name = "bn22/Mistral-7B-Instruct-v0.1-sharded"
10
  device = "cuda"
11
 
12
  bnb_config = transformers.BitsAndBytesConfig(
13
- load_in_8bit=True
14
  bnb_4bit_use_double_quant=True,
15
  bnb_4bit_quant_type="nf4",
16
  bnb_4bit_compute_dtype=torch.bfloat16
@@ -18,7 +18,7 @@ bnb_config = transformers.BitsAndBytesConfig(
18
 
19
  model = AutoModelForCausalLM.from_pretrained(
20
  model_name,
21
- load_in_8bit=True
22
  torch_dtype=torch.bfloat16,
23
  quantization_config=bnb_config,
24
  device_map='auto'
 
10
  device = "cuda"
11
 
12
  bnb_config = transformers.BitsAndBytesConfig(
13
+ load_in_8bit=True,
14
  bnb_4bit_use_double_quant=True,
15
  bnb_4bit_quant_type="nf4",
16
  bnb_4bit_compute_dtype=torch.bfloat16
 
18
 
19
  model = AutoModelForCausalLM.from_pretrained(
20
  model_name,
21
+ load_in_8bit=True,
22
  torch_dtype=torch.bfloat16,
23
  quantization_config=bnb_config,
24
  device_map='auto'