Update app.py
Browse files
app.py
CHANGED
|
@@ -43,7 +43,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
|
|
| 43 |
|
| 44 |
|
| 45 |
READER_MODEL = "HuggingFaceH4/zephyr-7b-beta"
|
| 46 |
-
|
| 47 |
model = AutoModelForCausalLM.from_pretrained(READER_MODEL,quantization_config=bnb_config)
|
| 48 |
tokenizer = AutoTokenizer.from_pretrained(READER_MODEL)
|
| 49 |
|
|
|
|
| 43 |
|
| 44 |
|
| 45 |
READER_MODEL = "HuggingFaceH4/zephyr-7b-beta"
|
| 46 |
+
bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16)
|
| 47 |
model = AutoModelForCausalLM.from_pretrained(READER_MODEL,quantization_config=bnb_config)
|
| 48 |
tokenizer = AutoTokenizer.from_pretrained(READER_MODEL)
|
| 49 |
|