LoftQ
/

Meta-Llama-3-8B-4bit-64rank

Text Generation

text-generation-inference

Inference Endpoints

4-bit precision

Model card Files Files and versions Community

LoftQ commited on Apr 18, 2024

Commit

68abef6

•

1 Parent(s): 5db0e98

Update config.json

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/mnt/t-qingzhang/yxli2123_model_zoo/Meta-Llama-3-8B-4bit-64rank-5iter",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -20,7 +20,7 @@
   "quantization_config": {
     "_load_in_4bit": true,
     "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "float32",
     "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": false,

 {
+  "_name_or_path": "LoftQ/Meta-Llama-3-8B-4bit-64rank-5iter",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "quantization_config": {
     "_load_in_4bit": true,
     "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
     "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": false,