pankajroark
/

llama-fp16-engine

pankajroark commited on Oct 25, 2023

Commit

bd472e5

1 Parent(s): a3806a1

update no-quant engine

Files changed (3) hide show

7b-no-quant-tp1/config.json CHANGED Viewed

@@ -32,14 +32,14 @@
     "layernorm_quantization_plugin": false,
     "lookup_plugin": false,
     "nccl_plugin": false,
-    "paged_kv_cache": false,
     "quantize_per_token_plugin": false,
     "quantize_tensor_plugin": false,
     "remove_input_padding": true,
     "rmsnorm_plugin": false,
     "rmsnorm_quantization_plugin": false,
     "smooth_quant_gemm_plugin": false,
-    "tokens_per_block": 0,
     "use_custom_all_reduce": false,
     "weight_only_groupwise_quant_matmul_plugin": false,
     "weight_only_quant_matmul_plugin": false

     "layernorm_quantization_plugin": false,
     "lookup_plugin": false,
     "nccl_plugin": false,
+    "paged_kv_cache": true,
     "quantize_per_token_plugin": false,
     "quantize_tensor_plugin": false,
     "remove_input_padding": true,
     "rmsnorm_plugin": false,
     "rmsnorm_quantization_plugin": false,
     "smooth_quant_gemm_plugin": false,
+    "tokens_per_block": 64,
     "use_custom_all_reduce": false,
     "weight_only_groupwise_quant_matmul_plugin": false,
     "weight_only_quant_matmul_plugin": false

7b-no-quant-tp1/llama_float16_tp1_rank0.engine CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a7558d8628cbe8ce2e09377743fe47d33235d33b201fb23ebffd1b6d6ca1905
-size 13480868308

 version https://git-lfs.github.com/spec/v1
+oid sha256:44082abbf34483808729d46983500858093fddc8e4eb31c282471f93dba3fd33
+size 13480840252

7b-no-quant-tp1/model.cache CHANGED Viewed

Binary files a/7b-no-quant-tp1/model.cache and b/7b-no-quant-tp1/model.cache differ