lvkaokao commited on
Commit
1ef1c7e
·
verified ·
1 Parent(s): ee24cb6

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +16 -16
README.md CHANGED
@@ -47,22 +47,22 @@ cd auto-round/examples/language-modeling
47
  python3 eval_042/evluation.py --model_name "Intel/Qwen2-7B-int4-inc" --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k,cmmlu,ceval-valid
48
  ```
49
 
50
- | Metric | BF16 | INT4 | INT4-lmhead |
51
- | -------------- | ------ | ------ | ----------- |
52
- | Avg | 0.6655 | 0.6624 | 0.6597 |
53
- | mmlu | 0.6706 | 0.6650 | 0.6584 |
54
- | cmmlu | 0.8310 | 0.8151 | 0.8088 |
55
- | ceval-valid | 0.8232 | 0.8053 | 0.7979 |
56
- | lambada_openai | 0.7182 | 0.7254 | 0.7138 |
57
- | hellaswag | 0.5823 | 0.5758 | 0.5768 |
58
- | winogrande | 0.7222 | 0.7174 | 0.7277 |
59
- | piqa | 0.7911 | 0.7905 | 0.7884 |
60
- | truthfulqa_mc1 | 0.3647 | 0.3599 | 0.3525 |
61
- | openbookqa | 0.3520 | 0.3420 | 0.3460 |
62
- | boolq | 0.8183 | 0.8168 | 0.8211 |
63
- | arc_easy | 0.7660 | 0.7643 | 0.7630 |
64
- | arc_challenge | 0.4505 | 0.4625 | 0.4582 |
65
- | gsm8k 5 shots | 0.7619 | 0.7710 | 0.7635 |
66
 
67
 
68
 
 
47
  python3 eval_042/evluation.py --model_name "Intel/Qwen2-7B-int4-inc" --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k,cmmlu,ceval-valid
48
  ```
49
 
50
+ | Metric | BF16 | INT4 |
51
+ | -------------- | ------ | ------ |
52
+ | Avg | 0.6655 | 0.6624 |
53
+ | mmlu | 0.6706 | 0.6650 |
54
+ | cmmlu | 0.8310 | 0.8151 |
55
+ | ceval-valid | 0.8232 | 0.8053 |
56
+ | lambada_openai | 0.7182 | 0.7254 |
57
+ | hellaswag | 0.5823 | 0.5758 |
58
+ | winogrande | 0.7222 | 0.7174 |
59
+ | piqa | 0.7911 | 0.7905 |
60
+ | truthfulqa_mc1 | 0.3647 | 0.3599 |
61
+ | openbookqa | 0.3520 | 0.3420 |
62
+ | boolq | 0.8183 | 0.8168 |
63
+ | arc_easy | 0.7660 | 0.7643 |
64
+ | arc_challenge | 0.4505 | 0.4625 |
65
+ | gsm8k 5 shots | 0.7619 | 0.7710 |
66
 
67
 
68