| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 55.55555556 | |
| 10 60.00000000 | |
| 11 54.54545455 | |
| 12 58.33333333 | |
| 13 53.84615385 | |
| 14 50.00000000 | |
| 15 53.33333333 | |
| 16 56.25000000 | |
| 17 58.82352941 | |
| 18 55.55555556 | |
| 19 57.89473684 | |
| 20 60.00000000 | |
| 21 61.90476190 | |
| 22 59.09090909 | |
| 23 56.52173913 | |
| 24 58.33333333 | |
| 25 60.00000000 | |
| 26 57.69230769 | |
| 27 59.25925926 | |
| 28 60.71428571 | |
| 29 62.06896552 | |
| 30 63.33333333 | |
| 31 61.29032258 | |
| 32 62.50000000 | |
| 33 60.60606061 | |
| 34 61.76470588 | |
| 35 62.85714286 | |
| 36 61.11111111 | |
| 37 59.45945946 | |
| 38 60.52631579 | |
| 39 58.97435897 | |
| 40 57.50000000 | |
| 41 56.09756098 | |
| 42 54.76190476 | |
| 43 55.81395349 | |
| 44 54.54545455 | |
| 45 55.55555556 | |
| 46 54.34782609 | |
| 47 53.19148936 | |
| 48 54.16666667 | |
| 49 55.10204082 | |
| 50 54.00000000 | |
| 51 52.94117647 | |
| 52 53.84615385 | |
| 53 52.83018868 | |
| 54 51.85185185 | |
| 55 52.72727273 | |
| 56 53.57142857 | |
| 57 54.38596491 | |
| 58 53.44827586 | |
| 59 52.54237288 | |
| 60 51.66666667 | |
| 61 52.45901639 | |
| 62 51.61290323 | |
| 63 50.79365079 | |
| 64 50.00000000 | |
| 65 49.23076923 | |
| 66 48.48484848 | |
| 67 47.76119403 | |
| 68 47.05882353 | |
| 69 46.37681159 | |
| 70 47.14285714 | |
| 71 47.88732394 | |
| 72 47.22222222 | |
| 73 46.57534247 | |
| 74 47.29729730 | |
| 75 48.00000000 | |
| 76 48.68421053 | |
| 77 49.35064935 | |
| 78 50.00000000 | |
| 79 49.36708861 | |
| 80 50.00000000 | |
| 81 50.61728395 | |
| 82 50.00000000 | |
| 83 50.60240964 | |
| 84 51.19047619 | |
| 85 50.58823529 | |
| 86 50.00000000 | |
| 87 50.57471264 | |
| 88 51.13636364 | |
| 89 50.56179775 | |
| 90 50.00000000 | |
| 91 49.45054945 | |
| 92 48.91304348 | |
| 93 48.38709677 | |
| 94 48.93617021 | |
| 95 48.42105263 | |
| 96 48.95833333 | |
| 97 48.45360825 | |
| 98 47.95918367 | |
| 99 47.47474747 | |
| 100 47.00000000 | |
| 101 47.52475248 | |
| 102 47.05882353 | |
| 103 46.60194175 | |
| 104 47.11538462 | |
| 105 47.61904762 | |
| 106 48.11320755 | |
| 107 47.66355140 | |
| 108 48.14814815 | |
| 109 48.62385321 | |
| 110 49.09090909 | |
| 111 49.54954955 | |
| 112 49.10714286 | |
| 113 48.67256637 | |
| 114 49.12280702 | |
| 115 49.56521739 | |
| 116 50.00000000 | |
| 117 49.57264957 | |
| 118 49.15254237 | |
| 119 49.57983193 | |
| 120 49.16666667 | |
| 121 49.58677686 | |
| 122 49.18032787 | |
| 123 48.78048780 | |
| 124 48.38709677 | |
| 125 48.80000000 | |
| 126 48.41269841 | |
| 127 48.03149606 | |
| 128 47.65625000 | |
| 129 47.28682171 | |
| 130 46.92307692 | |
| 131 46.56488550 | |
| 132 46.96969697 | |
| 133 46.61654135 | |
| 134 46.26865672 | |
| 135 45.92592593 | |
| 136 45.58823529 | |
| 137 45.25547445 | |
| 138 44.92753623 | |
| 139 45.32374101 | |
| 140 45.71428571 | |
| 141 46.09929078 | |
| 142 46.47887324 | |
| 143 46.85314685 | |
| 144 47.22222222 | |
| 145 47.58620690 | |
| 146 47.94520548 | |
| 147 47.61904762 | |
| 148 47.97297297 | |
| 149 47.65100671 | |
| 150 47.33333333 | |
| 151 47.01986755 | |
| 152 47.36842105 | |
| 153 47.71241830 | |
| 154 47.40259740 | |
| 155 47.09677419 | |
| 156 46.79487179 | |
| 157 46.49681529 | |
| 158 46.83544304 | |
| 159 46.54088050 | |
| 160 46.87500000 | |
| 161 46.58385093 | |
| 162 46.29629630 | |
| 163 46.01226994 | |
| 164 45.73170732 | |
| 165 46.06060606 | |
| 166 45.78313253 | |
| 167 46.10778443 | |
| 168 45.83333333 | |
| 169 46.15384615 | |
| 170 45.88235294 | |
| 171 46.19883041 | |
| 172 45.93023256 | |
| 173 45.66473988 | |
| 174 45.40229885 | |
| 175 45.14285714 | |
| 176 44.88636364 | |
| 177 44.63276836 | |
| 178 44.38202247 | |
| 179 44.69273743 | |
| 180 44.44444444 | |
| 181 44.19889503 | |
| 182 44.50549451 | |
| 183 44.80874317 | |
| 184 44.56521739 | |
| 185 44.86486486 | |
| 186 45.16129032 | |
| 187 44.91978610 | |
| 188 45.21276596 | |
| 189 45.50264550 | |
| 190 45.26315789 | |
| 191 45.54973822 | |
| 192 45.31250000 | |
| 193 45.07772021 | |
| 194 45.36082474 | |
| 195 45.64102564 | |
| 196 45.91836735 | |
| 197 45.68527919 | |
| 198 45.95959596 | |
| 199 45.72864322 | |
| 200 45.50000000 | |
| 201 45.27363184 | |
| 202 45.04950495 | |
| 203 44.82758621 | |
| 204 44.60784314 | |
| 205 44.87804878 | |
| 206 44.66019417 | |
| 207 44.92753623 | |
| 208 44.71153846 | |
| 209 44.49760766 | |
| 210 44.28571429 | |
| 211 44.54976303 | |
| 212 44.33962264 | |
| 213 44.13145540 | |
| 214 43.92523364 | |
| 215 44.18604651 | |
| 216 44.44444444 | |
| 217 44.23963134 | |
| 218 44.49541284 | |
| 219 44.74885845 | |
| 220 45.00000000 | |
| 221 44.79638009 | |
| 222 44.59459459 | |
| 223 44.84304933 | |
| 224 45.08928571 | |
| 225 45.33333333 | |
| 226 45.13274336 | |
| 227 44.93392070 | |
| 228 45.17543860 | |
| 229 44.97816594 | |
| 230 44.78260870 | |
| 231 45.02164502 | |
| 232 45.25862069 | |
| 233 45.49356223 | |
| 234 45.72649573 | |
| 235 45.53191489 | |
| 236 45.33898305 | |
| 237 45.14767932 | |
| 238 44.95798319 | |
| 239 44.76987448 | |
| 240 45.00000000 | |
| 241 44.81327801 | |
| 242 45.04132231 | |
| 243 45.26748971 | |
| 244 45.08196721 | |
| 245 45.30612245 | |
| 246 45.12195122 | |
| 247 44.93927126 | |
| 248 45.16129032 | |
| 249 44.97991968 | |
| 250 44.80000000 | |
| 251 44.62151394 | |
| 252 44.44444444 | |
| 253 44.26877470 | |
| 254 44.09448819 | |
| 255 43.92156863 | |
| 256 44.14062500 | |
| 257 44.35797665 | |
| 258 44.18604651 | |
| 259 44.01544402 | |
| 260 44.23076923 | |
| 261 44.06130268 | |
| 262 43.89312977 | |
| 263 43.72623574 | |
| 264 43.56060606 | |
| 265 43.39622642 | |
| 266 43.60902256 | |
| 267 43.44569288 | |
| 268 43.28358209 | |
| 269 43.49442379 | |
| 270 43.70370370 | |
| 271 43.54243542 | |
| 272 43.75000000 | |
| 273 43.58974359 | |
| 274 43.43065693 | |
| 275 43.27272727 | |
| 276 43.47826087 | |
| 277 43.68231047 | |
| 278 43.52517986 | |
| 279 43.36917563 | |
| 280 43.21428571 | |
| 281 43.06049822 | |
| 282 42.90780142 | |
| 283 42.75618375 | |
| 284 42.60563380 | |
| 285 42.45614035 | |
| 286 42.30769231 | |
| 287 42.50871080 | |
| 288 42.36111111 | |
| 289 42.21453287 | |
| 290 42.41379310 | |
| 291 42.61168385 | |
| 292 42.80821918 | |
| 293 43.00341297 | |
| 294 42.85714286 | |
| 295 42.71186441 | |
| 296 42.56756757 | |
| 297 42.42424242 | |
| 298 42.61744966 | |
| 299 42.80936455 | |
| 300 43.00000000 | |
| 301 42.85714286 | |
| 302 43.04635762 | |
| 303 42.90429043 | |
| 304 43.09210526 | |
| 305 43.27868852 | |
| 306 43.13725490 | |
| 307 43.32247557 | |
| 308 43.18181818 | |
| 309 43.04207120 | |
| 310 42.90322581 | |
| 311 43.08681672 | |
| 312 43.26923077 | |
| 313 43.13099042 | |
| 314 43.31210191 | |
| 315 43.17460317 | |
| 316 43.03797468 | |
| 317 43.21766562 | |
| 318 43.08176101 | |
| 319 43.26018809 | |
| 320 43.12500000 | |
| 321 42.99065421 | |
| 322 43.16770186 | |
| 323 43.03405573 | |
| 324 42.90123457 | |
| 325 43.07692308 | |
| 326 42.94478528 | |
| 327 43.11926606 | |
| 328 42.98780488 | |
| 329 42.85714286 | |
| 330 42.72727273 | |
| 331 42.59818731 | |
| 332 42.46987952 | |
| 333 42.64264264 | |
| 334 42.51497006 | |
| 335 42.68656716 | |
| 336 42.85714286 | |
| 337 42.72997033 | |
| 338 42.60355030 | |
| 339 42.47787611 | |
| 340 42.64705882 | |
| 341 42.81524927 | |
| 342 42.98245614 | |
| 343 42.85714286 | |
| 344 42.73255814 | |
| 345 42.60869565 | |
| 346 42.77456647 | |
| 347 42.93948127 | |
| 348 42.81609195 | |
| 349 42.69340974 | |
| 350 42.85714286 | |
| 351 42.73504274 | |
| 352 42.61363636 | |
| 353 42.77620397 | |
| 354 42.65536723 | |
| 355 42.53521127 | |
| 356 42.69662921 | |
| 357 42.57703081 | |
| 358 42.45810056 | |
| 359 42.33983287 | |
| 360 42.50000000 | |
| 361 42.65927978 | |
| 362 42.81767956 | |
| 363 42.69972452 | |
| 364 42.58241758 | |
| 365 42.46575342 | |
| 366 42.62295082 | |
| 367 42.50681199 | |
| 368 42.39130435 | |
| 369 42.27642276 | |
| 370 42.43243243 | |
| 371 42.31805930 | |
| 372 42.20430108 | |
| 373 42.09115282 | |
| 374 41.97860963 | |
| 375 42.13333333 | |
| 376 42.02127660 | |
| 377 42.17506631 | |
| 378 42.32804233 | |
| 379 42.21635884 | |
| 380 42.10526316 | |
| 381 42.25721785 | |
| 382 42.14659686 | |
| 383 42.29765013 | |
| 384 42.18750000 | |
| 385 42.33766234 | |
| 386 42.22797927 | |
| 387 42.11886305 | |
| 388 42.01030928 | |
| 389 42.15938303 | |
| 390 42.30769231 | |
| 391 42.45524297 | |
| 392 42.60204082 | |
| 393 42.49363868 | |
| 394 42.38578680 | |
| 395 42.53164557 | |
| 396 42.67676768 | |
| 397 42.82115869 | |
| 398 42.71356784 | |
| 399 42.85714286 | |
| 400 42.75000000 | |
| 401 42.89276808 | |
| 402 43.03482587 | |
| 403 42.92803970 | |
| 404 43.06930693 | |
| 405 43.20987654 | |
| 406 43.10344828 | |
| 407 42.99754300 | |
| 408 42.89215686 | |
| 409 43.03178484 | |
| 410 42.92682927 | |
| 411 42.82238443 | |
| 412 42.71844660 | |
| 413 42.61501211 | |
| 414 42.51207729 | |
| 415 42.40963855 | |
| 416 42.54807692 | |
| 417 42.44604317 | |
| 418 42.34449761 | |
| 419 42.48210024 | |
| 420 42.38095238 | |
| 421 42.51781473 | |
| 422 42.41706161 | |
| 423 42.31678487 | |
| 424 42.45283019 | |
| 425 42.58823529 | |
| 426 42.72300469 | |
| 427 42.85714286 | |
| 428 42.75700935 | |
| 429 42.89044289 | |
| 430 42.79069767 | |
| 431 42.92343387 | |
| 432 42.82407407 | |
| 433 42.72517321 | |
| 434 42.62672811 | |
| 435 42.52873563 | |
| 436 42.43119266 | |
| 437 42.33409611 | |
| 438 42.46575342 | |
| 439 42.59681093 | |
| 440 42.50000000 | |
| 441 42.40362812 | |
| 442 42.30769231 | |
| 443 42.21218962 | |
| 444 42.11711712 | |
| 445 42.02247191 | |
| 446 41.92825112 | |
| 447 41.83445190 | |
| 448 41.74107143 | |
| 449 41.64810690 | |
| 450 41.77777778 | |
| 451 41.90687361 | |
| 452 41.81415929 | |
| 453 41.94260486 | |
| 454 41.85022026 | |
| 455 41.75824176 | |
| 456 41.88596491 | |
| 457 42.01312910 | |
| 458 41.92139738 | |
| 459 41.83006536 | |
| 460 41.95652174 | |
| 461 41.86550976 | |
| 462 41.77489177 | |
| 463 41.90064795 | |
| 464 42.02586207 | |
| 465 41.93548387 | |
| 466 41.84549356 | |
| 467 41.97002141 | |
| 468 42.09401709 | |
| 469 42.21748401 | |
| 470 42.12765957 | |
| 471 42.03821656 | |
| 472 41.94915254 | |
| 473 42.07188161 | |
| 474 41.98312236 | |
| 475 41.89473684 | |
| 476 42.01680672 | |
| 477 41.92872117 | |
| 478 42.05020921 | |
| 479 41.96242171 | |
| 480 42.08333333 | |
| 481 41.99584200 | |
| 482 41.90871369 | |
| 483 41.82194617 | |
| 484 41.73553719 | |
| 485 41.64948454 | |
| 486 41.76954733 | |
| 487 41.88911704 | |
| 488 41.80327869 | |
| 489 41.71779141 | |
| 490 41.83673469 | |
| 491 41.75152749 | |
| 492 41.66666667 | |
| 493 41.58215010 | |
| 494 41.70040486 | |
| 495 41.81818182 | |
| 496 41.93548387 | |
| 497 41.85110664 | |
| 498 41.76706827 | |
| 499 41.68336673 | |
| 500 41.80000000 | |
| 501 41.71656687 | |
| 502 41.83266932 | |
| 503 41.74950298 | |
| 504 41.66666667 | |
| 505 41.58415842 | |
| 506 41.69960474 | |
| 507 41.61735700 | |
| 508 41.73228346 | |
| 509 41.65029470 | |
| 510 41.56862745 | |
| 511 41.48727984 | |
| 512 41.40625000 | |
| 513 41.32553606 | |
| 514 41.24513619 | |
| 515 41.16504854 | |
| 516 41.27906977 | |
| 517 41.39264990 | |
| 518 41.50579151 | |
| 519 41.42581888 | |
| 520 41.34615385 | |
| 521 41.26679463 | |
| 522 41.37931034 | |
| 523 41.30019120 | |
| 524 41.41221374 | |
| 525 41.33333333 | |
| 526 41.25475285 | |
| 527 41.17647059 | |
| 528 41.09848485 | |
| 529 41.02079395 | |
| 530 41.13207547 | |
| 531 41.05461394 | |
| 532 40.97744361 | |
| 533 40.90056285 | |
| 534 41.01123596 | |
| 535 41.12149533 | |
| 536 41.04477612 | |
| 537 41.15456238 | |
| 538 41.26394052 | |
| 539 41.18738404 | |
| 540 41.11111111 | |
| 541 41.03512015 | |
| 542 41.14391144 | |
| 543 41.06813996 | |
| 544 41.17647059 | |
| 545 41.10091743 | |
| 546 41.02564103 | |
| 547 40.95063985 | |
| 548 41.05839416 | |
| 549 40.98360656 | |
| 550 40.90909091 | |
| 551 41.01633394 | |
| 552 40.94202899 | |
| 553 40.86799277 | |
| 554 40.97472924 | |
| 555 40.90090090 | |
| 556 41.00719424 | |
| 557 41.11310592 | |
| 558 41.21863799 | |
| 559 41.14490161 | |
| 560 41.07142857 | |
| 561 40.99821747 | |
| 562 41.10320285 | |
| 563 41.20781528 | |
| 564 41.13475177 | |
| 565 41.23893805 | |
| 566 41.34275618 | |
| 567 41.26984127 | |
| 568 41.19718310 | |
| 569 41.12478032 | |
| 570 41.05263158 | |
| 571 40.98073555 | |
| 572 41.08391608 | |
| 573 41.01221640 | |
| 574 40.94076655 | |
| 575 40.86956522 | |
| 576 40.97222222 | |
| 577 41.07452340 | |
| 578 41.17647059 | |
| 579 41.10535406 | |
| 580 41.20689655 | |
| 581 41.30808950 | |
| 582 41.23711340 | |
| 583 41.33790738 | |
| 584 41.43835616 | |
| 585 41.36752137 | |
| 586 41.46757679 | |
| 587 41.39693356 | |
| 588 41.32653061 | |
| 589 41.25636672 | |
| 590 41.18644068 | |
| 591 41.11675127 | |
| 592 41.04729730 | |
| 593 41.14671164 | |
| 594 41.24579125 | |
| 595 41.17647059 | |
| 596 41.27516779 | |
| 597 41.20603015 | |
| 598 41.13712375 | |
| 599 41.23539232 | |
| 600 41.16666667 | |
| 601 41.09816972 | |
| 602 41.02990033 | |
| 603 41.12769486 | |
| 604 41.05960265 | |
| 605 40.99173554 | |
| 606 40.92409241 | |
| 607 41.02141680 | |
| 608 40.95394737 | |
| 609 40.88669951 | |
| 610 40.81967213 | |
| 611 40.91653028 | |
| 612 40.84967320 | |
| 613 40.94616639 | |
| 614 41.04234528 | |
| 615 40.97560976 | |
| 616 41.07142857 | |
| 617 41.16693679 | |
| 618 41.10032362 | |
| 619 41.03392569 | |
| 620 40.96774194 | |
| 621 40.90177134 | |
| 622 40.99678457 | |
| 623 40.93097913 | |
| 624 40.86538462 | |
| 625 40.80000000 | |
| 626 40.89456869 | |
| 627 40.82934609 | |
| 628 40.76433121 | |
| 629 40.85850556 | |
| 630 40.95238095 | |
| 631 40.88748019 | |
| 632 40.98101266 | |
| 633 40.91627172 | |
| 634 40.85173502 | |
| 635 40.78740157 | |
| 636 40.72327044 | |
| 637 40.65934066 | |
| 638 40.59561129 | |
| 639 40.68857590 | |
| 640 40.78125000 | |
| 641 40.71762871 | |
| 642 40.65420561 | |
| 643 40.59097978 | |
| 644 40.68322981 | |
| 645 40.62015504 | |
| 646 40.55727554 | |
| 647 40.49459042 | |
| 648 40.43209877 | |
| 649 40.36979969 | |
| 650 40.30769231 | |
| 651 40.24577573 | |
| 652 40.18404908 | |
| 653 40.12251149 | |
| 654 40.21406728 | |
| 655 40.15267176 | |
| 656 40.24390244 | |
| 657 40.18264840 | |
| 658 40.27355623 | |
| 659 40.21244310 | |
| 660 40.15151515 | |
| 661 40.09077156 | |
| 662 40.03021148 | |
| 663 39.96983409 | |
| 664 39.90963855 | |
| 665 40.00000000 | |
| 666 39.93993994 | |
| 667 40.02998501 | |
| 668 40.11976048 | |
| 669 40.20926756 | |
| 670 40.29850746 | |
| 671 40.23845007 | |
| 672 40.17857143 | |
| 673 40.11887073 | |
| 674 40.05934718 | |
| 675 40.00000000 | |
| 676 39.94082840 | |
| 677 39.88183161 | |
| 678 39.97050147 | |
| 679 39.91163476 | |
| 680 40.00000000 | |
| 681 40.08810573 | |
| 682 40.02932551 | |
| 683 39.97071742 | |
| 684 40.05847953 | |
| 685 40.00000000 | |
| 686 40.08746356 | |
| 687 40.17467249 | |
| 688 40.11627907 | |
| 689 40.05805515 | |
| 690 40.00000000 | |
| 691 39.94211288 | |
| 692 39.88439306 | |
| 693 39.82683983 | |
| 694 39.91354467 | |
| 695 39.85611511 | |
| 696 39.79885057 | |
| 697 39.74175036 | |
| 698 39.82808023 | |
| 699 39.91416309 | |
| 700 39.85714286 | |
| 701 39.94293866 | |
| 702 40.02849003 | |
| 703 40.11379801 | |
| 704 40.05681818 | |
| 705 40.00000000 | |
| 706 39.94334278 | |
| 707 39.88684583 | |
| 708 39.83050847 | |
| 709 39.77433004 | |
| 710 39.71830986 | |
| 711 39.80309423 | |
| 712 39.88764045 | |
| 713 39.97194951 | |
| 714 40.05602241 | |
| 715 40.00000000 | |
| 716 39.94413408 | |
| 717 39.88842399 | |
| 718 39.83286908 | |
| 719 39.77746871 | |
| 720 39.72222222 | |
| 721 39.66712899 | |
| 722 39.61218837 | |
| 723 39.69571231 | |
| 724 39.64088398 | |
| 725 39.72413793 | |
| 726 39.66942149 | |
| 727 39.61485557 | |
| 728 39.69780220 | |
| 729 39.64334705 | |
| 730 39.58904110 | |
| 731 39.53488372 | |
| 732 39.48087432 | |
| 733 39.56343793 | |
| 734 39.64577657 | |
| 735 39.72789116 | |
| 736 39.67391304 | |
| 737 39.75576662 | |
| 738 39.83739837 | |
| 739 39.91880920 | |
| 740 39.86486486 | |
| 741 39.81106613 | |
| 742 39.89218329 | |
| 743 39.97308210 | |
| 744 40.05376344 | |
| 745 40.00000000 | |
| 746 39.94638070 | |
| 747 40.02677376 | |
| 748 39.97326203 | |
| 749 39.91989319 | |
| 750 40.00000000 | |
| Final result: 40.0000 +/- 1.7900 | |
| Random chance: 25.0000 +/- 1.5822 | |