common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 76.92307692 14 78.57142857 15 80.00000000 16 81.25000000 17 82.35294118 18 77.77777778 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 65.51724138 30 66.66666667 31 64.51612903 32 65.62500000 33 63.63636364 34 61.76470588 35 62.85714286 36 61.11111111 37 62.16216216 38 60.52631579 39 61.53846154 40 62.50000000 41 63.41463415 42 61.90476190 43 60.46511628 44 59.09090909 45 60.00000000 46 60.86956522 47 61.70212766 48 62.50000000 49 63.26530612 50 62.00000000 51 60.78431373 52 61.53846154 53 60.37735849 54 59.25925926 55 58.18181818 56 58.92857143 57 59.64912281 58 60.34482759 59 61.01694915 60 61.66666667 61 62.29508197 62 61.29032258 63 61.90476190 64 62.50000000 65 61.53846154 66 62.12121212 67 62.68656716 68 63.23529412 69 63.76811594 70 62.85714286 71 61.97183099 72 61.11111111 73 61.64383562 74 62.16216216 75 62.66666667 76 61.84210526 77 62.33766234 78 62.82051282 79 62.02531646 80 62.50000000 81 61.72839506 82 62.19512195 83 61.44578313 84 61.90476190 85 62.35294118 86 62.79069767 87 63.21839080 88 63.63636364 89 64.04494382 90 64.44444444 91 64.83516484 92 65.21739130 93 64.51612903 94 63.82978723 95 64.21052632 96 64.58333333 97 63.91752577 98 64.28571429 99 64.64646465 100 65.00000000 101 64.35643564 102 64.70588235 103 64.07766990 104 63.46153846 105 63.80952381 106 63.20754717 107 63.55140187 108 62.96296296 109 62.38532110 110 62.72727273 111 63.06306306 112 63.39285714 113 63.71681416 114 64.03508772 115 64.34782609 116 64.65517241 117 64.10256410 118 64.40677966 119 63.86554622 120 64.16666667 121 63.63636364 122 63.93442623 123 64.22764228 124 63.70967742 125 64.00000000 126 64.28571429 127 64.56692913 128 64.84375000 129 64.34108527 130 64.61538462 131 64.88549618 132 65.15151515 133 65.41353383 134 65.67164179 135 65.18518519 136 64.70588235 137 64.96350365 138 65.21739130 139 65.46762590 140 65.71428571 141 65.24822695 142 65.49295775 143 65.73426573 144 65.97222222 145 66.20689655 146 65.75342466 147 65.98639456 148 66.21621622 149 66.44295302 150 66.66666667 151 66.22516556 152 66.44736842 153 66.66666667 154 66.23376623 155 66.45161290 156 66.66666667 157 66.87898089 158 66.45569620 159 66.66666667 160 66.25000000 161 66.45962733 162 66.66666667 163 66.25766871 164 66.46341463 165 66.06060606 166 66.26506024 167 65.86826347 168 66.07142857 169 65.68047337 170 65.29411765 171 65.49707602 172 65.69767442 173 65.89595376 174 66.09195402 175 65.71428571 176 65.90909091 177 66.10169492 178 66.29213483 179 66.48044693 180 66.66666667 181 66.29834254 182 66.48351648 183 66.12021858 184 66.30434783 185 66.48648649 186 66.66666667 187 66.31016043 188 65.95744681 189 65.60846561 190 65.26315789 191 65.44502618 192 65.62500000 193 65.28497409 194 65.46391753 195 65.64102564 196 65.81632653 197 65.98984772 198 65.65656566 199 65.32663317 200 65.50000000 201 65.67164179 202 65.84158416 203 66.00985222 204 66.17647059 205 65.85365854 206 65.53398058 207 65.70048309 208 65.86538462 209 66.02870813 210 66.19047619 211 66.35071090 212 66.03773585 213 66.19718310 214 66.35514019 215 66.51162791 216 66.20370370 217 66.35944700 218 66.05504587 219 66.21004566 220 65.90909091 221 66.06334842 222 66.21621622 223 65.91928251 224 66.07142857 225 65.77777778 226 65.92920354 227 66.07929515 228 66.22807018 229 66.37554585 230 66.52173913 231 66.23376623 232 65.94827586 233 66.09442060 234 65.81196581 235 65.53191489 236 65.25423729 237 64.97890295 238 65.12605042 239 65.27196653 240 65.41666667 241 65.56016598 242 65.70247934 243 65.43209877 244 65.16393443 245 65.30612245 246 65.44715447 247 65.58704453 248 65.72580645 249 65.46184739 250 65.60000000 251 65.73705179 252 65.87301587 253 66.00790514 254 65.74803150 255 65.88235294 256 65.62500000 257 65.75875486 258 65.89147287 259 65.63706564 260 65.76923077 261 65.90038314 262 66.03053435 263 66.15969582 264 66.28787879 265 66.41509434 266 66.16541353 267 66.29213483 268 66.41791045 269 66.54275093 270 66.66666667 271 66.78966790 272 66.54411765 273 66.30036630 274 66.05839416 275 65.81818182 276 65.94202899 277 66.06498195 278 65.82733813 279 65.94982079 280 66.07142857 281 66.19217082 282 66.31205674 283 66.07773852 284 66.19718310 285 66.31578947 286 66.43356643 287 66.55052265 288 66.66666667 289 66.43598616 290 66.55172414 291 66.66666667 292 66.78082192 293 66.89419795 294 66.66666667 295 66.77966102 296 66.89189189 297 67.00336700 298 67.11409396 299 67.22408027 300 67.33333333 301 67.10963455 302 67.21854305 303 67.32673267 304 67.43421053 305 67.54098361 306 67.32026144 307 67.42671010 308 67.53246753 309 67.63754045 310 67.74193548 311 67.84565916 312 67.62820513 313 67.73162939 314 67.51592357 315 67.61904762 316 67.72151899 317 67.82334385 318 67.61006289 319 67.39811912 320 67.18750000 321 67.28971963 322 67.39130435 323 67.49226006 324 67.28395062 325 67.38461538 326 67.48466258 327 67.27828746 328 67.37804878 329 67.47720365 330 67.27272727 331 67.37160121 332 67.46987952 333 67.56756757 334 67.66467066 335 67.46268657 336 67.55952381 337 67.65578635 338 67.45562130 339 67.55162242 340 67.64705882 341 67.74193548 342 67.83625731 343 67.93002915 344 68.02325581 345 67.82608696 346 67.91907514 347 67.72334294 348 67.52873563 349 67.62177650 350 67.42857143 351 67.52136752 352 67.61363636 353 67.70538244 354 67.51412429 355 67.32394366 356 67.41573034 357 67.22689076 358 67.31843575 359 67.13091922 360 67.22222222 361 67.31301939 362 67.40331492 363 67.21763085 364 67.30769231 365 67.39726027 366 67.21311475 367 67.02997275 368 67.11956522 369 67.20867209 370 67.29729730 371 67.38544474 372 67.47311828 373 67.56032172 374 67.64705882 375 67.46666667 376 67.55319149 377 67.37400531 378 67.46031746 379 67.28232190 380 67.36842105 381 67.19160105 382 67.01570681 383 66.84073107 384 66.92708333 385 66.75324675 386 66.83937824 387 66.92506460 388 66.75257732 389 66.83804627 390 66.92307692 391 67.00767263 392 67.09183673 393 67.17557252 394 67.25888325 395 67.34177215 396 67.17171717 397 67.25440806 398 67.08542714 399 67.16791980 400 67.00000000 401 67.08229426 402 66.91542289 403 66.99751861 404 67.07920792 405 67.16049383 406 67.24137931 407 67.32186732 408 67.40196078 409 67.23716381 410 67.31707317 411 67.39659367 412 67.47572816 413 67.55447942 414 67.63285024 415 67.46987952 416 67.54807692 417 67.38609113 418 67.46411483 419 67.30310263 420 67.38095238 421 67.22090261 422 67.29857820 423 67.37588652 424 67.21698113 425 67.29411765 426 67.13615023 427 67.21311475 428 67.05607477 429 67.13286713 430 66.97674419 431 67.05336427 432 67.12962963 433 66.97459584 434 66.82027650 435 66.89655172 436 66.97247706 437 67.04805492 438 66.89497717 439 66.97038724 440 66.81818182 441 66.89342404 442 66.74208145 443 66.59142212 444 66.66666667 445 66.74157303 446 66.81614350 447 66.89038031 448 66.96428571 449 67.03786192 450 66.88888889 451 66.96230599 452 67.03539823 453 67.10816777 454 67.18061674 455 67.03296703 456 67.10526316 457 67.17724289 458 67.24890830 459 67.32026144 460 67.17391304 461 67.24511931 462 67.31601732 463 67.17062635 464 67.02586207 465 67.09677419 466 67.16738197 467 67.23768737 468 67.30769231 469 67.16417910 470 67.23404255 471 67.30360934 472 67.37288136 473 67.23044397 474 67.29957806 475 67.15789474 476 67.22689076 477 67.29559748 478 67.36401674 479 67.43215031 480 67.50000000 481 67.56756757 482 67.63485477 483 67.70186335 484 67.76859504 485 67.83505155 486 67.90123457 487 67.76180698 488 67.82786885 489 67.89366053 490 67.75510204 491 67.61710794 492 67.68292683 493 67.54563895 494 67.61133603 495 67.47474747 496 67.54032258 497 67.60563380 498 67.67068273 499 67.73547094 500 67.80000000 501 67.66467066 502 67.72908367 503 67.79324056 504 67.65873016 505 67.72277228 506 67.58893281 507 67.45562130 508 67.51968504 509 67.38703340 510 67.45098039 511 67.31898239 512 67.38281250 513 67.25146199 514 67.31517510 515 67.37864078 516 67.44186047 517 67.50483559 518 67.56756757 519 67.43737958 520 67.30769231 521 67.37044146 522 67.43295019 523 67.30401530 524 67.17557252 525 67.04761905 526 66.92015209 527 66.79316888 528 66.66666667 529 66.54064272 530 66.60377358 531 66.47834275 532 66.54135338 533 66.60412758 534 66.47940075 535 66.35514019 536 66.41791045 537 66.48044693 538 66.54275093 539 66.60482375 540 66.48148148 541 66.54343808 542 66.60516605 543 66.48250460 544 66.54411765 545 66.60550459 546 66.48351648 547 66.36197441 548 66.42335766 549 66.48451730 550 66.54545455 551 66.42468240 552 66.30434783 553 66.36528029 554 66.42599278 555 66.48648649 556 66.36690647 557 66.42728905 558 66.30824373 559 66.36851521 560 66.42857143 561 66.48841355 562 66.54804270 563 66.60746004 564 66.66666667 565 66.54867257 566 66.60777385 567 66.49029982 568 66.54929577 569 66.43233743 570 66.31578947 571 66.37478109 572 66.25874126 573 66.31762653 574 66.37630662 575 66.43478261 576 66.49305556 577 66.55112652 578 66.60899654 579 66.66666667 580 66.72413793 581 66.60929432 582 66.66666667 583 66.72384220 584 66.78082192 585 66.83760684 586 66.72354949 587 66.78023850 588 66.83673469 589 66.89303905 590 66.94915254 591 67.00507614 592 66.89189189 593 66.77908938 594 66.83501684 595 66.89075630 596 66.77852349 597 66.66666667 598 66.55518395 599 66.61101836 600 66.66666667 601 66.72212978 602 66.61129568 603 66.50082919 604 66.55629139 605 66.44628099 606 66.50165017 607 66.39209226 608 66.44736842 609 66.33825944 610 66.39344262 611 66.28477905 612 66.33986928 613 66.23164763 614 66.28664495 615 66.17886179 616 66.07142857 617 66.12641815 618 66.18122977 619 66.07431341 620 66.12903226 621 66.18357488 622 66.07717042 623 66.13162119 624 66.18589744 625 66.08000000 626 65.97444089 627 66.02870813 628 66.08280255 629 65.97774245 630 66.03174603 631 66.08557845 632 65.98101266 633 65.87677725 634 65.93059937 635 65.98425197 636 66.03773585 637 65.93406593 638 65.98746082 639 66.04068858 640 66.09375000 641 66.14664587 642 66.04361371 643 65.94090202 644 65.99378882 645 65.89147287 646 65.94427245 647 65.99690881 648 65.89506173 649 65.79352851 650 65.84615385 651 65.89861751 652 65.95092025 653 65.84992343 654 65.74923547 655 65.64885496 656 65.54878049 657 65.44901065 658 65.50151976 659 65.40212443 660 65.45454545 661 65.35552194 662 65.40785498 663 65.46003017 664 65.51204819 665 65.56390977 666 65.46546547 667 65.51724138 668 65.41916168 669 65.32137519 670 65.37313433 671 65.27570790 672 65.32738095 673 65.23031204 674 65.28189911 675 65.18518519 676 65.08875740 677 65.14032496 678 65.19174041 679 65.24300442 680 65.29411765 681 65.19823789 682 65.10263930 683 65.00732064 684 65.05847953 685 65.10948905 686 65.01457726 687 64.91994178 688 64.97093023 689 65.02177068 690 65.07246377 691 65.12301013 692 65.17341040 693 65.22366522 694 65.27377522 695 65.17985612 696 65.08620690 697 64.99282640 698 64.89971347 699 64.94992847 700 65.00000000 701 65.04992867 702 65.09971510 703 65.14935989 704 65.19886364 705 65.24822695 706 65.29745042 707 65.34653465 708 65.39548023 709 65.44428773 710 65.49295775 711 65.54149086 712 65.58988764 713 65.63814867 714 65.54621849 715 65.45454545 716 65.50279330 717 65.55090656 718 65.45961003 719 65.36856745 720 65.41666667 721 65.32593620 722 65.37396122 723 65.42185339 724 65.33149171 725 65.37931034 726 65.28925620 727 65.19944979 728 65.24725275 729 65.29492455 730 65.34246575 731 65.38987688 732 65.43715847 733 65.48431105 734 65.53133515 735 65.57823129 736 65.62500000 737 65.53595658 738 65.58265583 739 65.62922869 740 65.54054054 741 65.58704453 742 65.49865229 743 65.54508748 744 65.45698925 745 65.50335570 746 65.54959786 747 65.46184739 748 65.37433155 749 65.28704940 750 65.33333333 Final result: 65.3333 +/- 1.7389 Random chance: 25.0083 +/- 1.5824