common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 23.07692308 14 21.42857143 15 26.66666667 16 31.25000000 17 29.41176471 18 27.77777778 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 31.81818182 45 33.33333333 46 32.60869565 47 34.04255319 48 35.41666667 49 34.69387755 50 36.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 34.37500000 65 33.84615385 66 34.84848485 67 34.32835821 68 33.82352941 69 34.78260870 70 34.28571429 71 33.80281690 72 34.72222222 73 35.61643836 74 35.13513514 75 34.66666667 76 34.21052632 77 33.76623377 78 33.33333333 79 32.91139241 80 33.75000000 81 33.33333333 82 34.14634146 83 33.73493976 84 33.33333333 85 32.94117647 86 32.55813953 87 32.18390805 88 31.81818182 89 32.58426966 90 32.22222222 91 31.86813187 92 31.52173913 93 31.18279570 94 30.85106383 95 30.52631579 96 31.25000000 97 31.95876289 98 31.63265306 99 32.32323232 100 33.00000000 101 33.66336634 102 33.33333333 103 33.00970874 104 32.69230769 105 32.38095238 106 32.07547170 107 32.71028037 108 33.33333333 109 33.02752294 110 33.63636364 111 33.33333333 112 33.92857143 113 33.62831858 114 34.21052632 115 33.91304348 116 33.62068966 117 33.33333333 118 33.05084746 119 32.77310924 120 32.50000000 121 33.05785124 122 32.78688525 123 32.52032520 124 33.06451613 125 32.80000000 126 32.53968254 127 32.28346457 128 32.03125000 129 31.78294574 130 32.30769231 131 32.06106870 132 31.81818182 133 31.57894737 134 31.34328358 135 31.85185185 136 31.61764706 137 32.11678832 138 32.60869565 139 33.09352518 140 32.85714286 141 32.62411348 142 32.39436620 143 32.16783217 144 32.63888889 145 32.41379310 146 32.19178082 147 31.97278912 148 31.75675676 149 31.54362416 150 31.33333333 151 31.78807947 152 31.57894737 153 31.37254902 154 31.16883117 155 30.96774194 156 31.41025641 157 31.21019108 158 31.64556962 159 31.44654088 160 31.25000000 161 31.67701863 162 32.09876543 163 31.90184049 164 31.70731707 165 32.12121212 166 32.53012048 167 32.33532934 168 32.14285714 169 31.95266272 170 31.76470588 171 31.57894737 172 31.39534884 173 31.21387283 174 31.60919540 175 32.00000000 176 32.38636364 177 32.20338983 178 32.02247191 179 32.40223464 180 32.77777778 181 32.59668508 182 32.41758242 183 32.24043716 184 32.06521739 185 31.89189189 186 32.25806452 187 32.08556150 188 31.91489362 189 32.27513228 190 32.63157895 191 32.46073298 192 32.29166667 193 32.12435233 194 32.47422680 195 32.30769231 196 32.14285714 197 32.48730964 198 32.82828283 199 32.66331658 200 32.50000000 201 32.33830846 202 32.67326733 203 32.51231527 204 32.35294118 205 32.68292683 206 32.52427184 207 32.85024155 208 32.69230769 209 32.53588517 210 32.38095238 211 32.22748815 212 32.07547170 213 31.92488263 214 31.77570093 215 31.62790698 216 31.94444444 217 32.25806452 218 32.56880734 219 32.42009132 220 32.27272727 221 32.12669683 222 31.98198198 223 31.83856502 224 32.14285714 225 32.00000000 226 32.30088496 227 32.59911894 228 32.45614035 229 32.31441048 230 32.17391304 231 32.03463203 232 31.89655172 233 32.18884120 234 32.47863248 235 32.34042553 236 32.62711864 237 32.91139241 238 33.19327731 239 33.05439331 240 32.91666667 241 32.78008299 242 32.64462810 243 32.51028807 244 32.37704918 245 32.24489796 246 32.11382114 247 32.38866397 248 32.25806452 249 32.53012048 250 32.40000000 251 32.66932271 252 32.53968254 253 32.41106719 254 32.28346457 255 32.15686275 256 32.03125000 257 31.90661479 258 31.78294574 259 31.66023166 260 31.53846154 261 31.80076628 262 31.67938931 263 31.55893536 264 31.81818182 265 32.07547170 266 31.95488722 267 31.83520599 268 31.71641791 269 31.59851301 270 31.48148148 271 31.36531365 272 31.25000000 273 31.50183150 274 31.38686131 275 31.27272727 276 31.15942029 277 31.04693141 278 30.93525180 279 30.82437276 280 31.07142857 281 31.31672598 282 31.20567376 283 31.44876325 284 31.69014085 285 31.57894737 286 31.81818182 287 32.05574913 288 32.29166667 289 32.17993080 290 32.06896552 291 32.30240550 292 32.19178082 293 32.42320819 294 32.31292517 295 32.54237288 296 32.77027027 297 32.65993266 298 32.55033557 299 32.44147157 300 32.33333333 301 32.22591362 302 32.11920530 303 32.01320132 304 31.90789474 305 31.80327869 306 31.69934641 307 31.59609121 308 31.81818182 309 31.71521036 310 31.61290323 311 31.51125402 312 31.73076923 313 31.62939297 314 31.52866242 315 31.42857143 316 31.32911392 317 31.23028391 318 31.13207547 319 31.03448276 320 30.93750000 321 30.84112150 322 31.05590062 323 30.95975232 324 30.86419753 325 30.76923077 326 30.67484663 327 30.88685015 328 31.09756098 329 31.00303951 330 31.21212121 331 31.11782477 332 31.32530120 333 31.23123123 334 31.13772455 335 31.34328358 336 31.54761905 337 31.45400593 338 31.36094675 339 31.56342183 340 31.47058824 341 31.67155425 342 31.87134503 343 31.77842566 344 31.97674419 345 32.17391304 346 32.08092486 347 32.27665706 348 32.18390805 349 32.09169054 350 32.00000000 351 31.90883191 352 31.81818182 353 31.72804533 354 31.63841808 355 31.83098592 356 31.74157303 357 31.65266106 358 31.84357542 359 31.75487465 360 31.66666667 361 31.85595568 362 31.76795580 363 31.95592287 364 31.86813187 365 31.78082192 366 31.69398907 367 31.88010899 368 31.79347826 369 31.70731707 370 31.62162162 371 31.53638814 372 31.72043011 373 31.63538874 374 31.81818182 375 31.73333333 376 31.64893617 377 31.83023873 378 31.74603175 379 31.66226913 380 31.84210526 381 31.75853018 382 31.67539267 383 31.59268930 384 31.51041667 385 31.42857143 386 31.34715026 387 31.26614987 388 31.44329897 389 31.36246787 390 31.28205128 391 31.45780051 392 31.37755102 393 31.55216285 394 31.72588832 395 31.64556962 396 31.56565657 397 31.73803526 398 31.65829146 399 31.57894737 400 31.50000000 401 31.67082294 402 31.84079602 403 31.76178660 404 31.68316832 405 31.60493827 406 31.52709360 407 31.44963145 408 31.37254902 409 31.29584352 410 31.21951220 411 31.14355231 412 31.31067961 413 31.47699758 414 31.64251208 415 31.80722892 416 31.73076923 417 31.65467626 418 31.57894737 419 31.50357995 420 31.66666667 421 31.82897862 422 31.75355450 423 31.67848700 424 31.60377358 425 31.52941176 426 31.45539906 427 31.38173302 428 31.30841121 429 31.23543124 430 31.16279070 431 31.09048724 432 31.25000000 433 31.40877598 434 31.56682028 435 31.49425287 436 31.42201835 437 31.35011442 438 31.50684932 439 31.66287016 440 31.59090909 441 31.51927438 442 31.44796380 443 31.37697517 444 31.53153153 445 31.46067416 446 31.39013453 447 31.31991051 448 31.25000000 449 31.40311804 450 31.55555556 451 31.48558758 452 31.41592920 453 31.56732892 454 31.49779736 455 31.42857143 456 31.35964912 457 31.29102845 458 31.22270742 459 31.15468410 460 31.08695652 461 31.01952278 462 31.16883117 463 31.31749460 464 31.46551724 465 31.61290323 466 31.54506438 467 31.47751606 468 31.62393162 469 31.55650320 470 31.48936170 471 31.63481953 472 31.77966102 473 31.71247357 474 31.85654008 475 32.00000000 476 31.93277311 477 32.07547170 478 32.00836820 479 31.94154489 480 31.87500000 481 31.80873181 482 31.95020747 483 31.88405797 484 32.02479339 485 32.16494845 486 32.09876543 487 32.23819302 488 32.17213115 489 32.31083845 490 32.44897959 491 32.58655804 492 32.72357724 493 32.65720081 494 32.79352227 495 32.72727273 496 32.66129032 497 32.59557344 498 32.53012048 499 32.46492986 500 32.40000000 501 32.53493014 502 32.47011952 503 32.40556660 504 32.53968254 505 32.47524752 506 32.41106719 507 32.54437870 508 32.67716535 509 32.80943026 510 32.94117647 511 32.87671233 512 32.81250000 513 32.74853801 514 32.68482490 515 32.81553398 516 32.75193798 517 32.88201161 518 32.81853282 519 32.75529865 520 32.69230769 521 32.62955854 522 32.56704981 523 32.50478011 524 32.44274809 525 32.38095238 526 32.31939163 527 32.25806452 528 32.19696970 529 32.32514178 530 32.45283019 531 32.39171375 532 32.33082707 533 32.45778612 534 32.39700375 535 32.33644860 536 32.27611940 537 32.40223464 538 32.34200743 539 32.28200371 540 32.22222222 541 32.16266174 542 32.10332103 543 32.04419890 544 31.98529412 545 31.92660550 546 31.86813187 547 31.99268739 548 32.11678832 549 32.05828780 550 32.00000000 551 31.94192377 552 32.06521739 553 32.18806510 554 32.12996390 555 32.07207207 556 32.01438849 557 31.95691203 558 31.89964158 559 31.84257603 560 31.96428571 561 32.08556150 562 32.02846975 563 31.97158082 564 31.91489362 565 32.03539823 566 32.15547703 567 32.09876543 568 32.21830986 569 32.33743409 570 32.28070175 571 32.39929947 572 32.34265734 573 32.28621291 574 32.40418118 575 32.52173913 576 32.63888889 577 32.75563258 578 32.87197232 579 32.98791019 580 33.10344828 581 33.04647160 582 33.16151203 583 33.10463122 584 33.21917808 585 33.16239316 586 33.27645051 587 33.21976150 588 33.16326531 589 33.10696095 590 33.05084746 591 32.99492386 592 32.93918919 593 32.88364250 594 32.99663300 595 32.94117647 596 32.88590604 597 32.83082077 598 32.77591973 599 32.72120200 600 32.66666667 601 32.77870216 602 32.72425249 603 32.83582090 604 32.78145695 605 32.72727273 606 32.83828383 607 32.78418451 608 32.73026316 609 32.84072250 610 32.78688525 611 32.73322422 612 32.84313725 613 32.78955954 614 32.73615635 615 32.68292683 616 32.79220779 617 32.90113452 618 32.84789644 619 32.95638126 620 32.90322581 621 32.85024155 622 32.95819936 623 32.90529695 624 32.85256410 625 32.80000000 626 32.74760383 627 32.85486443 628 32.96178344 629 32.90937997 630 32.85714286 631 32.96354992 632 32.91139241 633 32.85939968 634 32.96529968 635 32.91338583 636 32.86163522 637 32.81004710 638 32.75862069 639 32.70735524 640 32.81250000 641 32.76131045 642 32.71028037 643 32.65940902 644 32.60869565 645 32.71317829 646 32.66253870 647 32.61205564 648 32.71604938 649 32.66563945 650 32.76923077 651 32.87250384 652 32.97546012 653 33.07810107 654 33.02752294 655 32.97709924 656 33.07926829 657 33.18112633 658 33.13069909 659 33.23216995 660 33.18181818 661 33.13161876 662 33.08157100 663 33.03167421 664 32.98192771 665 32.93233083 666 33.03303303 667 33.13343328 668 33.08383234 669 33.03437967 670 33.13432836 671 33.08494784 672 33.03571429 673 32.98662704 674 32.93768546 675 33.03703704 676 33.13609467 677 33.08714919 678 33.03834808 679 32.98969072 680 32.94117647 681 32.89280470 682 32.99120235 683 32.94289898 684 33.04093567 685 32.99270073 686 32.94460641 687 32.89665211 688 32.84883721 689 32.94629898 690 32.89855072 691 32.99565847 692 33.09248555 693 33.04473304 694 32.99711816 695 32.94964029 696 33.04597701 697 32.99856528 698 32.95128940 699 32.90414878 700 33.00000000 701 32.95292439 702 32.90598291 703 32.85917496 704 32.81250000 705 32.76595745 706 32.86118980 707 32.95615276 708 32.90960452 709 32.86318759 710 32.81690141 711 32.77074543 712 32.86516854 713 32.81907433 714 32.77310924 715 32.86713287 716 32.82122905 717 32.91492329 718 32.86908078 719 32.96244784 720 32.91666667 721 32.87101248 722 32.82548476 723 32.78008299 724 32.73480663 725 32.68965517 726 32.78236915 727 32.73727648 728 32.69230769 729 32.78463649 730 32.87671233 731 32.83173735 732 32.92349727 733 32.87858117 734 32.83378747 735 32.78911565 736 32.74456522 737 32.83582090 738 32.79132791 739 32.74695535 740 32.70270270 741 32.65856950 742 32.61455526 743 32.57065949 744 32.66129032 745 32.61744966 746 32.70777480 747 32.79785810 748 32.75401070 749 32.84379172 750 32.93333333 Final result: 32.9333 +/- 1.7172 Random chance: 19.8992 +/- 1.4588