common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 71.42857143 8 75.00000000 9 66.66666667 10 70.00000000 11 63.63636364 12 66.66666667 13 61.53846154 14 64.28571429 15 66.66666667 16 68.75000000 17 70.58823529 18 66.66666667 19 68.42105263 20 65.00000000 21 66.66666667 22 68.18181818 23 65.21739130 24 66.66666667 25 64.00000000 26 61.53846154 27 62.96296296 28 60.71428571 29 62.06896552 30 63.33333333 31 61.29032258 32 59.37500000 33 57.57575758 34 55.88235294 35 57.14285714 36 55.55555556 37 56.75675676 38 55.26315789 39 56.41025641 40 57.50000000 41 58.53658537 42 57.14285714 43 55.81395349 44 54.54545455 45 55.55555556 46 56.52173913 47 57.44680851 48 58.33333333 49 59.18367347 50 58.00000000 51 56.86274510 52 57.69230769 53 56.60377358 54 55.55555556 55 54.54545455 56 55.35714286 57 56.14035088 58 56.89655172 59 57.62711864 60 58.33333333 61 59.01639344 62 58.06451613 63 58.73015873 64 59.37500000 65 58.46153846 66 59.09090909 67 59.70149254 68 60.29411765 69 60.86956522 70 60.00000000 71 59.15492958 72 58.33333333 73 58.90410959 74 59.45945946 75 60.00000000 76 59.21052632 77 59.74025974 78 60.25641026 79 59.49367089 80 60.00000000 81 59.25925926 82 59.75609756 83 59.03614458 84 59.52380952 85 60.00000000 86 60.46511628 87 60.91954023 88 61.36363636 89 61.79775281 90 62.22222222 91 62.63736264 92 63.04347826 93 62.36559140 94 61.70212766 95 62.10526316 96 62.50000000 97 61.85567010 98 62.24489796 99 62.62626263 100 63.00000000 101 62.37623762 102 62.74509804 103 62.13592233 104 61.53846154 105 61.90476190 106 61.32075472 107 61.68224299 108 61.11111111 109 60.55045872 110 60.90909091 111 61.26126126 112 61.60714286 113 61.94690265 114 61.40350877 115 61.73913043 116 61.20689655 117 60.68376068 118 61.01694915 119 60.50420168 120 60.83333333 121 60.33057851 122 60.65573770 123 60.97560976 124 60.48387097 125 60.80000000 126 61.11111111 127 61.41732283 128 61.71875000 129 61.24031008 130 61.53846154 131 61.83206107 132 62.12121212 133 62.40601504 134 61.94029851 135 61.48148148 136 61.02941176 137 61.31386861 138 61.59420290 139 61.87050360 140 62.14285714 141 61.70212766 142 61.97183099 143 62.23776224 144 62.50000000 145 62.75862069 146 62.32876712 147 62.58503401 148 62.83783784 149 63.08724832 150 63.33333333 151 62.91390728 152 63.15789474 153 63.39869281 154 62.98701299 155 63.22580645 156 63.46153846 157 63.69426752 158 63.29113924 159 63.52201258 160 63.12500000 161 63.35403727 162 63.58024691 163 63.19018405 164 63.41463415 165 63.03030303 166 63.25301205 167 62.87425150 168 63.09523810 169 62.72189349 170 62.35294118 171 62.57309942 172 62.79069767 173 63.00578035 174 63.21839080 175 62.85714286 176 63.06818182 177 63.27683616 178 63.48314607 179 63.68715084 180 63.88888889 181 63.53591160 182 63.73626374 183 63.38797814 184 63.58695652 185 63.78378378 186 63.97849462 187 63.63636364 188 63.29787234 189 62.96296296 190 62.63157895 191 62.82722513 192 63.02083333 193 62.69430052 194 62.88659794 195 63.07692308 196 63.26530612 197 63.45177665 198 63.13131313 199 62.81407035 200 63.00000000 201 63.18407960 202 63.36633663 203 63.54679803 204 63.72549020 205 63.41463415 206 63.10679612 207 63.28502415 208 63.46153846 209 63.63636364 210 63.80952381 211 63.98104265 212 63.67924528 213 63.84976526 214 64.01869159 215 64.18604651 216 63.88888889 217 64.05529954 218 63.76146789 219 63.92694064 220 63.63636364 221 63.80090498 222 63.96396396 223 63.67713004 224 63.83928571 225 63.55555556 226 63.71681416 227 63.87665198 228 64.03508772 229 64.19213974 230 64.34782609 231 64.06926407 232 63.79310345 233 63.94849785 234 64.10256410 235 63.82978723 236 63.55932203 237 63.71308017 238 63.86554622 239 64.01673640 240 64.16666667 241 64.31535270 242 64.46280992 243 64.19753086 244 63.93442623 245 64.08163265 246 64.22764228 247 64.37246964 248 64.51612903 249 64.25702811 250 64.40000000 251 64.54183267 252 64.68253968 253 64.82213439 254 64.56692913 255 64.70588235 256 64.45312500 257 64.20233463 258 64.34108527 259 64.09266409 260 64.23076923 261 64.36781609 262 64.50381679 263 64.63878327 264 64.77272727 265 64.90566038 266 64.66165414 267 64.79400749 268 64.92537313 269 65.05576208 270 65.18518519 271 65.31365314 272 65.07352941 273 64.83516484 274 64.59854015 275 64.36363636 276 64.49275362 277 64.62093863 278 64.38848921 279 64.51612903 280 64.64285714 281 64.76868327 282 64.89361702 283 64.66431095 284 64.78873239 285 64.91228070 286 65.03496503 287 65.15679443 288 65.27777778 289 65.05190311 290 65.17241379 291 65.29209622 292 65.41095890 293 65.52901024 294 65.30612245 295 65.42372881 296 65.54054054 297 65.65656566 298 65.77181208 299 65.88628763 300 66.00000000 301 65.78073090 302 65.89403974 303 66.00660066 304 66.11842105 305 66.22950820 306 66.01307190 307 66.12377850 308 66.23376623 309 66.34304207 310 66.45161290 311 66.55948553 312 66.66666667 313 66.77316294 314 66.56050955 315 66.66666667 316 66.77215190 317 66.87697161 318 66.66666667 319 66.45768025 320 66.25000000 321 66.35514019 322 66.45962733 323 66.56346749 324 66.35802469 325 66.46153846 326 66.56441718 327 66.36085627 328 66.46341463 329 66.56534954 330 66.66666667 331 66.76737160 332 66.86746988 333 66.96696697 334 67.06586826 335 66.86567164 336 66.96428571 337 67.06231454 338 66.86390533 339 66.96165192 340 67.05882353 341 67.15542522 342 67.25146199 343 67.34693878 344 67.44186047 345 67.24637681 346 67.34104046 347 67.14697406 348 66.95402299 349 67.04871060 350 66.85714286 351 66.95156695 352 67.04545455 353 67.13881020 354 66.94915254 355 66.76056338 356 66.85393258 357 66.66666667 358 66.75977654 359 66.57381616 360 66.66666667 361 66.75900277 362 66.85082873 363 66.66666667 364 66.75824176 365 66.84931507 366 66.66666667 367 66.48501362 368 66.30434783 369 66.39566396 370 66.48648649 371 66.57681941 372 66.66666667 373 66.75603217 374 66.84491979 375 66.66666667 376 66.75531915 377 66.57824934 378 66.66666667 379 66.49076517 380 66.57894737 381 66.40419948 382 66.23036649 383 66.05744125 384 66.14583333 385 65.97402597 386 66.06217617 387 66.14987080 388 65.97938144 389 66.06683805 390 65.89743590 391 65.72890026 392 65.81632653 393 65.90330789 394 65.98984772 395 66.07594937 396 65.90909091 397 65.99496222 398 65.82914573 399 65.91478697 400 65.75000000 401 65.83541147 402 65.67164179 403 65.75682382 404 65.84158416 405 65.92592593 406 66.00985222 407 66.09336609 408 66.17647059 409 66.01466993 410 66.09756098 411 66.18004866 412 66.26213592 413 66.34382567 414 66.42512077 415 66.26506024 416 66.34615385 417 66.18705036 418 66.26794258 419 66.10978520 420 66.19047619 421 66.03325416 422 65.87677725 423 65.95744681 424 65.80188679 425 65.88235294 426 65.72769953 427 65.80796253 428 65.65420561 429 65.73426573 430 65.81395349 431 65.89327146 432 65.97222222 433 65.81986143 434 65.89861751 435 65.97701149 436 66.05504587 437 66.13272311 438 65.98173516 439 66.05922551 440 65.90909091 441 65.98639456 442 65.83710407 443 65.68848758 444 65.76576577 445 65.84269663 446 65.91928251 447 65.99552573 448 66.07142857 449 66.14699332 450 66.00000000 451 66.07538803 452 66.15044248 453 66.22516556 454 66.29955947 455 66.15384615 456 66.22807018 457 66.30196937 458 66.37554585 459 66.44880174 460 66.30434783 461 66.37744035 462 66.45021645 463 66.30669546 464 66.16379310 465 66.23655914 466 66.30901288 467 66.38115632 468 66.45299145 469 66.31130064 470 66.38297872 471 66.45435244 472 66.52542373 473 66.38477801 474 66.45569620 475 66.31578947 476 66.38655462 477 66.45702306 478 66.52719665 479 66.59707724 480 66.66666667 481 66.73596674 482 66.80497925 483 66.87370600 484 66.94214876 485 67.01030928 486 67.07818930 487 66.94045175 488 67.00819672 489 67.07566462 490 66.93877551 491 66.80244399 492 66.86991870 493 66.73427992 494 66.80161943 495 66.86868687 496 66.93548387 497 67.00201207 498 67.06827309 499 67.13426854 500 67.20000000 501 67.06586826 502 67.13147410 503 67.19681909 504 67.26190476 505 67.32673267 506 67.19367589 507 67.06114398 508 67.12598425 509 66.99410609 510 66.86274510 511 66.73189824 512 66.79687500 513 66.66666667 514 66.73151751 515 66.79611650 516 66.86046512 517 66.92456480 518 66.98841699 519 67.05202312 520 66.92307692 521 66.98656430 522 67.04980843 523 66.92160612 524 66.98473282 525 66.85714286 526 66.73003802 527 66.60341556 528 66.47727273 529 66.35160681 530 66.41509434 531 66.47834275 532 66.54135338 533 66.60412758 534 66.47940075 535 66.35514019 536 66.41791045 537 66.48044693 538 66.54275093 539 66.60482375 540 66.48148148 541 66.54343808 542 66.60516605 543 66.48250460 544 66.54411765 545 66.60550459 546 66.48351648 547 66.36197441 548 66.42335766 549 66.48451730 550 66.54545455 551 66.42468240 552 66.30434783 553 66.36528029 554 66.42599278 555 66.48648649 556 66.36690647 557 66.42728905 558 66.30824373 559 66.36851521 560 66.42857143 561 66.48841355 562 66.54804270 563 66.60746004 564 66.48936170 565 66.37168142 566 66.43109541 567 66.49029982 568 66.54929577 569 66.43233743 570 66.31578947 571 66.37478109 572 66.25874126 573 66.31762653 574 66.37630662 575 66.43478261 576 66.49305556 577 66.55112652 578 66.60899654 579 66.66666667 580 66.55172414 581 66.43717728 582 66.49484536 583 66.55231561 584 66.60958904 585 66.66666667 586 66.55290102 587 66.60988075 588 66.66666667 589 66.72325976 590 66.77966102 591 66.83587140 592 66.72297297 593 66.61045531 594 66.66666667 595 66.72268908 596 66.61073826 597 66.49916248 598 66.38795987 599 66.44407346 600 66.50000000 601 66.55574043 602 66.44518272 603 66.33499171 604 66.39072848 605 66.44628099 606 66.50165017 607 66.55683690 608 66.61184211 609 66.50246305 610 66.55737705 611 66.44844517 612 66.50326797 613 66.39477977 614 66.44951140 615 66.34146341 616 66.23376623 617 66.28849271 618 66.34304207 619 66.23586430 620 66.29032258 621 66.34460548 622 66.23794212 623 66.29213483 624 66.34615385 625 66.24000000 626 66.13418530 627 66.18819777 628 66.24203822 629 66.13672496 630 66.19047619 631 66.24405705 632 66.13924051 633 66.03475513 634 66.08832808 635 66.14173228 636 66.19496855 637 66.09105181 638 66.14420063 639 66.19718310 640 66.09375000 641 66.14664587 642 66.04361371 643 65.94090202 644 65.99378882 645 65.89147287 646 65.94427245 647 65.84234930 648 65.74074074 649 65.63944530 650 65.69230769 651 65.74500768 652 65.79754601 653 65.69678407 654 65.59633028 655 65.49618321 656 65.39634146 657 65.29680365 658 65.19756839 659 65.09863429 660 65.15151515 661 65.05295008 662 65.10574018 663 65.15837104 664 65.21084337 665 65.26315789 666 65.16516517 667 65.21739130 668 65.11976048 669 65.02242152 670 65.07462687 671 64.97764531 672 65.02976190 673 64.93313522 674 64.98516320 675 64.88888889 676 64.79289941 677 64.84490399 678 64.89675516 679 64.94845361 680 65.00000000 681 64.90455213 682 64.80938416 683 64.71449488 684 64.76608187 685 64.81751825 686 64.72303207 687 64.62882096 688 64.68023256 689 64.58635704 690 64.63768116 691 64.54413893 692 64.59537572 693 64.64646465 694 64.69740634 695 64.60431655 696 64.51149425 697 64.41893831 698 64.32664756 699 64.37768240 700 64.42857143 701 64.47931526 702 64.52991453 703 64.58036984 704 64.63068182 705 64.68085106 706 64.73087819 707 64.78076379 708 64.83050847 709 64.88011283 710 64.92957746 711 64.97890295 712 65.02808989 713 65.07713885 714 64.98599440 715 64.89510490 716 64.94413408 717 64.85355649 718 64.76323120 719 64.67315716 720 64.72222222 721 64.63245492 722 64.68144044 723 64.73029046 724 64.64088398 725 64.68965517 726 64.60055096 727 64.51169188 728 64.56043956 729 64.60905350 730 64.65753425 731 64.70588235 732 64.75409836 733 64.80218281 734 64.85013624 735 64.89795918 736 64.94565217 737 64.85753053 738 64.90514905 739 64.81732070 740 64.72972973 741 64.77732794 742 64.69002695 743 64.73755047 744 64.65053763 745 64.69798658 746 64.74530831 747 64.65863454 748 64.57219251 749 64.48598131 750 64.53333333 Final result: 64.5333 +/- 1.7481 Random chance: 25.0083 +/- 1.5824