common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 47.05882353 18 44.44444444 19 47.36842105 20 50.00000000 21 52.38095238 22 50.00000000 23 47.82608696 24 50.00000000 25 52.00000000 26 50.00000000 27 51.85185185 28 50.00000000 29 51.72413793 30 53.33333333 31 54.83870968 32 56.25000000 33 54.54545455 34 55.88235294 35 57.14285714 36 55.55555556 37 54.05405405 38 52.63157895 39 51.28205128 40 50.00000000 41 48.78048780 42 47.61904762 43 48.83720930 44 47.72727273 45 48.88888889 46 47.82608696 47 46.80851064 48 47.91666667 49 46.93877551 50 46.00000000 51 45.09803922 52 44.23076923 53 43.39622642 54 44.44444444 55 45.45454545 56 46.42857143 57 47.36842105 58 48.27586207 59 47.45762712 60 46.66666667 61 47.54098361 62 46.77419355 63 46.03174603 64 45.31250000 65 44.61538462 66 43.93939394 67 43.28358209 68 42.64705882 69 42.02898551 70 42.85714286 71 43.66197183 72 43.05555556 73 42.46575342 74 43.24324324 75 44.00000000 76 44.73684211 77 45.45454545 78 46.15384615 79 45.56962025 80 46.25000000 81 46.91358025 82 46.34146341 83 46.98795181 84 47.61904762 85 47.05882353 86 46.51162791 87 47.12643678 88 47.72727273 89 47.19101124 90 46.66666667 91 46.15384615 92 45.65217391 93 45.16129032 94 45.74468085 95 45.26315789 96 45.83333333 97 45.36082474 98 44.89795918 99 44.44444444 100 44.00000000 101 44.55445545 102 44.11764706 103 43.68932039 104 44.23076923 105 44.76190476 106 45.28301887 107 44.85981308 108 45.37037037 109 44.95412844 110 44.54545455 111 45.04504505 112 44.64285714 113 44.24778761 114 44.73684211 115 45.21739130 116 45.68965517 117 45.29914530 118 44.91525424 119 45.37815126 120 45.00000000 121 45.45454545 122 45.08196721 123 44.71544715 124 45.16129032 125 45.60000000 126 45.23809524 127 44.88188976 128 44.53125000 129 44.18604651 130 43.84615385 131 44.27480916 132 44.69696970 133 44.36090226 134 44.02985075 135 43.70370370 136 43.38235294 137 43.06569343 138 42.75362319 139 43.16546763 140 43.57142857 141 43.97163121 142 44.36619718 143 44.75524476 144 44.44444444 145 44.82758621 146 45.20547945 147 44.89795918 148 45.27027027 149 44.96644295 150 44.66666667 151 44.37086093 152 44.73684211 153 45.09803922 154 44.80519481 155 44.51612903 156 44.23076923 157 44.58598726 158 44.93670886 159 44.65408805 160 45.00000000 161 44.72049689 162 44.44444444 163 44.17177914 164 43.90243902 165 44.24242424 166 43.97590361 167 44.31137725 168 44.04761905 169 43.78698225 170 43.52941176 171 43.27485380 172 43.02325581 173 42.77456647 174 42.52873563 175 42.28571429 176 42.61363636 177 42.37288136 178 42.13483146 179 42.45810056 180 42.22222222 181 41.98895028 182 42.30769231 183 42.62295082 184 42.39130435 185 42.70270270 186 43.01075269 187 42.78074866 188 43.08510638 189 43.38624339 190 43.15789474 191 43.45549738 192 43.75000000 193 43.52331606 194 43.29896907 195 43.58974359 196 43.87755102 197 43.65482234 198 43.93939394 199 43.71859296 200 43.50000000 201 43.28358209 202 43.06930693 203 42.85714286 204 42.64705882 205 42.92682927 206 42.71844660 207 42.99516908 208 42.78846154 209 42.58373206 210 42.38095238 211 42.65402844 212 42.45283019 213 42.25352113 214 42.05607477 215 42.32558140 216 42.59259259 217 42.39631336 218 42.66055046 219 42.92237443 220 43.18181818 221 42.98642534 222 42.79279279 223 43.04932735 224 42.85714286 225 43.11111111 226 42.92035398 227 42.73127753 228 42.98245614 229 42.79475983 230 42.60869565 231 42.42424242 232 42.67241379 233 42.48927039 234 42.73504274 235 42.55319149 236 42.37288136 237 42.19409283 238 42.01680672 239 41.84100418 240 42.08333333 241 41.90871369 242 42.14876033 243 42.38683128 244 42.21311475 245 42.44897959 246 42.27642276 247 42.51012146 248 42.33870968 249 42.16867470 250 42.00000000 251 41.83266932 252 41.66666667 253 41.50197628 254 41.33858268 255 41.17647059 256 41.40625000 257 41.63424125 258 41.47286822 259 41.31274131 260 41.53846154 261 41.37931034 262 41.22137405 263 41.06463878 264 40.90909091 265 41.13207547 266 41.35338346 267 41.19850187 268 41.04477612 269 41.26394052 270 41.48148148 271 41.32841328 272 41.54411765 273 41.39194139 274 41.24087591 275 41.09090909 276 41.30434783 277 41.51624549 278 41.36690647 279 41.21863799 280 41.07142857 281 41.28113879 282 41.13475177 283 40.98939929 284 40.84507042 285 41.05263158 286 40.90909091 287 41.11498258 288 40.97222222 289 40.83044983 290 41.03448276 291 41.23711340 292 41.43835616 293 41.63822526 294 41.49659864 295 41.35593220 296 41.21621622 297 41.07744108 298 41.27516779 299 41.47157191 300 41.66666667 301 41.52823920 302 41.72185430 303 41.58415842 304 41.44736842 305 41.63934426 306 41.50326797 307 41.69381107 308 41.55844156 309 41.42394822 310 41.29032258 311 41.47909968 312 41.66666667 313 41.85303514 314 42.03821656 315 41.90476190 316 41.77215190 317 41.95583596 318 41.82389937 319 42.00626959 320 41.87500000 321 41.74454829 322 41.92546584 323 41.79566563 324 41.66666667 325 41.84615385 326 41.71779141 327 41.89602446 328 41.76829268 329 41.64133739 330 41.51515152 331 41.38972810 332 41.26506024 333 41.44144144 334 41.31736527 335 41.49253731 336 41.36904762 337 41.24629080 338 41.12426036 339 41.00294985 340 41.17647059 341 41.34897361 342 41.52046784 343 41.39941691 344 41.27906977 345 41.15942029 346 41.32947977 347 41.49855908 348 41.37931034 349 41.26074499 350 41.42857143 351 41.31054131 352 41.47727273 353 41.35977337 354 41.24293785 355 41.12676056 356 41.29213483 357 41.45658263 358 41.34078212 359 41.22562674 360 41.38888889 361 41.55124654 362 41.71270718 363 41.59779614 364 41.48351648 365 41.36986301 366 41.53005464 367 41.41689373 368 41.30434783 369 41.19241192 370 41.35135135 371 41.23989218 372 41.12903226 373 41.01876676 374 40.90909091 375 41.06666667 376 40.95744681 377 41.11405836 378 41.26984127 379 41.16094987 380 41.05263158 381 41.20734908 382 41.36125654 383 41.51436031 384 41.40625000 385 41.55844156 386 41.45077720 387 41.34366925 388 41.23711340 389 41.38817481 390 41.53846154 391 41.68797954 392 41.83673469 393 41.73027990 394 41.62436548 395 41.77215190 396 41.91919192 397 42.06549118 398 41.95979899 399 42.10526316 400 42.00000000 401 42.14463840 402 42.03980100 403 41.93548387 404 41.83168317 405 41.97530864 406 41.87192118 407 41.76904177 408 41.66666667 409 41.80929095 410 41.70731707 411 41.60583942 412 41.50485437 413 41.40435835 414 41.54589372 415 41.44578313 416 41.58653846 417 41.48681055 418 41.38755981 419 41.52744630 420 41.42857143 421 41.56769596 422 41.46919431 423 41.37115839 424 41.50943396 425 41.64705882 426 41.78403756 427 41.68618267 428 41.58878505 429 41.72494172 430 41.62790698 431 41.76334107 432 41.89814815 433 41.80138568 434 41.93548387 435 41.83908046 436 41.74311927 437 41.64759725 438 41.78082192 439 41.91343964 440 41.81818182 441 41.72335601 442 41.62895928 443 41.53498871 444 41.44144144 445 41.34831461 446 41.47982063 447 41.38702461 448 41.29464286 449 41.20267261 450 41.33333333 451 41.46341463 452 41.37168142 453 41.50110375 454 41.40969163 455 41.31868132 456 41.44736842 457 41.57549234 458 41.48471616 459 41.39433551 460 41.52173913 461 41.43167028 462 41.34199134 463 41.46868251 464 41.59482759 465 41.50537634 466 41.63090129 467 41.75588865 468 41.66666667 469 41.79104478 470 41.70212766 471 41.61358811 472 41.52542373 473 41.64904863 474 41.56118143 475 41.47368421 476 41.59663866 477 41.50943396 478 41.63179916 479 41.75365344 480 41.87500000 481 41.78794179 482 41.70124481 483 41.61490683 484 41.52892562 485 41.44329897 486 41.56378601 487 41.68377823 488 41.59836066 489 41.51329243 490 41.63265306 491 41.54786151 492 41.46341463 493 41.37931034 494 41.49797571 495 41.61616162 496 41.73387097 497 41.64989940 498 41.56626506 499 41.68336673 500 41.80000000 501 41.71656687 502 41.63346614 503 41.55069583 504 41.46825397 505 41.38613861 506 41.50197628 507 41.42011834 508 41.33858268 509 41.25736739 510 41.37254902 511 41.29158513 512 41.21093750 513 41.13060429 514 41.05058366 515 41.16504854 516 41.27906977 517 41.39264990 518 41.50579151 519 41.42581888 520 41.53846154 521 41.45873321 522 41.57088123 523 41.49139579 524 41.41221374 525 41.33333333 526 41.25475285 527 41.17647059 528 41.28787879 529 41.39886578 530 41.50943396 531 41.43126177 532 41.35338346 533 41.27579737 534 41.38576779 535 41.49532710 536 41.41791045 537 41.52700186 538 41.63568773 539 41.55844156 540 41.48148148 541 41.40480591 542 41.51291513 543 41.43646409 544 41.54411765 545 41.46788991 546 41.39194139 547 41.31627057 548 41.42335766 549 41.34790528 550 41.27272727 551 41.37931034 552 41.30434783 553 41.22965642 554 41.15523466 555 41.08108108 556 41.18705036 557 41.29263914 558 41.39784946 559 41.32379249 560 41.25000000 561 41.17647059 562 41.28113879 563 41.38543517 564 41.31205674 565 41.41592920 566 41.34275618 567 41.26984127 568 41.37323944 569 41.30052724 570 41.22807018 571 41.15586690 572 41.25874126 573 41.18673647 574 41.11498258 575 41.04347826 576 41.14583333 577 41.24783362 578 41.17647059 579 41.10535406 580 41.20689655 581 41.13597246 582 41.06529210 583 41.16638079 584 41.26712329 585 41.36752137 586 41.46757679 587 41.39693356 588 41.32653061 589 41.25636672 590 41.18644068 591 41.11675127 592 41.04729730 593 41.14671164 594 41.24579125 595 41.17647059 596 41.27516779 597 41.20603015 598 41.13712375 599 41.23539232 600 41.16666667 601 41.09816972 602 41.02990033 603 41.12769486 604 41.05960265 605 40.99173554 606 40.92409241 607 41.02141680 608 40.95394737 609 40.88669951 610 40.81967213 611 40.91653028 612 40.84967320 613 40.94616639 614 40.87947883 615 40.81300813 616 40.90909091 617 41.00486224 618 40.93851133 619 40.87237480 620 40.80645161 621 40.74074074 622 40.67524116 623 40.60995185 624 40.54487179 625 40.48000000 626 40.57507987 627 40.51036683 628 40.44585987 629 40.54054054 630 40.63492063 631 40.72900158 632 40.82278481 633 40.75829384 634 40.69400631 635 40.62992126 636 40.56603774 637 40.50235479 638 40.43887147 639 40.53208138 640 40.62500000 641 40.56162246 642 40.49844237 643 40.43545879 644 40.52795031 645 40.46511628 646 40.40247678 647 40.34003091 648 40.27777778 649 40.21571649 650 40.15384615 651 40.09216590 652 40.03067485 653 39.96937213 654 40.06116208 655 40.00000000 656 39.93902439 657 39.87823440 658 39.96960486 659 39.90895296 660 39.84848485 661 39.78819970 662 39.72809668 663 39.66817496 664 39.60843373 665 39.54887218 666 39.48948949 667 39.58020990 668 39.67065868 669 39.76083707 670 39.85074627 671 39.79135618 672 39.73214286 673 39.67310550 674 39.61424332 675 39.55555556 676 39.49704142 677 39.43870015 678 39.52802360 679 39.46980854 680 39.55882353 681 39.64757709 682 39.58944282 683 39.53147877 684 39.61988304 685 39.56204380 686 39.65014577 687 39.59243086 688 39.53488372 689 39.47750363 690 39.42028986 691 39.50795948 692 39.45086705 693 39.39393939 694 39.33717579 695 39.28057554 696 39.36781609 697 39.31133429 698 39.39828080 699 39.48497854 700 39.42857143 701 39.51497860 702 39.60113960 703 39.68705548 704 39.77272727 705 39.71631206 706 39.66005666 707 39.60396040 708 39.54802260 709 39.49224260 710 39.43661972 711 39.52180028 712 39.60674157 713 39.69144460 714 39.63585434 715 39.58041958 716 39.52513966 717 39.47001395 718 39.41504178 719 39.49930459 720 39.44444444 721 39.38973648 722 39.33518006 723 39.41908714 724 39.36464088 725 39.44827586 726 39.53168044 727 39.47730399 728 39.42307692 729 39.36899863 730 39.31506849 731 39.26128591 732 39.20765027 733 39.29058663 734 39.37329700 735 39.31972789 736 39.26630435 737 39.34871099 738 39.43089431 739 39.51285521 740 39.45945946 741 39.40620783 742 39.48787062 743 39.56931359 744 39.65053763 745 39.59731544 746 39.54423592 747 39.49129853 748 39.43850267 749 39.51935915 750 39.60000000 Final result: 39.6000 +/- 1.7870 Random chance: 25.0000 +/- 1.5822