common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 50.00000000 13 46.15384615 14 42.85714286 15 46.66666667 16 50.00000000 17 52.94117647 18 50.00000000 19 52.63157895 20 55.00000000 21 57.14285714 22 54.54545455 23 52.17391304 24 54.16666667 25 56.00000000 26 53.84615385 27 55.55555556 28 57.14285714 29 58.62068966 30 60.00000000 31 61.29032258 32 62.50000000 33 60.60606061 34 61.76470588 35 62.85714286 36 63.88888889 37 62.16216216 38 60.52631579 39 58.97435897 40 57.50000000 41 56.09756098 42 54.76190476 43 55.81395349 44 54.54545455 45 53.33333333 46 52.17391304 47 51.06382979 48 52.08333333 49 51.02040816 50 50.00000000 51 49.01960784 52 48.07692308 53 47.16981132 54 46.29629630 55 47.27272727 56 48.21428571 57 49.12280702 58 50.00000000 59 50.84745763 60 50.00000000 61 50.81967213 62 50.00000000 63 49.20634921 64 48.43750000 65 47.69230769 66 46.96969697 67 46.26865672 68 45.58823529 69 44.92753623 70 45.71428571 71 46.47887324 72 45.83333333 73 45.20547945 74 45.94594595 75 46.66666667 76 47.36842105 77 48.05194805 78 48.71794872 79 48.10126582 80 48.75000000 81 49.38271605 82 48.78048780 83 49.39759036 84 50.00000000 85 49.41176471 86 48.83720930 87 49.42528736 88 50.00000000 89 49.43820225 90 48.88888889 91 48.35164835 92 47.82608696 93 47.31182796 94 46.80851064 95 46.31578947 96 46.87500000 97 47.42268041 98 46.93877551 99 47.47474747 100 47.00000000 101 47.52475248 102 47.05882353 103 46.60194175 104 47.11538462 105 47.61904762 106 48.11320755 107 47.66355140 108 47.22222222 109 46.78899083 110 47.27272727 111 47.74774775 112 48.21428571 113 47.78761062 114 48.24561404 115 48.69565217 116 49.13793103 117 49.57264957 118 49.15254237 119 49.57983193 120 49.16666667 121 49.58677686 122 49.18032787 123 48.78048780 124 49.19354839 125 49.60000000 126 49.20634921 127 48.81889764 128 48.43750000 129 48.06201550 130 47.69230769 131 47.32824427 132 47.72727273 133 47.36842105 134 47.01492537 135 46.66666667 136 47.05882353 137 46.71532847 138 46.37681159 139 46.76258993 140 47.14285714 141 47.51773050 142 47.88732394 143 48.25174825 144 47.91666667 145 47.58620690 146 47.94520548 147 47.61904762 148 47.97297297 149 47.65100671 150 47.33333333 151 47.01986755 152 47.36842105 153 47.71241830 154 47.40259740 155 47.09677419 156 46.79487179 157 47.13375796 158 47.46835443 159 47.16981132 160 47.50000000 161 47.20496894 162 46.91358025 163 46.62576687 164 46.34146341 165 46.66666667 166 46.38554217 167 46.70658683 168 46.42857143 169 46.15384615 170 45.88235294 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 45.45454545 177 45.19774011 178 44.94382022 179 45.25139665 180 45.00000000 181 44.75138122 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 44.68085106 189 44.97354497 190 44.73684211 191 45.02617801 192 45.31250000 193 45.07772021 194 45.36082474 195 45.64102564 196 45.91836735 197 45.68527919 198 45.95959596 199 45.72864322 200 45.50000000 201 45.27363184 202 45.04950495 203 44.82758621 204 44.60784314 205 44.87804878 206 44.66019417 207 44.92753623 208 44.71153846 209 44.49760766 210 44.28571429 211 44.54976303 212 44.33962264 213 44.13145540 214 43.92523364 215 44.18604651 216 44.44444444 217 44.23963134 218 44.49541284 219 44.74885845 220 45.00000000 221 44.79638009 222 44.59459459 223 44.84304933 224 45.08928571 225 45.33333333 226 45.13274336 227 44.93392070 228 45.17543860 229 44.97816594 230 44.78260870 231 44.58874459 232 44.39655172 233 44.20600858 234 44.44444444 235 44.25531915 236 44.06779661 237 43.88185654 238 43.69747899 239 43.51464435 240 43.75000000 241 43.56846473 242 43.80165289 243 44.03292181 244 43.85245902 245 44.08163265 246 43.90243902 247 43.72469636 248 43.54838710 249 43.37349398 250 43.20000000 251 43.42629482 252 43.25396825 253 43.08300395 254 42.91338583 255 42.74509804 256 42.96875000 257 43.19066148 258 43.02325581 259 42.85714286 260 43.07692308 261 42.91187739 262 42.74809160 263 42.58555133 264 42.42424242 265 42.64150943 266 42.85714286 267 42.69662921 268 42.53731343 269 42.75092937 270 42.96296296 271 42.80442804 272 43.01470588 273 42.85714286 274 42.70072993 275 42.54545455 276 42.75362319 277 42.96028881 278 42.80575540 279 42.65232975 280 42.50000000 281 42.34875445 282 42.19858156 283 42.04946996 284 41.90140845 285 41.75438596 286 41.60839161 287 41.81184669 288 41.66666667 289 41.52249135 290 41.72413793 291 41.92439863 292 42.12328767 293 42.32081911 294 42.17687075 295 42.03389831 296 41.89189189 297 41.75084175 298 41.94630872 299 42.14046823 300 42.33333333 301 42.19269103 302 42.38410596 303 42.24422442 304 42.10526316 305 42.29508197 306 42.15686275 307 42.34527687 308 42.20779221 309 42.07119741 310 41.93548387 311 42.12218650 312 42.30769231 313 42.49201278 314 42.67515924 315 42.53968254 316 42.40506329 317 42.58675079 318 42.45283019 319 42.63322884 320 42.50000000 321 42.36760125 322 42.54658385 323 42.41486068 324 42.28395062 325 42.46153846 326 42.33128834 327 42.50764526 328 42.37804878 329 42.24924012 330 42.12121212 331 41.99395770 332 41.86746988 333 42.04204204 334 41.91616766 335 42.08955224 336 41.96428571 337 41.83976261 338 41.71597633 339 41.59292035 340 41.76470588 341 41.64222874 342 41.81286550 343 41.69096210 344 41.56976744 345 41.44927536 346 41.61849711 347 41.78674352 348 41.66666667 349 41.54727794 350 41.71428571 351 41.59544160 352 41.47727273 353 41.64305949 354 41.52542373 355 41.40845070 356 41.57303371 357 41.45658263 358 41.34078212 359 41.22562674 360 41.38888889 361 41.55124654 362 41.71270718 363 41.59779614 364 41.48351648 365 41.36986301 366 41.53005464 367 41.41689373 368 41.30434783 369 41.19241192 370 41.35135135 371 41.23989218 372 41.12903226 373 41.01876676 374 40.90909091 375 41.06666667 376 40.95744681 377 41.11405836 378 41.26984127 379 41.16094987 380 41.05263158 381 41.20734908 382 41.36125654 383 41.51436031 384 41.40625000 385 41.55844156 386 41.45077720 387 41.34366925 388 41.23711340 389 41.38817481 390 41.53846154 391 41.68797954 392 41.83673469 393 41.73027990 394 41.62436548 395 41.77215190 396 41.91919192 397 41.81360202 398 41.70854271 399 41.85463659 400 41.75000000 401 41.89526185 402 41.79104478 403 41.68734491 404 41.83168317 405 41.72839506 406 41.62561576 407 41.52334152 408 41.42156863 409 41.56479218 410 41.46341463 411 41.36253041 412 41.26213592 413 41.16222760 414 41.06280193 415 40.96385542 416 41.10576923 417 41.24700240 418 41.14832536 419 41.28878282 420 41.19047619 421 41.33016627 422 41.23222749 423 41.13475177 424 41.27358491 425 41.17647059 426 41.07981221 427 41.21779859 428 41.12149533 429 41.25874126 430 41.16279070 431 41.29930394 432 41.43518519 433 41.33949192 434 41.47465438 435 41.37931034 436 41.28440367 437 41.18993135 438 41.32420091 439 41.45785877 440 41.36363636 441 41.26984127 442 41.17647059 443 41.08352144 444 40.99099099 445 40.89887640 446 41.03139013 447 40.93959732 448 40.84821429 449 40.75723831 450 40.66666667 451 40.57649667 452 40.48672566 453 40.61810155 454 40.52863436 455 40.43956044 456 40.57017544 457 40.70021882 458 40.61135371 459 40.52287582 460 40.65217391 461 40.56399132 462 40.47619048 463 40.60475162 464 40.73275862 465 40.64516129 466 40.55793991 467 40.68522484 468 40.59829060 469 40.72494670 470 40.85106383 471 40.76433121 472 40.67796610 473 40.80338266 474 40.71729958 475 40.63157895 476 40.75630252 477 40.67085954 478 40.79497908 479 40.91858038 480 41.04166667 481 40.95634096 482 40.87136929 483 40.78674948 484 40.70247934 485 40.61855670 486 40.74074074 487 40.86242300 488 40.77868852 489 40.69529652 490 40.81632653 491 40.73319756 492 40.65040650 493 40.56795132 494 40.68825911 495 40.80808081 496 40.92741935 497 40.84507042 498 40.76305221 499 40.88176353 500 41.00000000 501 40.91816367 502 40.83665339 503 40.75546720 504 40.67460317 505 40.59405941 506 40.71146245 507 40.63116371 508 40.74803150 509 40.66797642 510 40.78431373 511 40.70450098 512 40.62500000 513 40.54580897 514 40.46692607 515 40.38834951 516 40.50387597 517 40.61895551 518 40.73359073 519 40.65510597 520 40.76923077 521 40.69097889 522 40.80459770 523 40.72657744 524 40.64885496 525 40.57142857 526 40.49429658 527 40.41745731 528 40.34090909 529 40.45368620 530 40.56603774 531 40.48964218 532 40.41353383 533 40.33771107 534 40.44943820 535 40.56074766 536 40.48507463 537 40.59590317 538 40.70631970 539 40.63079777 540 40.55555556 541 40.48059150 542 40.59040590 543 40.51565378 544 40.44117647 545 40.36697248 546 40.29304029 547 40.21937843 548 40.32846715 549 40.25500911 550 40.18181818 551 40.29038113 552 40.21739130 553 40.14466546 554 40.07220217 555 40.00000000 556 40.10791367 557 40.21543986 558 40.32258065 559 40.25044723 560 40.17857143 561 40.10695187 562 40.21352313 563 40.31971581 564 40.24822695 565 40.35398230 566 40.28268551 567 40.21164021 568 40.14084507 569 40.07029877 570 40.00000000 571 39.92994746 572 40.03496503 573 39.96509599 574 39.89547038 575 39.82608696 576 39.93055556 577 40.03466205 578 39.96539792 579 39.89637306 580 40.00000000 581 39.93115318 582 39.86254296 583 39.96569468 584 40.06849315 585 40.17094017 586 40.27303754 587 40.20442930 588 40.13605442 589 40.06791171 590 40.00000000 591 39.93231810 592 39.86486486 593 39.96627319 594 40.06734007 595 40.00000000 596 40.10067114 597 40.03350084 598 39.96655518 599 40.06677796 600 40.00000000 601 40.09983361 602 40.03322259 603 40.13266998 604 40.06622517 605 40.00000000 606 39.93399340 607 40.03294893 608 39.96710526 609 39.90147783 610 39.83606557 611 39.93453355 612 39.86928105 613 39.96737357 614 39.90228013 615 39.83739837 616 39.93506494 617 40.03241491 618 39.96763754 619 39.90306947 620 39.83870968 621 39.77455717 622 39.71061093 623 39.64686998 624 39.58333333 625 39.52000000 626 39.61661342 627 39.55342903 628 39.49044586 629 39.58664547 630 39.52380952 631 39.61965135 632 39.71518987 633 39.65244866 634 39.58990536 635 39.52755906 636 39.46540881 637 39.40345369 638 39.34169279 639 39.43661972 640 39.53125000 641 39.46957878 642 39.40809969 643 39.34681182 644 39.44099379 645 39.37984496 646 39.31888545 647 39.25811437 648 39.19753086 649 39.13713405 650 39.07692308 651 39.01689708 652 38.95705521 653 38.89739663 654 38.99082569 655 38.93129771 656 38.87195122 657 38.81278539 658 38.90577508 659 38.84673748 660 38.78787879 661 38.72919818 662 38.67069486 663 38.61236802 664 38.55421687 665 38.49624060 666 38.43843844 667 38.53073463 668 38.62275449 669 38.71449925 670 38.80597015 671 38.89716841 672 38.83928571 673 38.78157504 674 38.72403561 675 38.66666667 676 38.60946746 677 38.55243722 678 38.64306785 679 38.58615611 680 38.67647059 681 38.76651982 682 38.70967742 683 38.65300146 684 38.74269006 685 38.68613139 686 38.77551020 687 38.71906841 688 38.66279070 689 38.60667634 690 38.55072464 691 38.63965268 692 38.58381503 693 38.52813853 694 38.61671470 695 38.56115108 696 38.64942529 697 38.59397418 698 38.68194842 699 38.76967096 700 38.71428571 701 38.80171184 702 38.88888889 703 38.97581792 704 39.06250000 705 39.00709220 706 38.95184136 707 38.89674682 708 38.84180791 709 38.78702398 710 38.73239437 711 38.81856540 712 38.90449438 713 38.99018233 714 39.07563025 715 39.02097902 716 38.96648045 717 38.91213389 718 38.85793872 719 38.94297636 720 38.88888889 721 38.83495146 722 38.78116343 723 38.86583679 724 38.95027624 725 39.03448276 726 38.98071625 727 38.92709766 728 39.01098901 729 38.95747599 730 38.90410959 731 38.85088919 732 38.79781421 733 38.88130969 734 38.82833787 735 38.77551020 736 38.72282609 737 38.80597015 738 38.88888889 739 38.97158322 740 38.91891892 741 38.86639676 742 38.94878706 743 39.03095559 744 39.11290323 745 39.06040268 746 39.00804290 747 39.08969210 748 39.03743316 749 38.98531375 750 39.06666667 Final result: 39.0667 +/- 1.7827 Random chance: 25.0000 +/- 1.5822