common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 50.00000000 13 46.15384615 14 42.85714286 15 46.66666667 16 50.00000000 17 52.94117647 18 50.00000000 19 52.63157895 20 55.00000000 21 57.14285714 22 54.54545455 23 52.17391304 24 54.16666667 25 56.00000000 26 53.84615385 27 55.55555556 28 57.14285714 29 58.62068966 30 60.00000000 31 61.29032258 32 62.50000000 33 60.60606061 34 61.76470588 35 62.85714286 36 63.88888889 37 62.16216216 38 60.52631579 39 58.97435897 40 57.50000000 41 56.09756098 42 54.76190476 43 55.81395349 44 54.54545455 45 55.55555556 46 54.34782609 47 53.19148936 48 54.16666667 49 53.06122449 50 52.00000000 51 50.98039216 52 50.00000000 53 49.05660377 54 48.14814815 55 49.09090909 56 50.00000000 57 50.87719298 58 51.72413793 59 52.54237288 60 51.66666667 61 52.45901639 62 51.61290323 63 50.79365079 64 50.00000000 65 49.23076923 66 48.48484848 67 47.76119403 68 47.05882353 69 46.37681159 70 47.14285714 71 47.88732394 72 47.22222222 73 46.57534247 74 47.29729730 75 48.00000000 76 48.68421053 77 49.35064935 78 50.00000000 79 49.36708861 80 50.00000000 81 50.61728395 82 50.00000000 83 50.60240964 84 51.19047619 85 50.58823529 86 50.00000000 87 50.57471264 88 51.13636364 89 50.56179775 90 50.00000000 91 49.45054945 92 48.91304348 93 48.38709677 94 47.87234043 95 47.36842105 96 47.91666667 97 48.45360825 98 47.95918367 99 48.48484848 100 48.00000000 101 48.51485149 102 48.03921569 103 47.57281553 104 48.07692308 105 48.57142857 106 49.05660377 107 48.59813084 108 48.14814815 109 47.70642202 110 48.18181818 111 48.64864865 112 49.10714286 113 48.67256637 114 49.12280702 115 49.56521739 116 50.00000000 117 50.42735043 118 50.00000000 119 50.42016807 120 50.00000000 121 50.41322314 122 50.00000000 123 49.59349593 124 50.00000000 125 50.40000000 126 50.00000000 127 49.60629921 128 49.21875000 129 48.83720930 130 48.46153846 131 48.09160305 132 48.48484848 133 48.12030075 134 47.76119403 135 47.40740741 136 47.05882353 137 46.71532847 138 46.37681159 139 46.76258993 140 47.14285714 141 47.51773050 142 47.88732394 143 48.25174825 144 47.91666667 145 47.58620690 146 47.94520548 147 47.61904762 148 47.97297297 149 47.65100671 150 47.33333333 151 47.01986755 152 47.36842105 153 47.71241830 154 47.40259740 155 47.09677419 156 46.79487179 157 47.13375796 158 47.46835443 159 47.16981132 160 47.50000000 161 47.20496894 162 46.91358025 163 46.62576687 164 46.34146341 165 46.66666667 166 46.38554217 167 46.70658683 168 46.42857143 169 46.15384615 170 45.88235294 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 45.45454545 177 45.19774011 178 44.94382022 179 45.25139665 180 45.00000000 181 44.75138122 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 44.68085106 189 44.97354497 190 44.73684211 191 45.02617801 192 45.31250000 193 45.07772021 194 45.36082474 195 45.64102564 196 45.91836735 197 45.68527919 198 45.95959596 199 45.72864322 200 45.50000000 201 45.27363184 202 45.04950495 203 44.82758621 204 44.60784314 205 44.87804878 206 44.66019417 207 44.92753623 208 44.71153846 209 44.49760766 210 44.28571429 211 44.54976303 212 44.33962264 213 44.13145540 214 43.92523364 215 44.18604651 216 44.44444444 217 44.23963134 218 44.49541284 219 44.74885845 220 45.00000000 221 44.79638009 222 44.59459459 223 44.84304933 224 45.08928571 225 45.33333333 226 45.13274336 227 44.93392070 228 45.17543860 229 44.97816594 230 44.78260870 231 44.58874459 232 44.39655172 233 44.20600858 234 44.44444444 235 44.25531915 236 44.06779661 237 43.88185654 238 43.69747899 239 43.51464435 240 43.75000000 241 43.56846473 242 43.80165289 243 44.03292181 244 43.85245902 245 44.08163265 246 43.90243902 247 43.72469636 248 43.54838710 249 43.37349398 250 43.20000000 251 43.02788845 252 42.85714286 253 42.68774704 254 42.51968504 255 42.35294118 256 42.57812500 257 42.80155642 258 42.63565891 259 42.47104247 260 42.69230769 261 42.52873563 262 42.36641221 263 42.20532319 264 42.04545455 265 42.26415094 266 42.48120301 267 42.32209738 268 42.16417910 269 42.37918216 270 42.59259259 271 42.43542435 272 42.64705882 273 42.49084249 274 42.33576642 275 42.18181818 276 42.39130435 277 42.59927798 278 42.44604317 279 42.29390681 280 42.14285714 281 42.34875445 282 42.19858156 283 42.04946996 284 41.90140845 285 41.75438596 286 41.60839161 287 41.81184669 288 41.66666667 289 41.52249135 290 41.72413793 291 41.92439863 292 42.12328767 293 42.32081911 294 42.17687075 295 42.03389831 296 41.89189189 297 41.75084175 298 41.94630872 299 42.14046823 300 42.33333333 301 42.19269103 302 42.38410596 303 42.24422442 304 42.10526316 305 42.29508197 306 42.15686275 307 42.34527687 308 42.20779221 309 42.07119741 310 41.93548387 311 42.12218650 312 42.30769231 313 42.49201278 314 42.67515924 315 42.53968254 316 42.40506329 317 42.58675079 318 42.45283019 319 42.63322884 320 42.50000000 321 42.36760125 322 42.54658385 323 42.41486068 324 42.28395062 325 42.46153846 326 42.33128834 327 42.50764526 328 42.37804878 329 42.24924012 330 42.12121212 331 41.99395770 332 41.86746988 333 42.04204204 334 41.91616766 335 42.08955224 336 41.96428571 337 41.83976261 338 41.71597633 339 41.59292035 340 41.76470588 341 41.64222874 342 41.81286550 343 41.69096210 344 41.56976744 345 41.44927536 346 41.61849711 347 41.78674352 348 41.66666667 349 41.54727794 350 41.71428571 351 41.59544160 352 41.47727273 353 41.64305949 354 41.52542373 355 41.40845070 356 41.57303371 357 41.45658263 358 41.34078212 359 41.22562674 360 41.38888889 361 41.55124654 362 41.71270718 363 41.59779614 364 41.48351648 365 41.36986301 366 41.53005464 367 41.41689373 368 41.30434783 369 41.19241192 370 41.35135135 371 41.23989218 372 41.12903226 373 41.01876676 374 40.90909091 375 41.06666667 376 40.95744681 377 41.11405836 378 41.26984127 379 41.16094987 380 41.05263158 381 41.20734908 382 41.36125654 383 41.51436031 384 41.40625000 385 41.55844156 386 41.45077720 387 41.34366925 388 41.23711340 389 41.38817481 390 41.53846154 391 41.68797954 392 41.83673469 393 41.73027990 394 41.62436548 395 41.77215190 396 41.91919192 397 42.06549118 398 41.95979899 399 42.10526316 400 42.00000000 401 42.14463840 402 42.03980100 403 41.93548387 404 42.07920792 405 41.97530864 406 41.87192118 407 41.76904177 408 41.66666667 409 41.80929095 410 41.70731707 411 41.60583942 412 41.50485437 413 41.40435835 414 41.30434783 415 41.20481928 416 41.34615385 417 41.48681055 418 41.38755981 419 41.52744630 420 41.42857143 421 41.56769596 422 41.46919431 423 41.37115839 424 41.50943396 425 41.41176471 426 41.31455399 427 41.21779859 428 41.12149533 429 41.25874126 430 41.16279070 431 41.29930394 432 41.43518519 433 41.33949192 434 41.47465438 435 41.37931034 436 41.28440367 437 41.18993135 438 41.32420091 439 41.45785877 440 41.36363636 441 41.49659864 442 41.40271493 443 41.30925508 444 41.21621622 445 41.12359551 446 41.25560538 447 41.16331096 448 41.07142857 449 40.97995546 450 41.11111111 451 41.01995565 452 40.92920354 453 41.05960265 454 40.96916300 455 40.87912088 456 41.00877193 457 41.13785558 458 41.04803493 459 40.95860566 460 41.08695652 461 40.99783080 462 40.90909091 463 41.03671706 464 41.16379310 465 41.07526882 466 40.98712446 467 41.11349036 468 41.02564103 469 41.15138593 470 41.27659574 471 41.18895966 472 41.10169492 473 41.22621564 474 41.13924051 475 41.05263158 476 41.17647059 477 41.09014675 478 41.21338912 479 41.33611691 480 41.45833333 481 41.37214137 482 41.28630705 483 41.20082816 484 41.11570248 485 41.03092784 486 40.94650206 487 41.06776181 488 40.98360656 489 40.89979550 490 41.02040816 491 40.93686354 492 40.85365854 493 40.77079108 494 40.89068826 495 41.01010101 496 41.12903226 497 41.04627767 498 40.96385542 499 40.88176353 500 41.00000000 501 40.91816367 502 40.83665339 503 40.75546720 504 40.67460317 505 40.59405941 506 40.71146245 507 40.63116371 508 40.74803150 509 40.66797642 510 40.78431373 511 40.70450098 512 40.62500000 513 40.54580897 514 40.46692607 515 40.58252427 516 40.69767442 517 40.81237911 518 40.92664093 519 40.84778420 520 40.96153846 521 40.88291747 522 40.99616858 523 40.91778203 524 40.83969466 525 40.76190476 526 40.68441065 527 40.60721063 528 40.53030303 529 40.64272212 530 40.75471698 531 40.67796610 532 40.60150376 533 40.52532833 534 40.63670412 535 40.74766355 536 40.67164179 537 40.78212291 538 40.89219331 539 40.81632653 540 40.74074074 541 40.66543438 542 40.77490775 543 40.69981584 544 40.62500000 545 40.55045872 546 40.47619048 547 40.40219378 548 40.51094891 549 40.43715847 550 40.36363636 551 40.47186933 552 40.39855072 553 40.32549729 554 40.25270758 555 40.18018018 556 40.28776978 557 40.39497307 558 40.50179211 559 40.42933810 560 40.35714286 561 40.28520499 562 40.39145907 563 40.49733570 564 40.42553191 565 40.53097345 566 40.45936396 567 40.38800705 568 40.31690141 569 40.24604569 570 40.17543860 571 40.10507881 572 40.20979021 573 40.13961606 574 40.06968641 575 40.00000000 576 40.10416667 577 40.20797227 578 40.13840830 579 40.06908463 580 40.17241379 581 40.10327022 582 40.03436426 583 40.13722127 584 40.23972603 585 40.34188034 586 40.44368601 587 40.37478705 588 40.30612245 589 40.23769100 590 40.16949153 591 40.10152284 592 40.03378378 593 40.13490725 594 40.23569024 595 40.16806723 596 40.26845638 597 40.20100503 598 40.13377926 599 40.23372287 600 40.16666667 601 40.09983361 602 40.03322259 603 40.13266998 604 40.06622517 605 40.00000000 606 39.93399340 607 40.03294893 608 39.96710526 609 39.90147783 610 39.83606557 611 39.93453355 612 39.86928105 613 39.96737357 614 39.90228013 615 39.83739837 616 39.93506494 617 40.03241491 618 39.96763754 619 39.90306947 620 39.83870968 621 39.77455717 622 39.71061093 623 39.64686998 624 39.58333333 625 39.52000000 626 39.61661342 627 39.55342903 628 39.49044586 629 39.58664547 630 39.52380952 631 39.61965135 632 39.71518987 633 39.65244866 634 39.58990536 635 39.52755906 636 39.46540881 637 39.40345369 638 39.34169279 639 39.43661972 640 39.53125000 641 39.46957878 642 39.40809969 643 39.34681182 644 39.44099379 645 39.37984496 646 39.31888545 647 39.25811437 648 39.19753086 649 39.13713405 650 39.07692308 651 39.01689708 652 38.95705521 653 38.89739663 654 38.99082569 655 38.93129771 656 38.87195122 657 38.81278539 658 38.90577508 659 38.84673748 660 38.78787879 661 38.72919818 662 38.67069486 663 38.61236802 664 38.55421687 665 38.49624060 666 38.43843844 667 38.53073463 668 38.62275449 669 38.71449925 670 38.80597015 671 38.89716841 672 38.83928571 673 38.78157504 674 38.87240356 675 38.81481481 676 38.75739645 677 38.70014771 678 38.79056047 679 38.73343152 680 38.82352941 681 38.91336270 682 38.85630499 683 38.79941435 684 38.88888889 685 38.83211679 686 38.92128280 687 38.86462882 688 38.80813953 689 38.75181422 690 38.69565217 691 38.78437048 692 38.72832370 693 38.67243867 694 38.61671470 695 38.56115108 696 38.50574713 697 38.45050215 698 38.53868195 699 38.62660944 700 38.71428571 701 38.80171184 702 38.88888889 703 38.97581792 704 39.06250000 705 39.00709220 706 38.95184136 707 38.89674682 708 38.84180791 709 38.78702398 710 38.73239437 711 38.81856540 712 38.90449438 713 38.99018233 714 39.07563025 715 39.02097902 716 38.96648045 717 38.91213389 718 38.85793872 719 38.94297636 720 38.88888889 721 38.83495146 722 38.78116343 723 38.86583679 724 38.95027624 725 39.03448276 726 38.98071625 727 38.92709766 728 39.01098901 729 38.95747599 730 38.90410959 731 38.85088919 732 38.79781421 733 38.88130969 734 38.82833787 735 38.77551020 736 38.72282609 737 38.80597015 738 38.88888889 739 38.97158322 740 38.91891892 741 38.86639676 742 38.94878706 743 39.03095559 744 39.11290323 745 39.06040268 746 39.00804290 747 39.08969210 748 39.03743316 749 38.98531375 750 39.06666667 Final result: 39.0667 +/- 1.7827 Random chance: 25.0000 +/- 1.5822