common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 55.55555556 10 60.00000000 11 54.54545455 12 58.33333333 13 53.84615385 14 50.00000000 15 53.33333333 16 56.25000000 17 58.82352941 18 55.55555556 19 57.89473684 20 60.00000000 21 61.90476190 22 59.09090909 23 56.52173913 24 58.33333333 25 60.00000000 26 57.69230769 27 59.25925926 28 60.71428571 29 62.06896552 30 63.33333333 31 61.29032258 32 62.50000000 33 60.60606061 34 61.76470588 35 62.85714286 36 61.11111111 37 59.45945946 38 60.52631579 39 58.97435897 40 57.50000000 41 56.09756098 42 54.76190476 43 55.81395349 44 54.54545455 45 55.55555556 46 54.34782609 47 53.19148936 48 54.16666667 49 55.10204082 50 54.00000000 51 52.94117647 52 53.84615385 53 52.83018868 54 51.85185185 55 52.72727273 56 53.57142857 57 54.38596491 58 53.44827586 59 52.54237288 60 51.66666667 61 52.45901639 62 51.61290323 63 50.79365079 64 50.00000000 65 49.23076923 66 48.48484848 67 47.76119403 68 47.05882353 69 46.37681159 70 47.14285714 71 47.88732394 72 47.22222222 73 46.57534247 74 47.29729730 75 48.00000000 76 48.68421053 77 49.35064935 78 50.00000000 79 49.36708861 80 50.00000000 81 50.61728395 82 50.00000000 83 50.60240964 84 51.19047619 85 50.58823529 86 50.00000000 87 50.57471264 88 51.13636364 89 50.56179775 90 50.00000000 91 49.45054945 92 48.91304348 93 48.38709677 94 48.93617021 95 48.42105263 96 48.95833333 97 48.45360825 98 47.95918367 99 47.47474747 100 47.00000000 101 47.52475248 102 47.05882353 103 46.60194175 104 47.11538462 105 47.61904762 106 48.11320755 107 47.66355140 108 48.14814815 109 48.62385321 110 49.09090909 111 49.54954955 112 49.10714286 113 48.67256637 114 49.12280702 115 49.56521739 116 50.00000000 117 49.57264957 118 49.15254237 119 49.57983193 120 49.16666667 121 49.58677686 122 49.18032787 123 48.78048780 124 48.38709677 125 48.80000000 126 48.41269841 127 48.03149606 128 47.65625000 129 47.28682171 130 46.92307692 131 46.56488550 132 46.96969697 133 46.61654135 134 46.26865672 135 45.92592593 136 45.58823529 137 45.25547445 138 44.92753623 139 45.32374101 140 45.71428571 141 46.09929078 142 46.47887324 143 46.85314685 144 47.22222222 145 47.58620690 146 47.94520548 147 47.61904762 148 47.97297297 149 47.65100671 150 47.33333333 151 47.01986755 152 47.36842105 153 47.71241830 154 47.40259740 155 47.09677419 156 46.79487179 157 46.49681529 158 46.83544304 159 46.54088050 160 46.87500000 161 46.58385093 162 46.29629630 163 46.01226994 164 45.73170732 165 46.06060606 166 45.78313253 167 46.10778443 168 45.83333333 169 46.15384615 170 45.88235294 171 46.19883041 172 45.93023256 173 45.66473988 174 45.40229885 175 45.14285714 176 44.88636364 177 44.63276836 178 44.38202247 179 44.69273743 180 44.44444444 181 44.19889503 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 45.21276596 189 45.50264550 190 45.26315789 191 45.54973822 192 45.31250000 193 45.07772021 194 45.36082474 195 45.64102564 196 45.91836735 197 45.68527919 198 45.95959596 199 45.72864322 200 45.50000000 201 45.27363184 202 45.04950495 203 44.82758621 204 44.60784314 205 44.87804878 206 44.66019417 207 44.92753623 208 44.71153846 209 44.49760766 210 44.28571429 211 44.54976303 212 44.33962264 213 44.13145540 214 43.92523364 215 44.18604651 216 44.44444444 217 44.23963134 218 44.49541284 219 44.74885845 220 45.00000000 221 44.79638009 222 44.59459459 223 44.84304933 224 45.08928571 225 45.33333333 226 45.13274336 227 44.93392070 228 45.17543860 229 44.97816594 230 44.78260870 231 45.02164502 232 45.25862069 233 45.49356223 234 45.72649573 235 45.53191489 236 45.33898305 237 45.14767932 238 44.95798319 239 44.76987448 240 45.00000000 241 44.81327801 242 45.04132231 243 45.26748971 244 45.08196721 245 45.30612245 246 45.12195122 247 44.93927126 248 45.16129032 249 44.97991968 250 44.80000000 251 44.62151394 252 44.44444444 253 44.26877470 254 44.09448819 255 43.92156863 256 44.14062500 257 44.35797665 258 44.18604651 259 44.01544402 260 44.23076923 261 44.06130268 262 43.89312977 263 43.72623574 264 43.56060606 265 43.39622642 266 43.60902256 267 43.44569288 268 43.28358209 269 43.49442379 270 43.70370370 271 43.54243542 272 43.75000000 273 43.58974359 274 43.43065693 275 43.27272727 276 43.47826087 277 43.68231047 278 43.52517986 279 43.36917563 280 43.21428571 281 43.06049822 282 42.90780142 283 42.75618375 284 42.60563380 285 42.45614035 286 42.30769231 287 42.50871080 288 42.36111111 289 42.21453287 290 42.41379310 291 42.61168385 292 42.80821918 293 43.00341297 294 42.85714286 295 42.71186441 296 42.56756757 297 42.42424242 298 42.61744966 299 42.80936455 300 43.00000000 301 42.85714286 302 43.04635762 303 42.90429043 304 43.09210526 305 43.27868852 306 43.13725490 307 43.32247557 308 43.18181818 309 43.04207120 310 42.90322581 311 43.08681672 312 43.26923077 313 43.13099042 314 43.31210191 315 43.17460317 316 43.03797468 317 43.21766562 318 43.08176101 319 43.26018809 320 43.12500000 321 42.99065421 322 43.16770186 323 43.03405573 324 42.90123457 325 43.07692308 326 42.94478528 327 43.11926606 328 42.98780488 329 42.85714286 330 42.72727273 331 42.59818731 332 42.46987952 333 42.64264264 334 42.51497006 335 42.68656716 336 42.85714286 337 42.72997033 338 42.60355030 339 42.47787611 340 42.64705882 341 42.81524927 342 42.98245614 343 42.85714286 344 42.73255814 345 42.60869565 346 42.77456647 347 42.93948127 348 42.81609195 349 42.69340974 350 42.85714286 351 42.73504274 352 42.61363636 353 42.77620397 354 42.65536723 355 42.53521127 356 42.69662921 357 42.57703081 358 42.45810056 359 42.33983287 360 42.50000000 361 42.65927978 362 42.81767956 363 42.69972452 364 42.58241758 365 42.46575342 366 42.62295082 367 42.50681199 368 42.39130435 369 42.27642276 370 42.43243243 371 42.31805930 372 42.20430108 373 42.09115282 374 41.97860963 375 42.13333333 376 42.02127660 377 42.17506631 378 42.32804233 379 42.21635884 380 42.10526316 381 42.25721785 382 42.14659686 383 42.29765013 384 42.18750000 385 42.33766234 386 42.22797927 387 42.11886305 388 42.01030928 389 42.15938303 390 42.30769231 391 42.45524297 392 42.60204082 393 42.49363868 394 42.38578680 395 42.53164557 396 42.67676768 397 42.82115869 398 42.71356784 399 42.85714286 400 42.75000000 401 42.89276808 402 43.03482587 403 42.92803970 404 43.06930693 405 43.20987654 406 43.10344828 407 42.99754300 408 42.89215686 409 43.03178484 410 42.92682927 411 42.82238443 412 42.71844660 413 42.61501211 414 42.51207729 415 42.40963855 416 42.54807692 417 42.44604317 418 42.34449761 419 42.48210024 420 42.38095238 421 42.51781473 422 42.41706161 423 42.31678487 424 42.45283019 425 42.58823529 426 42.72300469 427 42.85714286 428 42.75700935 429 42.89044289 430 42.79069767 431 42.92343387 432 42.82407407 433 42.72517321 434 42.62672811 435 42.52873563 436 42.43119266 437 42.33409611 438 42.46575342 439 42.59681093 440 42.50000000 441 42.40362812 442 42.30769231 443 42.21218962 444 42.11711712 445 42.02247191 446 41.92825112 447 41.83445190 448 41.74107143 449 41.64810690 450 41.77777778 451 41.90687361 452 41.81415929 453 41.94260486 454 41.85022026 455 41.75824176 456 41.88596491 457 42.01312910 458 41.92139738 459 41.83006536 460 41.95652174 461 41.86550976 462 41.77489177 463 41.90064795 464 42.02586207 465 41.93548387 466 41.84549356 467 41.97002141 468 42.09401709 469 42.21748401 470 42.12765957 471 42.03821656 472 41.94915254 473 42.07188161 474 41.98312236 475 41.89473684 476 42.01680672 477 41.92872117 478 42.05020921 479 41.96242171 480 42.08333333 481 41.99584200 482 41.90871369 483 41.82194617 484 41.73553719 485 41.64948454 486 41.76954733 487 41.88911704 488 41.80327869 489 41.71779141 490 41.83673469 491 41.75152749 492 41.66666667 493 41.58215010 494 41.70040486 495 41.81818182 496 41.93548387 497 41.85110664 498 41.76706827 499 41.68336673 500 41.80000000 501 41.71656687 502 41.83266932 503 41.74950298 504 41.66666667 505 41.58415842 506 41.69960474 507 41.61735700 508 41.73228346 509 41.65029470 510 41.56862745 511 41.48727984 512 41.40625000 513 41.32553606 514 41.24513619 515 41.16504854 516 41.27906977 517 41.39264990 518 41.50579151 519 41.42581888 520 41.34615385 521 41.26679463 522 41.37931034 523 41.30019120 524 41.41221374 525 41.33333333 526 41.25475285 527 41.17647059 528 41.09848485 529 41.02079395 530 41.13207547 531 41.05461394 532 40.97744361 533 40.90056285 534 41.01123596 535 41.12149533 536 41.04477612 537 41.15456238 538 41.26394052 539 41.18738404 540 41.11111111 541 41.03512015 542 41.14391144 543 41.06813996 544 41.17647059 545 41.10091743 546 41.02564103 547 40.95063985 548 41.05839416 549 40.98360656 550 40.90909091 551 41.01633394 552 40.94202899 553 40.86799277 554 40.97472924 555 40.90090090 556 41.00719424 557 41.11310592 558 41.21863799 559 41.14490161 560 41.07142857 561 40.99821747 562 41.10320285 563 41.20781528 564 41.13475177 565 41.23893805 566 41.34275618 567 41.26984127 568 41.19718310 569 41.12478032 570 41.05263158 571 40.98073555 572 41.08391608 573 41.01221640 574 40.94076655 575 40.86956522 576 40.97222222 577 41.07452340 578 41.17647059 579 41.10535406 580 41.20689655 581 41.30808950 582 41.23711340 583 41.33790738 584 41.43835616 585 41.36752137 586 41.46757679 587 41.39693356 588 41.32653061 589 41.25636672 590 41.18644068 591 41.11675127 592 41.04729730 593 41.14671164 594 41.24579125 595 41.17647059 596 41.27516779 597 41.20603015 598 41.13712375 599 41.23539232 600 41.16666667 601 41.09816972 602 41.02990033 603 41.12769486 604 41.05960265 605 40.99173554 606 40.92409241 607 41.02141680 608 40.95394737 609 40.88669951 610 40.81967213 611 40.91653028 612 40.84967320 613 40.94616639 614 41.04234528 615 40.97560976 616 41.07142857 617 41.16693679 618 41.10032362 619 41.03392569 620 40.96774194 621 40.90177134 622 40.99678457 623 40.93097913 624 40.86538462 625 40.80000000 626 40.89456869 627 40.82934609 628 40.76433121 629 40.85850556 630 40.95238095 631 40.88748019 632 40.98101266 633 40.91627172 634 40.85173502 635 40.78740157 636 40.72327044 637 40.65934066 638 40.59561129 639 40.68857590 640 40.78125000 641 40.71762871 642 40.65420561 643 40.59097978 644 40.68322981 645 40.62015504 646 40.55727554 647 40.49459042 648 40.43209877 649 40.36979969 650 40.30769231 651 40.24577573 652 40.18404908 653 40.12251149 654 40.21406728 655 40.15267176 656 40.24390244 657 40.18264840 658 40.27355623 659 40.21244310 660 40.15151515 661 40.09077156 662 40.03021148 663 39.96983409 664 39.90963855 665 40.00000000 666 39.93993994 667 40.02998501 668 40.11976048 669 40.20926756 670 40.29850746 671 40.23845007 672 40.17857143 673 40.11887073 674 40.05934718 675 40.00000000 676 39.94082840 677 39.88183161 678 39.97050147 679 39.91163476 680 40.00000000 681 40.08810573 682 40.02932551 683 39.97071742 684 40.05847953 685 40.00000000 686 40.08746356 687 40.17467249 688 40.11627907 689 40.05805515 690 40.00000000 691 39.94211288 692 39.88439306 693 39.82683983 694 39.91354467 695 39.85611511 696 39.79885057 697 39.74175036 698 39.82808023 699 39.91416309 700 39.85714286 701 39.94293866 702 40.02849003 703 40.11379801 704 40.05681818 705 40.00000000 706 39.94334278 707 39.88684583 708 39.83050847 709 39.77433004 710 39.71830986 711 39.80309423 712 39.88764045 713 39.97194951 714 40.05602241 715 40.00000000 716 39.94413408 717 39.88842399 718 39.83286908 719 39.77746871 720 39.72222222 721 39.66712899 722 39.61218837 723 39.69571231 724 39.64088398 725 39.72413793 726 39.66942149 727 39.61485557 728 39.69780220 729 39.64334705 730 39.58904110 731 39.53488372 732 39.48087432 733 39.56343793 734 39.64577657 735 39.72789116 736 39.67391304 737 39.75576662 738 39.83739837 739 39.91880920 740 39.86486486 741 39.81106613 742 39.89218329 743 39.97308210 744 40.05376344 745 40.00000000 746 39.94638070 747 40.02677376 748 39.97326203 749 39.91989319 750 40.00000000 Final result: 40.0000 +/- 1.7900 Random chance: 25.0000 +/- 1.5822