common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 23.07692308 14 21.42857143 15 20.00000000 16 25.00000000 17 23.52941176 18 22.22222222 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 19.23076923 27 18.51851852 28 17.85714286 29 20.68965517 30 20.00000000 31 22.58064516 32 25.00000000 33 27.27272727 34 26.47058824 35 25.71428571 36 25.00000000 37 27.02702703 38 26.31578947 39 28.20512821 40 27.50000000 41 26.82926829 42 26.19047619 43 27.90697674 44 27.27272727 45 28.88888889 46 28.26086957 47 29.78723404 48 31.25000000 49 30.61224490 50 32.00000000 51 31.37254902 52 30.76923077 53 30.18867925 54 29.62962963 55 30.90909091 56 30.35714286 57 29.82456140 58 29.31034483 59 30.50847458 60 31.66666667 61 31.14754098 62 32.25806452 63 31.74603175 64 31.25000000 65 30.76923077 66 30.30303030 67 29.85074627 68 29.41176471 69 30.43478261 70 30.00000000 71 29.57746479 72 30.55555556 73 30.13698630 74 29.72972973 75 29.33333333 76 28.94736842 77 28.57142857 78 28.20512821 79 27.84810127 80 28.75000000 81 28.39506173 82 29.26829268 83 28.91566265 84 28.57142857 85 28.23529412 86 27.90697674 87 27.58620690 88 27.27272727 89 28.08988764 90 27.77777778 91 27.47252747 92 27.17391304 93 26.88172043 94 26.59574468 95 27.36842105 96 28.12500000 97 28.86597938 98 28.57142857 99 29.29292929 100 30.00000000 101 30.69306931 102 30.39215686 103 30.09708738 104 29.80769231 105 29.52380952 106 29.24528302 107 29.90654206 108 30.55555556 109 30.27522936 110 30.90909091 111 30.63063063 112 31.25000000 113 31.85840708 114 32.45614035 115 32.17391304 116 31.89655172 117 31.62393162 118 31.35593220 119 31.09243697 120 31.66666667 121 32.23140496 122 31.96721311 123 31.70731707 124 32.25806452 125 32.80000000 126 32.53968254 127 32.28346457 128 32.03125000 129 32.55813953 130 33.07692308 131 32.82442748 132 32.57575758 133 32.33082707 134 32.08955224 135 32.59259259 136 32.35294118 137 32.84671533 138 33.33333333 139 33.81294964 140 33.57142857 141 33.33333333 142 33.09859155 143 32.86713287 144 33.33333333 145 33.10344828 146 32.87671233 147 33.33333333 148 33.10810811 149 32.88590604 150 32.66666667 151 33.11258278 152 32.89473684 153 32.67973856 154 32.46753247 155 32.90322581 156 33.33333333 157 33.12101911 158 33.54430380 159 33.33333333 160 33.12500000 161 32.91925466 162 32.71604938 163 32.51533742 164 32.31707317 165 32.72727273 166 32.53012048 167 32.33532934 168 32.73809524 169 32.54437870 170 32.35294118 171 32.16374269 172 31.97674419 173 31.79190751 174 32.18390805 175 32.00000000 176 32.38636364 177 32.20338983 178 32.02247191 179 32.40223464 180 32.77777778 181 32.59668508 182 32.41758242 183 32.24043716 184 32.06521739 185 31.89189189 186 32.25806452 187 32.08556150 188 31.91489362 189 31.74603175 190 31.57894737 191 31.41361257 192 31.25000000 193 31.08808290 194 31.44329897 195 31.28205128 196 31.12244898 197 31.47208122 198 31.81818182 199 31.65829146 200 31.50000000 201 31.34328358 202 31.18811881 203 31.03448276 204 30.88235294 205 31.21951220 206 31.06796117 207 31.40096618 208 31.25000000 209 31.10047847 210 30.95238095 211 30.80568720 212 30.66037736 213 30.51643192 214 30.37383178 215 30.23255814 216 30.09259259 217 30.41474654 218 30.73394495 219 30.59360731 220 30.45454545 221 30.31674208 222 30.18018018 223 30.49327354 224 30.80357143 225 30.66666667 226 30.53097345 227 30.83700441 228 30.70175439 229 30.56768559 230 30.43478261 231 30.30303030 232 30.17241379 233 30.47210300 234 30.76923077 235 30.63829787 236 30.93220339 237 31.22362869 238 31.51260504 239 31.38075314 240 31.25000000 241 31.12033195 242 30.99173554 243 30.86419753 244 30.73770492 245 30.61224490 246 30.48780488 247 30.76923077 248 30.64516129 249 30.92369478 250 30.80000000 251 31.07569721 252 30.95238095 253 30.83003953 254 30.70866142 255 30.58823529 256 30.46875000 257 30.35019455 258 30.23255814 259 30.11583012 260 30.00000000 261 30.26819923 262 30.15267176 263 30.03802281 264 30.30303030 265 30.56603774 266 30.45112782 267 30.33707865 268 30.22388060 269 30.11152416 270 30.00000000 271 29.88929889 272 29.77941176 273 30.03663004 274 29.92700730 275 29.81818182 276 29.71014493 277 29.60288809 278 29.49640288 279 29.39068100 280 29.28571429 281 29.53736655 282 29.43262411 283 29.68197880 284 29.57746479 285 29.47368421 286 29.72027972 287 29.96515679 288 29.86111111 289 29.75778547 290 29.65517241 291 29.89690722 292 29.79452055 293 30.03412969 294 29.93197279 295 30.16949153 296 30.40540541 297 30.63973064 298 30.53691275 299 30.43478261 300 30.33333333 301 30.23255814 302 30.13245033 303 30.03300330 304 29.93421053 305 29.83606557 306 29.73856209 307 29.64169381 308 29.87012987 309 29.77346278 310 29.67741935 311 29.58199357 312 29.80769231 313 29.71246006 314 29.61783439 315 29.52380952 316 29.43037975 317 29.33753943 318 29.24528302 319 29.15360502 320 29.06250000 321 28.97196262 322 29.19254658 323 29.10216718 324 29.01234568 325 28.92307692 326 28.83435583 327 29.05198777 328 29.26829268 329 29.17933131 330 29.39393939 331 29.30513595 332 29.51807229 333 29.72972973 334 29.64071856 335 29.85074627 336 30.05952381 337 29.97032641 338 29.88165680 339 30.08849558 340 30.00000000 341 30.20527859 342 30.40935673 343 30.32069971 344 30.52325581 345 30.72463768 346 30.63583815 347 30.83573487 348 30.74712644 349 30.65902579 350 30.57142857 351 30.48433048 352 30.39772727 353 30.31161473 354 30.22598870 355 30.42253521 356 30.33707865 357 30.25210084 358 30.16759777 359 30.08356546 360 30.00000000 361 29.91689751 362 29.83425414 363 30.02754821 364 29.94505495 365 29.86301370 366 29.78142077 367 29.97275204 368 29.89130435 369 29.81029810 370 29.72972973 371 29.64959569 372 29.83870968 373 29.75871314 374 29.94652406 375 29.86666667 376 29.78723404 377 29.97347480 378 29.89417989 379 29.81530343 380 30.00000000 381 29.92125984 382 29.84293194 383 29.76501305 384 29.68750000 385 29.61038961 386 29.53367876 387 29.45736434 388 29.63917526 389 29.56298201 390 29.48717949 391 29.66751918 392 29.59183673 393 29.77099237 394 29.94923858 395 29.87341772 396 30.05050505 397 30.22670025 398 30.15075377 399 30.07518797 400 30.00000000 401 30.17456359 402 30.34825871 403 30.27295285 404 30.19801980 405 30.12345679 406 30.04926108 407 29.97542998 408 29.90196078 409 29.82885086 410 29.75609756 411 29.68369830 412 29.85436893 413 29.78208232 414 29.95169082 415 30.12048193 416 30.04807692 417 30.21582734 418 30.14354067 419 30.07159905 420 30.23809524 421 30.40380048 422 30.33175355 423 30.49645390 424 30.42452830 425 30.35294118 426 30.28169014 427 30.21077283 428 30.14018692 429 30.06993007 430 30.00000000 431 29.93039443 432 30.09259259 433 30.25404157 434 30.41474654 435 30.34482759 436 30.27522936 437 30.20594966 438 30.36529680 439 30.52391800 440 30.45454545 441 30.38548753 442 30.31674208 443 30.24830700 444 30.40540541 445 30.33707865 446 30.26905830 447 30.20134228 448 30.13392857 449 30.28953229 450 30.22222222 451 30.15521064 452 30.08849558 453 30.24282561 454 30.17621145 455 30.10989011 456 30.04385965 457 29.97811816 458 29.91266376 459 29.84749455 460 29.78260870 461 29.71800434 462 29.87012987 463 30.02159827 464 30.17241379 465 30.32258065 466 30.25751073 467 30.19271949 468 30.34188034 469 30.27718550 470 30.21276596 471 30.36093418 472 30.50847458 473 30.44397463 474 30.59071730 475 30.73684211 476 30.67226891 477 30.60796646 478 30.54393305 479 30.48016701 480 30.41666667 481 30.35343035 482 30.49792531 483 30.43478261 484 30.37190083 485 30.51546392 486 30.45267490 487 30.59548255 488 30.53278689 489 30.47034765 490 30.61224490 491 30.75356415 492 30.89430894 493 30.83164300 494 30.97165992 495 30.90909091 496 30.84677419 497 30.78470825 498 30.72289157 499 30.66132265 500 30.60000000 501 30.73852295 502 30.67729084 503 30.61630219 504 30.75396825 505 30.69306931 506 30.63241107 507 30.57199211 508 30.70866142 509 30.84479371 510 30.98039216 511 30.91976517 512 30.85937500 513 30.79922027 514 30.73929961 515 30.87378641 516 30.81395349 517 30.94777563 518 30.88803089 519 30.82851638 520 30.76923077 521 30.71017274 522 30.65134100 523 30.59273423 524 30.53435115 525 30.47619048 526 30.41825095 527 30.55028463 528 30.49242424 529 30.62381853 530 30.56603774 531 30.50847458 532 30.45112782 533 30.58161351 534 30.52434457 535 30.46728972 536 30.41044776 537 30.54003724 538 30.48327138 539 30.42671614 540 30.55555556 541 30.49907579 542 30.44280443 543 30.38674033 544 30.51470588 545 30.45871560 546 30.40293040 547 30.53016453 548 30.65693431 549 30.60109290 550 30.54545455 551 30.49001815 552 30.61594203 553 30.74141049 554 30.68592058 555 30.63063063 556 30.57553957 557 30.52064632 558 30.46594982 559 30.41144902 560 30.53571429 561 30.48128342 562 30.42704626 563 30.37300178 564 30.49645390 565 30.61946903 566 30.74204947 567 30.68783069 568 30.80985915 569 30.93145870 570 30.87719298 571 30.99824869 572 30.94405594 573 30.89005236 574 31.01045296 575 31.13043478 576 31.25000000 577 31.36915078 578 31.48788927 579 31.43350604 580 31.55172414 581 31.49741824 582 31.61512027 583 31.56089194 584 31.67808219 585 31.62393162 586 31.74061433 587 31.68654174 588 31.63265306 589 31.57894737 590 31.52542373 591 31.47208122 592 31.41891892 593 31.36593592 594 31.48148148 595 31.42857143 596 31.37583893 597 31.32328308 598 31.27090301 599 31.21869783 600 31.16666667 601 31.28119800 602 31.39534884 603 31.50912106 604 31.45695364 605 31.40495868 606 31.51815182 607 31.46622735 608 31.41447368 609 31.36288998 610 31.31147541 611 31.26022913 612 31.37254902 613 31.32137031 614 31.27035831 615 31.21951220 616 31.33116883 617 31.44246353 618 31.39158576 619 31.50242326 620 31.45161290 621 31.40096618 622 31.51125402 623 31.46067416 624 31.41025641 625 31.36000000 626 31.30990415 627 31.25996810 628 31.36942675 629 31.31955485 630 31.26984127 631 31.37876387 632 31.32911392 633 31.43759874 634 31.54574132 635 31.49606299 636 31.44654088 637 31.39717425 638 31.34796238 639 31.29890454 640 31.40625000 641 31.35725429 642 31.46417445 643 31.41524106 644 31.36645963 645 31.47286822 646 31.42414861 647 31.37557960 648 31.48148148 649 31.43297381 650 31.53846154 651 31.64362519 652 31.74846626 653 31.85298622 654 31.80428135 655 31.75572519 656 31.85975610 657 31.96347032 658 31.91489362 659 32.01820941 660 31.96969697 661 31.92133132 662 31.87311178 663 31.82503771 664 31.77710843 665 31.72932331 666 31.83183183 667 31.78410795 668 31.73652695 669 31.68908819 670 31.79104478 671 31.74366617 672 31.69642857 673 31.64933135 674 31.60237389 675 31.70370370 676 31.80473373 677 31.75775480 678 31.71091445 679 31.66421208 680 31.61764706 681 31.57121880 682 31.52492669 683 31.47877013 684 31.57894737 685 31.53284672 686 31.48688047 687 31.44104803 688 31.39534884 689 31.49492017 690 31.44927536 691 31.54848046 692 31.64739884 693 31.74603175 694 31.70028818 695 31.65467626 696 31.75287356 697 31.70731707 698 31.66189112 699 31.61659514 700 31.71428571 701 31.66904422 702 31.76638177 703 31.72119488 704 31.67613636 705 31.63120567 706 31.72804533 707 31.82461103 708 31.77966102 709 31.73483780 710 31.83098592 711 31.78621660 712 31.88202247 713 31.83730715 714 31.79271709 715 31.88811189 716 31.84357542 717 31.93863319 718 32.03342618 719 32.12795549 720 32.08333333 721 32.03883495 722 31.99445983 723 31.95020747 724 32.04419890 725 32.00000000 726 32.09366391 727 32.04951857 728 32.14285714 729 32.23593964 730 32.19178082 731 32.14774282 732 32.24043716 733 32.19645293 734 32.15258856 735 32.10884354 736 32.06521739 737 32.15739484 738 32.11382114 739 32.07036536 740 32.02702703 741 32.11875843 742 32.07547170 743 32.03230148 744 32.12365591 745 32.08053691 746 32.17158177 747 32.26238286 748 32.21925134 749 32.30974633 750 32.40000000 Final result: 32.4000 +/- 1.7100 Random chance: 19.8992 +/- 1.4588