common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 69.23076923 27 70.37037037 28 67.85714286 29 68.96551724 30 66.66666667 31 64.51612903 32 65.62500000 33 63.63636364 34 61.76470588 35 62.85714286 36 61.11111111 37 62.16216216 38 60.52631579 39 61.53846154 40 60.00000000 41 60.97560976 42 59.52380952 43 58.13953488 44 56.81818182 45 57.77777778 46 58.69565217 47 59.57446809 48 60.41666667 49 61.22448980 50 60.00000000 51 58.82352941 52 59.61538462 53 58.49056604 54 57.40740741 55 56.36363636 56 57.14285714 57 57.89473684 58 58.62068966 59 59.32203390 60 60.00000000 61 60.65573770 62 59.67741935 63 60.31746032 64 60.93750000 65 60.00000000 66 60.60606061 67 61.19402985 68 61.76470588 69 62.31884058 70 61.42857143 71 60.56338028 72 59.72222222 73 60.27397260 74 60.81081081 75 61.33333333 76 61.84210526 77 62.33766234 78 62.82051282 79 62.02531646 80 62.50000000 81 61.72839506 82 62.19512195 83 61.44578313 84 61.90476190 85 62.35294118 86 62.79069767 87 63.21839080 88 63.63636364 89 64.04494382 90 64.44444444 91 64.83516484 92 65.21739130 93 64.51612903 94 63.82978723 95 64.21052632 96 64.58333333 97 63.91752577 98 63.26530612 99 63.63636364 100 64.00000000 101 63.36633663 102 63.72549020 103 63.10679612 104 63.46153846 105 63.80952381 106 63.20754717 107 63.55140187 108 62.96296296 109 62.38532110 110 62.72727273 111 63.06306306 112 63.39285714 113 62.83185841 114 63.15789474 115 63.47826087 116 63.79310345 117 63.24786325 118 63.55932203 119 63.02521008 120 63.33333333 121 62.80991736 122 63.11475410 123 63.41463415 124 62.90322581 125 63.20000000 126 63.49206349 127 63.77952756 128 64.06250000 129 63.56589147 130 63.07692308 131 63.35877863 132 63.63636364 133 63.90977444 134 63.43283582 135 62.96296296 136 62.50000000 137 62.77372263 138 63.04347826 139 63.30935252 140 63.57142857 141 63.12056738 142 63.38028169 143 63.63636364 144 63.19444444 145 63.44827586 146 63.01369863 147 63.26530612 148 63.51351351 149 63.75838926 150 64.00000000 151 63.57615894 152 63.81578947 153 64.05228758 154 63.63636364 155 63.87096774 156 64.10256410 157 64.33121019 158 63.92405063 159 64.15094340 160 63.75000000 161 63.97515528 162 64.19753086 163 64.41717791 164 64.63414634 165 64.84848485 166 65.06024096 167 64.67065868 168 64.88095238 169 64.49704142 170 64.70588235 171 64.91228070 172 64.53488372 173 64.73988439 174 64.94252874 175 64.57142857 176 64.77272727 177 64.97175141 178 65.16853933 179 65.36312849 180 65.00000000 181 64.64088398 182 64.83516484 183 64.48087432 184 64.13043478 185 64.32432432 186 64.51612903 187 64.17112299 188 63.82978723 189 63.49206349 190 63.15789474 191 63.35078534 192 63.54166667 193 63.73056995 194 63.91752577 195 64.10256410 196 64.28571429 197 64.46700508 198 64.64646465 199 64.32160804 200 64.50000000 201 64.67661692 202 64.35643564 203 64.53201970 204 64.70588235 205 64.39024390 206 64.07766990 207 63.76811594 208 63.94230769 209 64.11483254 210 64.28571429 211 64.45497630 212 64.15094340 213 64.31924883 214 64.48598131 215 64.65116279 216 64.35185185 217 64.51612903 218 64.22018349 219 64.38356164 220 64.09090909 221 64.25339367 222 63.96396396 223 63.67713004 224 63.83928571 225 63.55555556 226 63.27433628 227 63.43612335 228 63.59649123 229 63.75545852 230 63.91304348 231 63.63636364 232 63.36206897 233 63.51931330 234 63.24786325 235 62.97872340 236 62.71186441 237 62.44725738 238 62.60504202 239 62.76150628 240 62.91666667 241 63.07053942 242 63.22314050 243 62.96296296 244 62.70491803 245 62.85714286 246 63.00813008 247 63.15789474 248 63.30645161 249 63.05220884 250 63.20000000 251 63.34661355 252 63.49206349 253 63.63636364 254 63.38582677 255 63.52941176 256 63.67187500 257 63.42412451 258 63.56589147 259 63.32046332 260 63.46153846 261 63.60153257 262 63.74045802 263 63.87832700 264 64.01515152 265 63.77358491 266 63.53383459 267 63.67041199 268 63.80597015 269 63.94052045 270 64.07407407 271 64.20664207 272 63.97058824 273 63.73626374 274 63.50364964 275 63.27272727 276 63.40579710 277 63.53790614 278 63.30935252 279 63.44086022 280 63.57142857 281 63.70106762 282 63.82978723 283 63.60424028 284 63.73239437 285 63.85964912 286 63.98601399 287 63.76306620 288 63.88888889 289 63.66782007 290 63.79310345 291 63.91752577 292 64.04109589 293 64.16382253 294 63.94557823 295 64.06779661 296 64.18918919 297 64.30976431 298 64.42953020 299 64.54849498 300 64.66666667 301 64.45182724 302 64.56953642 303 64.68646865 304 64.47368421 305 64.59016393 306 64.37908497 307 64.49511401 308 64.61038961 309 64.72491909 310 64.83870968 311 64.95176849 312 65.06410256 313 65.17571885 314 64.96815287 315 64.76190476 316 64.87341772 317 64.98422713 318 64.77987421 319 64.57680251 320 64.37500000 321 64.48598131 322 64.59627329 323 64.70588235 324 64.50617284 325 64.61538462 326 64.72392638 327 64.52599388 328 64.63414634 329 64.74164134 330 64.54545455 331 64.65256798 332 64.75903614 333 64.86486486 334 64.97005988 335 64.77611940 336 64.88095238 337 64.98516320 338 64.79289941 339 64.89675516 340 65.00000000 341 65.10263930 342 64.91228070 343 65.01457726 344 65.11627907 345 64.92753623 346 65.02890173 347 64.84149856 348 64.65517241 349 64.75644699 350 64.57142857 351 64.67236467 352 64.77272727 353 64.87252125 354 64.68926554 355 64.50704225 356 64.60674157 357 64.70588235 358 64.80446927 359 64.62395543 360 64.72222222 361 64.81994460 362 64.91712707 363 64.73829201 364 64.56043956 365 64.65753425 366 64.48087432 367 64.30517711 368 64.40217391 369 64.49864499 370 64.59459459 371 64.69002695 372 64.51612903 373 64.61126005 374 64.70588235 375 64.53333333 376 64.36170213 377 64.19098143 378 64.28571429 379 64.11609499 380 64.21052632 381 64.04199475 382 63.87434555 383 63.70757180 384 63.80208333 385 63.89610390 386 63.98963731 387 63.82428941 388 63.65979381 389 63.75321337 390 63.58974359 391 63.68286445 392 63.77551020 393 63.86768448 394 63.95939086 395 64.05063291 396 63.88888889 397 63.97984887 398 63.81909548 399 63.90977444 400 64.00000000 401 63.84039900 402 63.68159204 403 63.77171216 404 63.86138614 405 63.95061728 406 64.03940887 407 64.12776413 408 64.21568627 409 64.05867971 410 64.14634146 411 64.23357664 412 64.32038835 413 64.16464891 414 64.25120773 415 64.09638554 416 64.18269231 417 64.02877698 418 64.11483254 419 63.96181384 420 64.04761905 421 63.89548694 422 63.98104265 423 64.06619385 424 63.91509434 425 64.00000000 426 63.84976526 427 63.93442623 428 63.78504673 429 63.86946387 430 63.72093023 431 63.80510441 432 63.88888889 433 63.74133949 434 63.82488479 435 63.90804598 436 63.99082569 437 64.07322654 438 63.92694064 439 64.00911162 440 63.86363636 441 63.71882086 442 63.57466063 443 63.43115124 444 63.28828829 445 63.37078652 446 63.45291480 447 63.53467562 448 63.61607143 449 63.69710468 450 63.55555556 451 63.41463415 452 63.49557522 453 63.35540839 454 63.43612335 455 63.51648352 456 63.59649123 457 63.67614880 458 63.75545852 459 63.83442266 460 63.69565217 461 63.77440347 462 63.85281385 463 63.71490281 464 63.57758621 465 63.65591398 466 63.73390558 467 63.81156317 468 63.88888889 469 63.75266525 470 63.82978723 471 63.90658174 472 63.77118644 473 63.63636364 474 63.71308017 475 63.78947368 476 63.86554622 477 63.94129979 478 64.01673640 479 64.09185804 480 64.16666667 481 64.03326403 482 64.10788382 483 64.18219462 484 64.25619835 485 64.32989691 486 64.40329218 487 64.27104723 488 64.34426230 489 64.21267894 490 64.08163265 491 63.95112016 492 64.02439024 493 63.89452333 494 63.96761134 495 63.83838384 496 63.91129032 497 63.78269618 498 63.85542169 499 63.92785571 500 64.00000000 501 63.87225549 502 63.94422311 503 64.01590457 504 64.08730159 505 64.15841584 506 64.03162055 507 63.90532544 508 63.97637795 509 63.85068762 510 63.72549020 511 63.60078278 512 63.67187500 513 63.54775828 514 63.61867704 515 63.68932039 516 63.75968992 517 63.82978723 518 63.89961390 519 63.96917148 520 63.84615385 521 63.91554702 522 63.98467433 523 63.86233270 524 63.74045802 525 63.61904762 526 63.49809886 527 63.37760911 528 63.44696970 529 63.32703214 530 63.39622642 531 63.27683616 532 63.34586466 533 63.41463415 534 63.29588015 535 63.17757009 536 63.24626866 537 63.31471136 538 63.38289963 539 63.45083488 540 63.33333333 541 63.40110906 542 63.46863469 543 63.35174954 544 63.41911765 545 63.48623853 546 63.36996337 547 63.25411335 548 63.32116788 549 63.38797814 550 63.45454545 551 63.33938294 552 63.22463768 553 63.29113924 554 63.35740072 555 63.42342342 556 63.30935252 557 63.37522442 558 63.26164875 559 63.32737030 560 63.39285714 561 63.45811052 562 63.52313167 563 63.58792185 564 63.65248227 565 63.53982301 566 63.60424028 567 63.49206349 568 63.55633803 569 63.44463972 570 63.33333333 571 63.39754816 572 63.28671329 573 63.35078534 574 63.41463415 575 63.30434783 576 63.36805556 577 63.43154246 578 63.49480969 579 63.55785838 580 63.44827586 581 63.33907057 582 63.23024055 583 63.29331046 584 63.35616438 585 63.41880342 586 63.48122867 587 63.54344123 588 63.60544218 589 63.66723260 590 63.72881356 591 63.79018613 592 63.68243243 593 63.57504216 594 63.63636364 595 63.52941176 596 63.42281879 597 63.31658291 598 63.21070234 599 63.27212020 600 63.16666667 601 63.22795341 602 63.12292359 603 63.01824212 604 62.91390728 605 62.97520661 606 62.87128713 607 62.93245470 608 62.99342105 609 62.88998358 610 62.95081967 611 62.84779051 612 62.74509804 613 62.64274062 614 62.70358306 615 62.60162602 616 62.66233766 617 62.72285251 618 62.78317152 619 62.68174475 620 62.74193548 621 62.80193237 622 62.70096463 623 62.60032103 624 62.50000000 625 62.40000000 626 62.30031949 627 62.36044657 628 62.42038217 629 62.32114467 630 62.38095238 631 62.44057052 632 62.34177215 633 62.24328594 634 62.30283912 635 62.36220472 636 62.42138365 637 62.32339089 638 62.38244514 639 62.44131455 640 62.34375000 641 62.40249610 642 62.30529595 643 62.20839813 644 62.26708075 645 62.17054264 646 62.22910217 647 62.28748068 648 62.19135802 649 62.09553159 650 62.15384615 651 62.21198157 652 62.11656442 653 62.02143951 654 62.07951070 655 61.98473282 656 61.89024390 657 61.79604262 658 61.70212766 659 61.60849772 660 61.66666667 661 61.57337368 662 61.63141994 663 61.68929110 664 61.74698795 665 61.80451128 666 61.71171171 667 61.76911544 668 61.67664671 669 61.58445441 670 61.64179104 671 61.54992548 672 61.60714286 673 61.51560178 674 61.57270030 675 61.48148148 676 61.39053254 677 61.44756278 678 61.50442478 679 61.56111929 680 61.61764706 681 61.52716593 682 61.43695015 683 61.34699854 684 61.40350877 685 61.45985401 686 61.37026239 687 61.28093159 688 61.33720930 689 61.24818578 690 61.30434783 691 61.36034732 692 61.41618497 693 61.47186147 694 61.52737752 695 61.43884892 696 61.35057471 697 61.26255380 698 61.17478510 699 61.23032904 700 61.28571429 701 61.34094151 702 61.25356125 703 61.30867710 704 61.36363636 705 61.41843972 706 61.47308782 707 61.52758133 708 61.58192090 709 61.63610719 710 61.69014085 711 61.74402250 712 61.79775281 713 61.85133240 714 61.76470588 715 61.67832168 716 61.73184358 717 61.64574616 718 61.55988858 719 61.47426982 720 61.52777778 721 61.58113731 722 61.63434903 723 61.68741355 724 61.60220994 725 61.65517241 726 61.57024793 727 61.48555708 728 61.53846154 729 61.59122085 730 61.64383562 731 61.69630643 732 61.74863388 733 61.80081855 734 61.85286104 735 61.90476190 736 61.95652174 737 61.87245590 738 61.92411924 739 61.84032476 740 61.75675676 741 61.80836707 742 61.72506739 743 61.77658143 744 61.69354839 745 61.74496644 746 61.79624665 747 61.71352075 748 61.63101604 749 61.54873164 750 61.60000000 Final result: 61.6000 +/- 1.7771 Random chance: 25.0083 +/- 1.5824