common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 68.42105263 20 65.00000000 21 66.66666667 22 68.18181818 23 65.21739130 24 66.66666667 25 64.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 65.51724138 30 63.33333333 31 61.29032258 32 62.50000000 33 60.60606061 34 58.82352941 35 60.00000000 36 58.33333333 37 59.45945946 38 57.89473684 39 58.97435897 40 60.00000000 41 60.97560976 42 59.52380952 43 58.13953488 44 56.81818182 45 57.77777778 46 58.69565217 47 59.57446809 48 60.41666667 49 61.22448980 50 60.00000000 51 58.82352941 52 59.61538462 53 58.49056604 54 57.40740741 55 56.36363636 56 57.14285714 57 57.89473684 58 58.62068966 59 59.32203390 60 60.00000000 61 60.65573770 62 59.67741935 63 60.31746032 64 60.93750000 65 60.00000000 66 60.60606061 67 61.19402985 68 61.76470588 69 60.86956522 70 60.00000000 71 59.15492958 72 58.33333333 73 58.90410959 74 59.45945946 75 60.00000000 76 60.52631579 77 61.03896104 78 61.53846154 79 60.75949367 80 61.25000000 81 60.49382716 82 60.97560976 83 60.24096386 84 60.71428571 85 61.17647059 86 60.46511628 87 59.77011494 88 60.22727273 89 60.67415730 90 60.00000000 91 60.43956044 92 60.86956522 93 60.21505376 94 59.57446809 95 60.00000000 96 60.41666667 97 59.79381443 98 60.20408163 99 60.60606061 100 61.00000000 101 60.39603960 102 60.78431373 103 60.19417476 104 60.57692308 105 60.95238095 106 60.37735849 107 60.74766355 108 60.18518519 109 59.63302752 110 60.00000000 111 60.36036036 112 60.71428571 113 61.06194690 114 61.40350877 115 61.73913043 116 61.20689655 117 60.68376068 118 61.01694915 119 60.50420168 120 60.83333333 121 60.33057851 122 60.65573770 123 60.97560976 124 60.48387097 125 60.80000000 126 61.11111111 127 61.41732283 128 61.71875000 129 61.24031008 130 60.76923077 131 61.06870229 132 61.36363636 133 61.65413534 134 61.19402985 135 60.74074074 136 60.29411765 137 60.58394161 138 60.86956522 139 61.15107914 140 61.42857143 141 60.99290780 142 61.26760563 143 61.53846154 144 61.11111111 145 61.37931034 146 60.95890411 147 61.22448980 148 61.48648649 149 61.74496644 150 62.00000000 151 62.25165563 152 62.50000000 153 62.74509804 154 62.33766234 155 62.58064516 156 62.82051282 157 63.05732484 158 62.65822785 159 62.89308176 160 62.50000000 161 62.73291925 162 62.96296296 163 62.57668712 164 62.19512195 165 62.42424242 166 62.65060241 167 62.27544910 168 62.50000000 169 62.13017751 170 62.35294118 171 62.57309942 172 62.20930233 173 62.42774566 174 62.64367816 175 62.28571429 176 62.50000000 177 62.71186441 178 62.92134831 179 63.12849162 180 63.33333333 181 62.98342541 182 63.18681319 183 62.84153005 184 63.04347826 185 63.24324324 186 63.44086022 187 63.63636364 188 63.29787234 189 62.96296296 190 62.63157895 191 62.82722513 192 63.02083333 193 63.21243523 194 63.40206186 195 63.58974359 196 63.77551020 197 63.95939086 198 64.14141414 199 63.81909548 200 64.00000000 201 64.17910448 202 63.86138614 203 64.03940887 204 64.21568627 205 64.39024390 206 64.07766990 207 63.76811594 208 63.94230769 209 64.11483254 210 64.28571429 211 64.45497630 212 64.15094340 213 64.31924883 214 64.48598131 215 64.65116279 216 64.35185185 217 64.05529954 218 63.76146789 219 63.92694064 220 63.63636364 221 63.80090498 222 63.51351351 223 63.22869955 224 63.39285714 225 63.11111111 226 62.83185841 227 62.99559471 228 63.15789474 229 63.31877729 230 63.47826087 231 63.20346320 232 62.93103448 233 63.09012876 234 63.24786325 235 62.97872340 236 62.71186441 237 62.44725738 238 62.60504202 239 62.76150628 240 62.91666667 241 63.07053942 242 63.22314050 243 63.37448560 244 63.11475410 245 63.26530612 246 63.41463415 247 63.56275304 248 63.70967742 249 63.45381526 250 63.60000000 251 63.74501992 252 63.88888889 253 64.03162055 254 63.77952756 255 63.92156863 256 64.06250000 257 63.81322957 258 63.95348837 259 63.70656371 260 63.84615385 261 63.98467433 262 64.12213740 263 64.25855513 264 64.39393939 265 64.15094340 266 63.90977444 267 64.04494382 268 64.17910448 269 64.31226766 270 64.44444444 271 64.57564576 272 64.33823529 273 64.10256410 274 63.86861314 275 63.63636364 276 63.76811594 277 63.89891697 278 63.66906475 279 63.79928315 280 63.92857143 281 64.05693950 282 64.18439716 283 63.95759717 284 64.08450704 285 64.21052632 286 64.33566434 287 64.45993031 288 64.58333333 289 64.35986159 290 64.48275862 291 64.60481100 292 64.72602740 293 64.84641638 294 64.62585034 295 64.74576271 296 64.86486486 297 64.98316498 298 65.10067114 299 65.21739130 300 65.00000000 301 64.78405316 302 64.90066225 303 65.01650165 304 65.13157895 305 65.24590164 306 65.03267974 307 64.82084691 308 64.93506494 309 65.04854369 310 64.83870968 311 64.95176849 312 64.74358974 313 64.85623003 314 64.64968153 315 64.76190476 316 64.87341772 317 64.98422713 318 64.77987421 319 64.57680251 320 64.37500000 321 64.48598131 322 64.59627329 323 64.70588235 324 64.81481481 325 64.92307692 326 65.03067485 327 64.83180428 328 64.93902439 329 65.04559271 330 64.84848485 331 64.95468278 332 65.06024096 333 65.16516517 334 65.26946108 335 65.07462687 336 65.17857143 337 65.28189911 338 65.08875740 339 65.19174041 340 65.29411765 341 65.39589443 342 65.49707602 343 65.59766764 344 65.69767442 345 65.50724638 346 65.60693642 347 65.41786744 348 65.22988506 349 65.32951289 350 65.42857143 351 65.52706553 352 65.62500000 353 65.72237960 354 65.53672316 355 65.35211268 356 65.44943820 357 65.54621849 358 65.64245810 359 65.45961003 360 65.55555556 361 65.65096953 362 65.74585635 363 65.84022039 364 65.65934066 365 65.75342466 366 65.57377049 367 65.39509537 368 65.48913043 369 65.58265583 370 65.67567568 371 65.76819407 372 65.86021505 373 65.95174263 374 66.04278075 375 66.13333333 376 65.95744681 377 65.78249337 378 65.87301587 379 65.69920844 380 65.78947368 381 65.61679790 382 65.44502618 383 65.27415144 384 65.36458333 385 65.19480519 386 65.28497409 387 65.37467700 388 65.20618557 389 65.03856041 390 64.87179487 391 64.96163683 392 65.05102041 393 65.13994911 394 65.22842640 395 65.31645570 396 65.15151515 397 65.23929471 398 65.07537688 399 65.16290727 400 65.25000000 401 65.08728180 402 65.17412935 403 65.26054591 404 65.34653465 405 65.43209877 406 65.51724138 407 65.60196560 408 65.68627451 409 65.52567237 410 65.60975610 411 65.69343066 412 65.77669903 413 65.61743341 414 65.70048309 415 65.54216867 416 65.62500000 417 65.46762590 418 65.31100478 419 65.15513126 420 65.23809524 421 65.08313539 422 65.16587678 423 65.24822695 424 65.09433962 425 65.17647059 426 65.02347418 427 65.10538642 428 64.95327103 429 65.03496503 430 64.88372093 431 64.96519722 432 65.04629630 433 64.89607390 434 64.97695853 435 65.05747126 436 65.13761468 437 65.21739130 438 65.06849315 439 65.14806378 440 65.00000000 441 64.85260771 442 64.70588235 443 64.55981941 444 64.41441441 445 64.49438202 446 64.57399103 447 64.65324385 448 64.73214286 449 64.81069042 450 64.66666667 451 64.74501109 452 64.82300885 453 64.67991170 454 64.75770925 455 64.83516484 456 64.91228070 457 64.98905908 458 65.06550218 459 65.14161220 460 65.00000000 461 65.07592191 462 65.15151515 463 65.01079914 464 64.87068966 465 64.94623656 466 65.02145923 467 65.09635974 468 65.17094017 469 65.03198294 470 65.10638298 471 65.18046709 472 65.25423729 473 65.11627907 474 65.18987342 475 65.26315789 476 65.33613445 477 65.40880503 478 65.48117155 479 65.55323591 480 65.62500000 481 65.69646570 482 65.76763485 483 65.83850932 484 65.90909091 485 65.97938144 486 66.04938272 487 65.91375770 488 65.98360656 489 66.05316973 490 65.91836735 491 65.78411405 492 65.85365854 493 65.72008114 494 65.78947368 495 65.65656566 496 65.72580645 497 65.79476861 498 65.86345382 499 65.93186373 500 66.00000000 501 65.86826347 502 65.93625498 503 66.00397614 504 66.07142857 505 66.13861386 506 66.20553360 507 66.07495069 508 65.94488189 509 65.81532417 510 65.68627451 511 65.55772994 512 65.62500000 513 65.49707602 514 65.56420233 515 65.63106796 516 65.69767442 517 65.76402321 518 65.83011583 519 65.70327553 520 65.57692308 521 65.64299424 522 65.70881226 523 65.58317400 524 65.45801527 525 65.33333333 526 65.20912548 527 65.08538899 528 64.96212121 529 64.83931947 530 64.90566038 531 64.97175141 532 65.03759398 533 65.10318949 534 64.98127341 535 64.85981308 536 64.92537313 537 64.99068901 538 65.05576208 539 65.12059369 540 65.00000000 541 64.87985213 542 64.94464945 543 64.82504604 544 64.70588235 545 64.77064220 546 64.65201465 547 64.53382084 548 64.59854015 549 64.66302368 550 64.72727273 551 64.60980036 552 64.49275362 553 64.55696203 554 64.62093863 555 64.68468468 556 64.56834532 557 64.63195691 558 64.51612903 559 64.57960644 560 64.64285714 561 64.70588235 562 64.76868327 563 64.83126110 564 64.89361702 565 64.77876106 566 64.84098940 567 64.72663139 568 64.78873239 569 64.67486819 570 64.56140351 571 64.62346760 572 64.51048951 573 64.57242583 574 64.63414634 575 64.69565217 576 64.75694444 577 64.81802426 578 64.87889273 579 64.93955095 580 64.82758621 581 64.71600688 582 64.77663230 583 64.83704974 584 64.89726027 585 64.95726496 586 65.01706485 587 65.07666099 588 65.13605442 589 65.19524618 590 65.25423729 591 65.31302876 592 65.20270270 593 65.09274874 594 65.15151515 595 65.04201681 596 64.93288591 597 64.82412060 598 64.88294314 599 64.94156928 600 64.83333333 601 64.89184692 602 64.78405316 603 64.67661692 604 64.73509934 605 64.79338843 606 64.85148515 607 64.90939044 608 64.96710526 609 64.86042693 610 64.91803279 611 64.81178396 612 64.86928105 613 64.76345840 614 64.82084691 615 64.71544715 616 64.61038961 617 64.66774716 618 64.56310680 619 64.45880452 620 64.51612903 621 64.57326892 622 64.46945338 623 64.36597111 624 64.26282051 625 64.16000000 626 64.05750799 627 64.11483254 628 64.17197452 629 64.06995231 630 64.12698413 631 64.18383518 632 64.08227848 633 63.98104265 634 64.03785489 635 64.09448819 636 64.15094340 637 64.05023548 638 64.10658307 639 64.16275430 640 64.06250000 641 63.96255850 642 63.86292835 643 63.76360809 644 63.81987578 645 63.72093023 646 63.77708978 647 63.83307573 648 63.73456790 649 63.63636364 650 63.69230769 651 63.74807988 652 63.65030675 653 63.70597243 654 63.76146789 655 63.66412214 656 63.56707317 657 63.47031963 658 63.37386018 659 63.27769347 660 63.33333333 661 63.23751891 662 63.29305136 663 63.34841629 664 63.40361446 665 63.30827068 666 63.21321321 667 63.26836582 668 63.17365269 669 63.07922272 670 62.98507463 671 62.89120715 672 62.94642857 673 62.85289747 674 62.90801187 675 62.81481481 676 62.86982249 677 62.92466765 678 62.97935103 679 63.03387334 680 63.08823529 681 62.99559471 682 62.90322581 683 62.81112738 684 62.86549708 685 62.91970803 686 62.82798834 687 62.73653566 688 62.79069767 689 62.69956459 690 62.75362319 691 62.80752533 692 62.86127168 693 62.91486291 694 62.96829971 695 62.87769784 696 62.78735632 697 62.69727403 698 62.75071633 699 62.80400572 700 62.85714286 701 62.91012839 702 62.96296296 703 63.01564723 704 63.06818182 705 63.12056738 706 63.17280453 707 63.22489392 708 63.27683616 709 63.32863188 710 63.38028169 711 63.43178622 712 63.48314607 713 63.53436185 714 63.44537815 715 63.35664336 716 63.40782123 717 63.31938633 718 63.23119777 719 63.14325452 720 63.19444444 721 63.24549237 722 63.29639889 723 63.34716459 724 63.25966851 725 63.31034483 726 63.22314050 727 63.13617607 728 63.18681319 729 63.23731139 730 63.28767123 731 63.33789330 732 63.38797814 733 63.43792633 734 63.48773842 735 63.53741497 736 63.58695652 737 63.50067843 738 63.55013550 739 63.46414073 740 63.37837838 741 63.42780027 742 63.34231806 743 63.39165545 744 63.30645161 745 63.35570470 746 63.40482574 747 63.31994645 748 63.23529412 749 63.15086782 750 63.20000000 Final result: 63.2000 +/- 1.7621 Random chance: 25.0083 +/- 1.5824