| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 50.00000000 | |
| 13 46.15384615 | |
| 14 42.85714286 | |
| 15 46.66666667 | |
| 16 50.00000000 | |
| 17 52.94117647 | |
| 18 50.00000000 | |
| 19 52.63157895 | |
| 20 55.00000000 | |
| 21 57.14285714 | |
| 22 54.54545455 | |
| 23 52.17391304 | |
| 24 54.16666667 | |
| 25 56.00000000 | |
| 26 53.84615385 | |
| 27 55.55555556 | |
| 28 57.14285714 | |
| 29 58.62068966 | |
| 30 60.00000000 | |
| 31 58.06451613 | |
| 32 59.37500000 | |
| 33 57.57575758 | |
| 34 58.82352941 | |
| 35 60.00000000 | |
| 36 61.11111111 | |
| 37 62.16216216 | |
| 38 63.15789474 | |
| 39 61.53846154 | |
| 40 60.00000000 | |
| 41 58.53658537 | |
| 42 57.14285714 | |
| 43 58.13953488 | |
| 44 56.81818182 | |
| 45 57.77777778 | |
| 46 56.52173913 | |
| 47 55.31914894 | |
| 48 56.25000000 | |
| 49 57.14285714 | |
| 50 56.00000000 | |
| 51 54.90196078 | |
| 52 53.84615385 | |
| 53 52.83018868 | |
| 54 51.85185185 | |
| 55 52.72727273 | |
| 56 53.57142857 | |
| 57 54.38596491 | |
| 58 53.44827586 | |
| 59 52.54237288 | |
| 60 51.66666667 | |
| 61 52.45901639 | |
| 62 51.61290323 | |
| 63 50.79365079 | |
| 64 50.00000000 | |
| 65 49.23076923 | |
| 66 48.48484848 | |
| 67 47.76119403 | |
| 68 47.05882353 | |
| 69 46.37681159 | |
| 70 47.14285714 | |
| 71 47.88732394 | |
| 72 47.22222222 | |
| 73 46.57534247 | |
| 74 47.29729730 | |
| 75 48.00000000 | |
| 76 48.68421053 | |
| 77 49.35064935 | |
| 78 50.00000000 | |
| 79 49.36708861 | |
| 80 50.00000000 | |
| 81 50.61728395 | |
| 82 50.00000000 | |
| 83 50.60240964 | |
| 84 51.19047619 | |
| 85 50.58823529 | |
| 86 50.00000000 | |
| 87 50.57471264 | |
| 88 51.13636364 | |
| 89 50.56179775 | |
| 90 50.00000000 | |
| 91 49.45054945 | |
| 92 48.91304348 | |
| 93 48.38709677 | |
| 94 48.93617021 | |
| 95 48.42105263 | |
| 96 48.95833333 | |
| 97 49.48453608 | |
| 98 48.97959184 | |
| 99 49.49494949 | |
| 100 49.00000000 | |
| 101 48.51485149 | |
| 102 48.03921569 | |
| 103 47.57281553 | |
| 104 48.07692308 | |
| 105 48.57142857 | |
| 106 49.05660377 | |
| 107 48.59813084 | |
| 108 49.07407407 | |
| 109 49.54128440 | |
| 110 50.00000000 | |
| 111 50.45045045 | |
| 112 50.00000000 | |
| 113 49.55752212 | |
| 114 50.00000000 | |
| 115 50.43478261 | |
| 116 50.00000000 | |
| 117 49.57264957 | |
| 118 49.15254237 | |
| 119 49.57983193 | |
| 120 49.16666667 | |
| 121 49.58677686 | |
| 122 49.18032787 | |
| 123 48.78048780 | |
| 124 48.38709677 | |
| 125 48.80000000 | |
| 126 48.41269841 | |
| 127 48.03149606 | |
| 128 47.65625000 | |
| 129 47.28682171 | |
| 130 46.92307692 | |
| 131 46.56488550 | |
| 132 46.96969697 | |
| 133 46.61654135 | |
| 134 46.26865672 | |
| 135 45.92592593 | |
| 136 45.58823529 | |
| 137 45.25547445 | |
| 138 44.92753623 | |
| 139 45.32374101 | |
| 140 45.71428571 | |
| 141 46.09929078 | |
| 142 46.47887324 | |
| 143 46.85314685 | |
| 144 47.22222222 | |
| 145 47.58620690 | |
| 146 47.94520548 | |
| 147 47.61904762 | |
| 148 47.97297297 | |
| 149 47.65100671 | |
| 150 47.33333333 | |
| 151 47.01986755 | |
| 152 47.36842105 | |
| 153 47.71241830 | |
| 154 47.40259740 | |
| 155 47.09677419 | |
| 156 46.79487179 | |
| 157 47.13375796 | |
| 158 47.46835443 | |
| 159 47.16981132 | |
| 160 47.50000000 | |
| 161 47.20496894 | |
| 162 46.91358025 | |
| 163 46.62576687 | |
| 164 46.34146341 | |
| 165 46.66666667 | |
| 166 46.38554217 | |
| 167 46.70658683 | |
| 168 46.42857143 | |
| 169 46.74556213 | |
| 170 46.47058824 | |
| 171 46.78362573 | |
| 172 46.51162791 | |
| 173 46.24277457 | |
| 174 45.97701149 | |
| 175 45.71428571 | |
| 176 46.02272727 | |
| 177 45.76271186 | |
| 178 45.50561798 | |
| 179 45.81005587 | |
| 180 45.55555556 | |
| 181 45.30386740 | |
| 182 45.05494505 | |
| 183 45.35519126 | |
| 184 45.10869565 | |
| 185 45.40540541 | |
| 186 45.69892473 | |
| 187 45.45454545 | |
| 188 45.74468085 | |
| 189 46.03174603 | |
| 190 45.78947368 | |
| 191 46.07329843 | |
| 192 45.83333333 | |
| 193 45.59585492 | |
| 194 45.87628866 | |
| 195 46.15384615 | |
| 196 46.42857143 | |
| 197 46.19289340 | |
| 198 46.46464646 | |
| 199 46.23115578 | |
| 200 46.00000000 | |
| 201 45.77114428 | |
| 202 45.54455446 | |
| 203 45.32019704 | |
| 204 45.09803922 | |
| 205 45.36585366 | |
| 206 45.14563107 | |
| 207 45.41062802 | |
| 208 45.19230769 | |
| 209 44.97607656 | |
| 210 44.76190476 | |
| 211 45.02369668 | |
| 212 44.81132075 | |
| 213 44.60093897 | |
| 214 44.39252336 | |
| 215 44.65116279 | |
| 216 44.90740741 | |
| 217 44.70046083 | |
| 218 44.95412844 | |
| 219 45.20547945 | |
| 220 45.45454545 | |
| 221 45.24886878 | |
| 222 45.04504505 | |
| 223 45.29147982 | |
| 224 45.08928571 | |
| 225 45.33333333 | |
| 226 45.13274336 | |
| 227 44.93392070 | |
| 228 45.17543860 | |
| 229 44.97816594 | |
| 230 44.78260870 | |
| 231 45.02164502 | |
| 232 45.25862069 | |
| 233 45.06437768 | |
| 234 45.29914530 | |
| 235 45.10638298 | |
| 236 44.91525424 | |
| 237 44.72573840 | |
| 238 44.53781513 | |
| 239 44.35146444 | |
| 240 44.58333333 | |
| 241 44.39834025 | |
| 242 44.62809917 | |
| 243 44.85596708 | |
| 244 44.67213115 | |
| 245 44.89795918 | |
| 246 44.71544715 | |
| 247 44.93927126 | |
| 248 44.75806452 | |
| 249 44.57831325 | |
| 250 44.40000000 | |
| 251 44.22310757 | |
| 252 44.04761905 | |
| 253 43.87351779 | |
| 254 43.70078740 | |
| 255 43.52941176 | |
| 256 43.75000000 | |
| 257 43.96887160 | |
| 258 43.79844961 | |
| 259 43.62934363 | |
| 260 43.84615385 | |
| 261 43.67816092 | |
| 262 43.51145038 | |
| 263 43.34600760 | |
| 264 43.18181818 | |
| 265 43.01886792 | |
| 266 43.23308271 | |
| 267 43.07116105 | |
| 268 42.91044776 | |
| 269 43.12267658 | |
| 270 43.33333333 | |
| 271 43.17343173 | |
| 272 43.38235294 | |
| 273 43.22344322 | |
| 274 43.06569343 | |
| 275 42.90909091 | |
| 276 43.11594203 | |
| 277 43.32129964 | |
| 278 43.16546763 | |
| 279 43.01075269 | |
| 280 42.85714286 | |
| 281 42.70462633 | |
| 282 42.55319149 | |
| 283 42.40282686 | |
| 284 42.25352113 | |
| 285 42.10526316 | |
| 286 41.95804196 | |
| 287 42.16027875 | |
| 288 42.01388889 | |
| 289 41.86851211 | |
| 290 42.06896552 | |
| 291 42.26804124 | |
| 292 42.46575342 | |
| 293 42.66211604 | |
| 294 42.51700680 | |
| 295 42.37288136 | |
| 296 42.22972973 | |
| 297 42.08754209 | |
| 298 42.28187919 | |
| 299 42.47491639 | |
| 300 42.66666667 | |
| 301 42.52491694 | |
| 302 42.71523179 | |
| 303 42.57425743 | |
| 304 42.43421053 | |
| 305 42.29508197 | |
| 306 42.15686275 | |
| 307 42.34527687 | |
| 308 42.20779221 | |
| 309 42.07119741 | |
| 310 41.93548387 | |
| 311 42.12218650 | |
| 312 42.30769231 | |
| 313 42.17252396 | |
| 314 42.03821656 | |
| 315 41.90476190 | |
| 316 41.77215190 | |
| 317 41.95583596 | |
| 318 41.82389937 | |
| 319 42.00626959 | |
| 320 41.87500000 | |
| 321 41.74454829 | |
| 322 41.92546584 | |
| 323 41.79566563 | |
| 324 41.66666667 | |
| 325 41.84615385 | |
| 326 41.71779141 | |
| 327 41.89602446 | |
| 328 41.76829268 | |
| 329 41.64133739 | |
| 330 41.51515152 | |
| 331 41.38972810 | |
| 332 41.26506024 | |
| 333 41.44144144 | |
| 334 41.31736527 | |
| 335 41.49253731 | |
| 336 41.66666667 | |
| 337 41.54302671 | |
| 338 41.42011834 | |
| 339 41.29793510 | |
| 340 41.47058824 | |
| 341 41.64222874 | |
| 342 41.81286550 | |
| 343 41.69096210 | |
| 344 41.56976744 | |
| 345 41.44927536 | |
| 346 41.61849711 | |
| 347 41.78674352 | |
| 348 41.66666667 | |
| 349 41.54727794 | |
| 350 41.71428571 | |
| 351 41.59544160 | |
| 352 41.47727273 | |
| 353 41.64305949 | |
| 354 41.52542373 | |
| 355 41.40845070 | |
| 356 41.57303371 | |
| 357 41.45658263 | |
| 358 41.34078212 | |
| 359 41.22562674 | |
| 360 41.38888889 | |
| 361 41.55124654 | |
| 362 41.71270718 | |
| 363 41.59779614 | |
| 364 41.48351648 | |
| 365 41.36986301 | |
| 366 41.53005464 | |
| 367 41.41689373 | |
| 368 41.30434783 | |
| 369 41.19241192 | |
| 370 41.35135135 | |
| 371 41.23989218 | |
| 372 41.12903226 | |
| 373 41.01876676 | |
| 374 40.90909091 | |
| 375 41.06666667 | |
| 376 40.95744681 | |
| 377 41.11405836 | |
| 378 41.26984127 | |
| 379 41.16094987 | |
| 380 41.05263158 | |
| 381 41.20734908 | |
| 382 41.36125654 | |
| 383 41.51436031 | |
| 384 41.40625000 | |
| 385 41.55844156 | |
| 386 41.45077720 | |
| 387 41.34366925 | |
| 388 41.23711340 | |
| 389 41.38817481 | |
| 390 41.53846154 | |
| 391 41.68797954 | |
| 392 41.83673469 | |
| 393 41.73027990 | |
| 394 41.62436548 | |
| 395 41.77215190 | |
| 396 41.66666667 | |
| 397 41.81360202 | |
| 398 41.70854271 | |
| 399 41.85463659 | |
| 400 41.75000000 | |
| 401 41.89526185 | |
| 402 42.03980100 | |
| 403 41.93548387 | |
| 404 42.07920792 | |
| 405 42.22222222 | |
| 406 42.11822660 | |
| 407 42.01474201 | |
| 408 41.91176471 | |
| 409 42.05378973 | |
| 410 41.95121951 | |
| 411 41.84914842 | |
| 412 41.74757282 | |
| 413 41.64648910 | |
| 414 41.54589372 | |
| 415 41.44578313 | |
| 416 41.58653846 | |
| 417 41.48681055 | |
| 418 41.38755981 | |
| 419 41.52744630 | |
| 420 41.42857143 | |
| 421 41.56769596 | |
| 422 41.46919431 | |
| 423 41.37115839 | |
| 424 41.50943396 | |
| 425 41.64705882 | |
| 426 41.78403756 | |
| 427 41.92037471 | |
| 428 41.82242991 | |
| 429 41.95804196 | |
| 430 41.86046512 | |
| 431 41.99535963 | |
| 432 42.12962963 | |
| 433 42.03233256 | |
| 434 41.93548387 | |
| 435 41.83908046 | |
| 436 41.74311927 | |
| 437 41.64759725 | |
| 438 41.78082192 | |
| 439 41.91343964 | |
| 440 41.81818182 | |
| 441 41.72335601 | |
| 442 41.62895928 | |
| 443 41.53498871 | |
| 444 41.44144144 | |
| 445 41.34831461 | |
| 446 41.25560538 | |
| 447 41.16331096 | |
| 448 41.07142857 | |
| 449 40.97995546 | |
| 450 41.11111111 | |
| 451 41.24168514 | |
| 452 41.15044248 | |
| 453 41.28035320 | |
| 454 41.18942731 | |
| 455 41.09890110 | |
| 456 41.22807018 | |
| 457 41.35667396 | |
| 458 41.26637555 | |
| 459 41.17647059 | |
| 460 41.30434783 | |
| 461 41.21475054 | |
| 462 41.12554113 | |
| 463 41.03671706 | |
| 464 41.16379310 | |
| 465 41.07526882 | |
| 466 40.98712446 | |
| 467 41.11349036 | |
| 468 41.23931624 | |
| 469 41.36460554 | |
| 470 41.27659574 | |
| 471 41.18895966 | |
| 472 41.10169492 | |
| 473 41.22621564 | |
| 474 41.13924051 | |
| 475 41.05263158 | |
| 476 41.17647059 | |
| 477 41.09014675 | |
| 478 41.21338912 | |
| 479 41.12734864 | |
| 480 41.25000000 | |
| 481 41.16424116 | |
| 482 41.07883817 | |
| 483 40.99378882 | |
| 484 40.90909091 | |
| 485 40.82474227 | |
| 486 40.94650206 | |
| 487 41.06776181 | |
| 488 40.98360656 | |
| 489 40.89979550 | |
| 490 41.02040816 | |
| 491 40.93686354 | |
| 492 40.85365854 | |
| 493 40.77079108 | |
| 494 40.89068826 | |
| 495 41.01010101 | |
| 496 41.12903226 | |
| 497 41.04627767 | |
| 498 40.96385542 | |
| 499 40.88176353 | |
| 500 41.00000000 | |
| 501 40.91816367 | |
| 502 40.83665339 | |
| 503 40.75546720 | |
| 504 40.67460317 | |
| 505 40.59405941 | |
| 506 40.71146245 | |
| 507 40.63116371 | |
| 508 40.74803150 | |
| 509 40.66797642 | |
| 510 40.78431373 | |
| 511 40.70450098 | |
| 512 40.62500000 | |
| 513 40.54580897 | |
| 514 40.46692607 | |
| 515 40.38834951 | |
| 516 40.50387597 | |
| 517 40.61895551 | |
| 518 40.73359073 | |
| 519 40.84778420 | |
| 520 40.76923077 | |
| 521 40.69097889 | |
| 522 40.80459770 | |
| 523 40.72657744 | |
| 524 40.83969466 | |
| 525 40.76190476 | |
| 526 40.68441065 | |
| 527 40.60721063 | |
| 528 40.53030303 | |
| 529 40.45368620 | |
| 530 40.56603774 | |
| 531 40.48964218 | |
| 532 40.41353383 | |
| 533 40.33771107 | |
| 534 40.44943820 | |
| 535 40.56074766 | |
| 536 40.48507463 | |
| 537 40.59590317 | |
| 538 40.70631970 | |
| 539 40.63079777 | |
| 540 40.55555556 | |
| 541 40.48059150 | |
| 542 40.59040590 | |
| 543 40.51565378 | |
| 544 40.62500000 | |
| 545 40.55045872 | |
| 546 40.47619048 | |
| 547 40.40219378 | |
| 548 40.51094891 | |
| 549 40.43715847 | |
| 550 40.36363636 | |
| 551 40.47186933 | |
| 552 40.39855072 | |
| 553 40.32549729 | |
| 554 40.43321300 | |
| 555 40.36036036 | |
| 556 40.46762590 | |
| 557 40.57450628 | |
| 558 40.68100358 | |
| 559 40.60822898 | |
| 560 40.53571429 | |
| 561 40.46345811 | |
| 562 40.56939502 | |
| 563 40.67495560 | |
| 564 40.60283688 | |
| 565 40.70796460 | |
| 566 40.63604240 | |
| 567 40.56437390 | |
| 568 40.49295775 | |
| 569 40.42179262 | |
| 570 40.35087719 | |
| 571 40.28021016 | |
| 572 40.38461538 | |
| 573 40.31413613 | |
| 574 40.24390244 | |
| 575 40.17391304 | |
| 576 40.27777778 | |
| 577 40.38128250 | |
| 578 40.48442907 | |
| 579 40.41450777 | |
| 580 40.51724138 | |
| 581 40.61962134 | |
| 582 40.54982818 | |
| 583 40.65180103 | |
| 584 40.75342466 | |
| 585 40.68376068 | |
| 586 40.78498294 | |
| 587 40.71550256 | |
| 588 40.64625850 | |
| 589 40.57724958 | |
| 590 40.50847458 | |
| 591 40.43993232 | |
| 592 40.37162162 | |
| 593 40.47217538 | |
| 594 40.57239057 | |
| 595 40.50420168 | |
| 596 40.60402685 | |
| 597 40.53601340 | |
| 598 40.46822742 | |
| 599 40.56761269 | |
| 600 40.50000000 | |
| 601 40.43261231 | |
| 602 40.36544850 | |
| 603 40.46434494 | |
| 604 40.39735099 | |
| 605 40.33057851 | |
| 606 40.26402640 | |
| 607 40.36243822 | |
| 608 40.29605263 | |
| 609 40.22988506 | |
| 610 40.16393443 | |
| 611 40.26186579 | |
| 612 40.19607843 | |
| 613 40.29363785 | |
| 614 40.39087948 | |
| 615 40.32520325 | |
| 616 40.42207792 | |
| 617 40.51863857 | |
| 618 40.45307443 | |
| 619 40.38772213 | |
| 620 40.32258065 | |
| 621 40.25764895 | |
| 622 40.19292605 | |
| 623 40.12841091 | |
| 624 40.06410256 | |
| 625 40.00000000 | |
| 626 40.09584665 | |
| 627 40.03189793 | |
| 628 39.96815287 | |
| 629 40.06359300 | |
| 630 40.15873016 | |
| 631 40.09508716 | |
| 632 40.18987342 | |
| 633 40.12638231 | |
| 634 40.06309148 | |
| 635 40.00000000 | |
| 636 39.93710692 | |
| 637 39.87441130 | |
| 638 39.81191223 | |
| 639 39.90610329 | |
| 640 40.00000000 | |
| 641 39.93759750 | |
| 642 39.87538941 | |
| 643 39.81337481 | |
| 644 39.90683230 | |
| 645 39.84496124 | |
| 646 39.78328173 | |
| 647 39.72179289 | |
| 648 39.66049383 | |
| 649 39.59938367 | |
| 650 39.53846154 | |
| 651 39.47772657 | |
| 652 39.41717791 | |
| 653 39.35681470 | |
| 654 39.44954128 | |
| 655 39.38931298 | |
| 656 39.48170732 | |
| 657 39.42161339 | |
| 658 39.51367781 | |
| 659 39.45371775 | |
| 660 39.39393939 | |
| 661 39.33434191 | |
| 662 39.27492447 | |
| 663 39.21568627 | |
| 664 39.15662651 | |
| 665 39.24812030 | |
| 666 39.18918919 | |
| 667 39.28035982 | |
| 668 39.37125749 | |
| 669 39.31240658 | |
| 670 39.40298507 | |
| 671 39.34426230 | |
| 672 39.28571429 | |
| 673 39.22734027 | |
| 674 39.16913947 | |
| 675 39.11111111 | |
| 676 39.05325444 | |
| 677 38.99556869 | |
| 678 38.93805310 | |
| 679 38.88070692 | |
| 680 38.97058824 | |
| 681 39.06020558 | |
| 682 39.00293255 | |
| 683 38.94582723 | |
| 684 39.03508772 | |
| 685 38.97810219 | |
| 686 39.06705539 | |
| 687 39.15574964 | |
| 688 39.09883721 | |
| 689 39.04208999 | |
| 690 38.98550725 | |
| 691 39.07380608 | |
| 692 39.01734104 | |
| 693 38.96103896 | |
| 694 39.04899135 | |
| 695 38.99280576 | |
| 696 38.93678161 | |
| 697 38.88091822 | |
| 698 38.96848138 | |
| 699 39.05579399 | |
| 700 39.00000000 | |
| 701 39.08701854 | |
| 702 39.17378917 | |
| 703 39.26031294 | |
| 704 39.34659091 | |
| 705 39.29078014 | |
| 706 39.23512748 | |
| 707 39.17963225 | |
| 708 39.12429379 | |
| 709 39.06911142 | |
| 710 39.01408451 | |
| 711 39.09985935 | |
| 712 39.18539326 | |
| 713 39.27068724 | |
| 714 39.35574230 | |
| 715 39.30069930 | |
| 716 39.24581006 | |
| 717 39.19107392 | |
| 718 39.13649025 | |
| 719 39.08205841 | |
| 720 39.02777778 | |
| 721 38.97364771 | |
| 722 38.91966759 | |
| 723 39.00414938 | |
| 724 38.95027624 | |
| 725 39.03448276 | |
| 726 39.11845730 | |
| 727 39.06464924 | |
| 728 39.14835165 | |
| 729 39.09465021 | |
| 730 39.04109589 | |
| 731 38.98768810 | |
| 732 38.93442623 | |
| 733 39.01773533 | |
| 734 39.10081744 | |
| 735 39.18367347 | |
| 736 39.13043478 | |
| 737 39.21302578 | |
| 738 39.29539295 | |
| 739 39.37753721 | |
| 740 39.32432432 | |
| 741 39.27125506 | |
| 742 39.35309973 | |
| 743 39.43472409 | |
| 744 39.51612903 | |
| 745 39.46308725 | |
| 746 39.41018767 | |
| 747 39.35742972 | |
| 748 39.30481283 | |
| 749 39.25233645 | |
| 750 39.33333333 | |
| Final result: 39.3333 +/- 1.7849 | |
| Random chance: 25.0000 +/- 1.5822 | |