| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 12.50000000 | |
| 9 11.11111111 | |
| 10 10.00000000 | |
| 11 9.09090909 | |
| 12 16.66666667 | |
| 13 23.07692308 | |
| 14 21.42857143 | |
| 15 26.66666667 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 31.57894737 | |
| 20 30.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 26.08695652 | |
| 24 29.16666667 | |
| 25 28.00000000 | |
| 26 26.92307692 | |
| 27 25.92592593 | |
| 28 25.00000000 | |
| 29 27.58620690 | |
| 30 26.66666667 | |
| 31 25.80645161 | |
| 32 28.12500000 | |
| 33 30.30303030 | |
| 34 29.41176471 | |
| 35 28.57142857 | |
| 36 27.77777778 | |
| 37 29.72972973 | |
| 38 28.94736842 | |
| 39 30.76923077 | |
| 40 30.00000000 | |
| 41 29.26829268 | |
| 42 28.57142857 | |
| 43 30.23255814 | |
| 44 31.81818182 | |
| 45 33.33333333 | |
| 46 32.60869565 | |
| 47 34.04255319 | |
| 48 35.41666667 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 35.29411765 | |
| 52 34.61538462 | |
| 53 33.96226415 | |
| 54 33.33333333 | |
| 55 34.54545455 | |
| 56 33.92857143 | |
| 57 33.33333333 | |
| 58 32.75862069 | |
| 59 32.20338983 | |
| 60 33.33333333 | |
| 61 32.78688525 | |
| 62 33.87096774 | |
| 63 33.33333333 | |
| 64 34.37500000 | |
| 65 33.84615385 | |
| 66 34.84848485 | |
| 67 34.32835821 | |
| 68 33.82352941 | |
| 69 34.78260870 | |
| 70 34.28571429 | |
| 71 33.80281690 | |
| 72 34.72222222 | |
| 73 35.61643836 | |
| 74 35.13513514 | |
| 75 34.66666667 | |
| 76 34.21052632 | |
| 77 33.76623377 | |
| 78 33.33333333 | |
| 79 32.91139241 | |
| 80 33.75000000 | |
| 81 33.33333333 | |
| 82 34.14634146 | |
| 83 33.73493976 | |
| 84 33.33333333 | |
| 85 32.94117647 | |
| 86 32.55813953 | |
| 87 32.18390805 | |
| 88 31.81818182 | |
| 89 32.58426966 | |
| 90 32.22222222 | |
| 91 31.86813187 | |
| 92 31.52173913 | |
| 93 31.18279570 | |
| 94 30.85106383 | |
| 95 30.52631579 | |
| 96 31.25000000 | |
| 97 31.95876289 | |
| 98 31.63265306 | |
| 99 32.32323232 | |
| 100 33.00000000 | |
| 101 33.66336634 | |
| 102 33.33333333 | |
| 103 33.00970874 | |
| 104 32.69230769 | |
| 105 32.38095238 | |
| 106 32.07547170 | |
| 107 32.71028037 | |
| 108 33.33333333 | |
| 109 33.02752294 | |
| 110 33.63636364 | |
| 111 33.33333333 | |
| 112 33.92857143 | |
| 113 33.62831858 | |
| 114 34.21052632 | |
| 115 33.91304348 | |
| 116 33.62068966 | |
| 117 33.33333333 | |
| 118 33.05084746 | |
| 119 32.77310924 | |
| 120 32.50000000 | |
| 121 33.05785124 | |
| 122 32.78688525 | |
| 123 32.52032520 | |
| 124 33.06451613 | |
| 125 32.80000000 | |
| 126 32.53968254 | |
| 127 32.28346457 | |
| 128 32.03125000 | |
| 129 31.78294574 | |
| 130 32.30769231 | |
| 131 32.06106870 | |
| 132 31.81818182 | |
| 133 31.57894737 | |
| 134 31.34328358 | |
| 135 31.85185185 | |
| 136 31.61764706 | |
| 137 32.11678832 | |
| 138 32.60869565 | |
| 139 33.09352518 | |
| 140 32.85714286 | |
| 141 32.62411348 | |
| 142 32.39436620 | |
| 143 32.16783217 | |
| 144 32.63888889 | |
| 145 32.41379310 | |
| 146 32.19178082 | |
| 147 31.97278912 | |
| 148 31.75675676 | |
| 149 31.54362416 | |
| 150 31.33333333 | |
| 151 31.78807947 | |
| 152 31.57894737 | |
| 153 31.37254902 | |
| 154 31.16883117 | |
| 155 30.96774194 | |
| 156 31.41025641 | |
| 157 31.21019108 | |
| 158 31.64556962 | |
| 159 31.44654088 | |
| 160 31.25000000 | |
| 161 31.67701863 | |
| 162 32.09876543 | |
| 163 31.90184049 | |
| 164 31.70731707 | |
| 165 32.12121212 | |
| 166 32.53012048 | |
| 167 32.33532934 | |
| 168 32.14285714 | |
| 169 31.95266272 | |
| 170 31.76470588 | |
| 171 31.57894737 | |
| 172 31.39534884 | |
| 173 31.21387283 | |
| 174 31.60919540 | |
| 175 32.00000000 | |
| 176 32.38636364 | |
| 177 32.20338983 | |
| 178 32.02247191 | |
| 179 32.40223464 | |
| 180 32.77777778 | |
| 181 32.59668508 | |
| 182 32.41758242 | |
| 183 32.24043716 | |
| 184 32.06521739 | |
| 185 31.89189189 | |
| 186 32.25806452 | |
| 187 32.08556150 | |
| 188 31.91489362 | |
| 189 32.27513228 | |
| 190 32.63157895 | |
| 191 32.46073298 | |
| 192 32.29166667 | |
| 193 32.12435233 | |
| 194 32.47422680 | |
| 195 32.30769231 | |
| 196 32.14285714 | |
| 197 32.48730964 | |
| 198 32.82828283 | |
| 199 32.66331658 | |
| 200 32.50000000 | |
| 201 32.33830846 | |
| 202 32.67326733 | |
| 203 32.51231527 | |
| 204 32.35294118 | |
| 205 32.68292683 | |
| 206 32.52427184 | |
| 207 32.85024155 | |
| 208 32.69230769 | |
| 209 32.53588517 | |
| 210 32.38095238 | |
| 211 32.22748815 | |
| 212 32.07547170 | |
| 213 31.92488263 | |
| 214 31.77570093 | |
| 215 31.62790698 | |
| 216 31.94444444 | |
| 217 32.25806452 | |
| 218 32.56880734 | |
| 219 32.42009132 | |
| 220 32.27272727 | |
| 221 32.12669683 | |
| 222 31.98198198 | |
| 223 31.83856502 | |
| 224 32.14285714 | |
| 225 32.00000000 | |
| 226 32.30088496 | |
| 227 32.59911894 | |
| 228 32.45614035 | |
| 229 32.31441048 | |
| 230 32.17391304 | |
| 231 32.03463203 | |
| 232 31.89655172 | |
| 233 32.18884120 | |
| 234 32.47863248 | |
| 235 32.34042553 | |
| 236 32.62711864 | |
| 237 32.91139241 | |
| 238 33.19327731 | |
| 239 33.05439331 | |
| 240 32.91666667 | |
| 241 32.78008299 | |
| 242 32.64462810 | |
| 243 32.51028807 | |
| 244 32.37704918 | |
| 245 32.24489796 | |
| 246 32.11382114 | |
| 247 32.38866397 | |
| 248 32.25806452 | |
| 249 32.53012048 | |
| 250 32.40000000 | |
| 251 32.66932271 | |
| 252 32.53968254 | |
| 253 32.41106719 | |
| 254 32.28346457 | |
| 255 32.15686275 | |
| 256 32.03125000 | |
| 257 31.90661479 | |
| 258 31.78294574 | |
| 259 31.66023166 | |
| 260 31.53846154 | |
| 261 31.80076628 | |
| 262 31.67938931 | |
| 263 31.55893536 | |
| 264 31.81818182 | |
| 265 32.07547170 | |
| 266 31.95488722 | |
| 267 31.83520599 | |
| 268 31.71641791 | |
| 269 31.59851301 | |
| 270 31.48148148 | |
| 271 31.36531365 | |
| 272 31.25000000 | |
| 273 31.50183150 | |
| 274 31.38686131 | |
| 275 31.27272727 | |
| 276 31.15942029 | |
| 277 31.04693141 | |
| 278 30.93525180 | |
| 279 30.82437276 | |
| 280 31.07142857 | |
| 281 31.31672598 | |
| 282 31.20567376 | |
| 283 31.44876325 | |
| 284 31.69014085 | |
| 285 31.57894737 | |
| 286 31.81818182 | |
| 287 32.05574913 | |
| 288 32.29166667 | |
| 289 32.17993080 | |
| 290 32.06896552 | |
| 291 32.30240550 | |
| 292 32.19178082 | |
| 293 32.42320819 | |
| 294 32.31292517 | |
| 295 32.54237288 | |
| 296 32.77027027 | |
| 297 32.65993266 | |
| 298 32.55033557 | |
| 299 32.44147157 | |
| 300 32.33333333 | |
| 301 32.22591362 | |
| 302 32.11920530 | |
| 303 32.01320132 | |
| 304 31.90789474 | |
| 305 31.80327869 | |
| 306 31.69934641 | |
| 307 31.59609121 | |
| 308 31.81818182 | |
| 309 31.71521036 | |
| 310 31.61290323 | |
| 311 31.51125402 | |
| 312 31.73076923 | |
| 313 31.62939297 | |
| 314 31.52866242 | |
| 315 31.42857143 | |
| 316 31.32911392 | |
| 317 31.23028391 | |
| 318 31.13207547 | |
| 319 31.03448276 | |
| 320 30.93750000 | |
| 321 30.84112150 | |
| 322 31.05590062 | |
| 323 30.95975232 | |
| 324 30.86419753 | |
| 325 30.76923077 | |
| 326 30.67484663 | |
| 327 30.88685015 | |
| 328 31.09756098 | |
| 329 31.00303951 | |
| 330 31.21212121 | |
| 331 31.11782477 | |
| 332 31.32530120 | |
| 333 31.23123123 | |
| 334 31.13772455 | |
| 335 31.34328358 | |
| 336 31.54761905 | |
| 337 31.45400593 | |
| 338 31.36094675 | |
| 339 31.56342183 | |
| 340 31.47058824 | |
| 341 31.67155425 | |
| 342 31.87134503 | |
| 343 31.77842566 | |
| 344 31.97674419 | |
| 345 32.17391304 | |
| 346 32.08092486 | |
| 347 32.27665706 | |
| 348 32.18390805 | |
| 349 32.09169054 | |
| 350 32.00000000 | |
| 351 31.90883191 | |
| 352 31.81818182 | |
| 353 31.72804533 | |
| 354 31.63841808 | |
| 355 31.83098592 | |
| 356 31.74157303 | |
| 357 31.65266106 | |
| 358 31.84357542 | |
| 359 31.75487465 | |
| 360 31.66666667 | |
| 361 31.85595568 | |
| 362 31.76795580 | |
| 363 31.95592287 | |
| 364 31.86813187 | |
| 365 31.78082192 | |
| 366 31.69398907 | |
| 367 31.88010899 | |
| 368 31.79347826 | |
| 369 31.70731707 | |
| 370 31.62162162 | |
| 371 31.53638814 | |
| 372 31.72043011 | |
| 373 31.63538874 | |
| 374 31.81818182 | |
| 375 31.73333333 | |
| 376 31.64893617 | |
| 377 31.83023873 | |
| 378 31.74603175 | |
| 379 31.66226913 | |
| 380 31.84210526 | |
| 381 31.75853018 | |
| 382 31.67539267 | |
| 383 31.59268930 | |
| 384 31.51041667 | |
| 385 31.42857143 | |
| 386 31.34715026 | |
| 387 31.26614987 | |
| 388 31.44329897 | |
| 389 31.36246787 | |
| 390 31.28205128 | |
| 391 31.45780051 | |
| 392 31.37755102 | |
| 393 31.55216285 | |
| 394 31.72588832 | |
| 395 31.64556962 | |
| 396 31.56565657 | |
| 397 31.73803526 | |
| 398 31.65829146 | |
| 399 31.57894737 | |
| 400 31.50000000 | |
| 401 31.67082294 | |
| 402 31.84079602 | |
| 403 31.76178660 | |
| 404 31.68316832 | |
| 405 31.60493827 | |
| 406 31.52709360 | |
| 407 31.44963145 | |
| 408 31.37254902 | |
| 409 31.29584352 | |
| 410 31.21951220 | |
| 411 31.14355231 | |
| 412 31.31067961 | |
| 413 31.47699758 | |
| 414 31.64251208 | |
| 415 31.80722892 | |
| 416 31.73076923 | |
| 417 31.65467626 | |
| 418 31.57894737 | |
| 419 31.50357995 | |
| 420 31.66666667 | |
| 421 31.82897862 | |
| 422 31.75355450 | |
| 423 31.67848700 | |
| 424 31.60377358 | |
| 425 31.52941176 | |
| 426 31.45539906 | |
| 427 31.38173302 | |
| 428 31.30841121 | |
| 429 31.23543124 | |
| 430 31.16279070 | |
| 431 31.09048724 | |
| 432 31.25000000 | |
| 433 31.40877598 | |
| 434 31.56682028 | |
| 435 31.49425287 | |
| 436 31.42201835 | |
| 437 31.35011442 | |
| 438 31.50684932 | |
| 439 31.66287016 | |
| 440 31.59090909 | |
| 441 31.51927438 | |
| 442 31.44796380 | |
| 443 31.37697517 | |
| 444 31.53153153 | |
| 445 31.46067416 | |
| 446 31.39013453 | |
| 447 31.31991051 | |
| 448 31.25000000 | |
| 449 31.40311804 | |
| 450 31.55555556 | |
| 451 31.48558758 | |
| 452 31.41592920 | |
| 453 31.56732892 | |
| 454 31.49779736 | |
| 455 31.42857143 | |
| 456 31.35964912 | |
| 457 31.29102845 | |
| 458 31.22270742 | |
| 459 31.15468410 | |
| 460 31.08695652 | |
| 461 31.01952278 | |
| 462 31.16883117 | |
| 463 31.31749460 | |
| 464 31.46551724 | |
| 465 31.61290323 | |
| 466 31.54506438 | |
| 467 31.47751606 | |
| 468 31.62393162 | |
| 469 31.55650320 | |
| 470 31.48936170 | |
| 471 31.63481953 | |
| 472 31.77966102 | |
| 473 31.71247357 | |
| 474 31.85654008 | |
| 475 32.00000000 | |
| 476 31.93277311 | |
| 477 32.07547170 | |
| 478 32.00836820 | |
| 479 31.94154489 | |
| 480 31.87500000 | |
| 481 31.80873181 | |
| 482 31.95020747 | |
| 483 31.88405797 | |
| 484 32.02479339 | |
| 485 32.16494845 | |
| 486 32.09876543 | |
| 487 32.23819302 | |
| 488 32.17213115 | |
| 489 32.31083845 | |
| 490 32.44897959 | |
| 491 32.58655804 | |
| 492 32.72357724 | |
| 493 32.65720081 | |
| 494 32.79352227 | |
| 495 32.72727273 | |
| 496 32.66129032 | |
| 497 32.59557344 | |
| 498 32.53012048 | |
| 499 32.46492986 | |
| 500 32.40000000 | |
| 501 32.53493014 | |
| 502 32.47011952 | |
| 503 32.40556660 | |
| 504 32.53968254 | |
| 505 32.47524752 | |
| 506 32.41106719 | |
| 507 32.54437870 | |
| 508 32.67716535 | |
| 509 32.80943026 | |
| 510 32.94117647 | |
| 511 32.87671233 | |
| 512 32.81250000 | |
| 513 32.74853801 | |
| 514 32.68482490 | |
| 515 32.81553398 | |
| 516 32.75193798 | |
| 517 32.88201161 | |
| 518 32.81853282 | |
| 519 32.75529865 | |
| 520 32.69230769 | |
| 521 32.62955854 | |
| 522 32.56704981 | |
| 523 32.50478011 | |
| 524 32.44274809 | |
| 525 32.38095238 | |
| 526 32.31939163 | |
| 527 32.25806452 | |
| 528 32.19696970 | |
| 529 32.32514178 | |
| 530 32.45283019 | |
| 531 32.39171375 | |
| 532 32.33082707 | |
| 533 32.45778612 | |
| 534 32.39700375 | |
| 535 32.33644860 | |
| 536 32.27611940 | |
| 537 32.40223464 | |
| 538 32.34200743 | |
| 539 32.28200371 | |
| 540 32.22222222 | |
| 541 32.16266174 | |
| 542 32.10332103 | |
| 543 32.04419890 | |
| 544 31.98529412 | |
| 545 31.92660550 | |
| 546 31.86813187 | |
| 547 31.99268739 | |
| 548 32.11678832 | |
| 549 32.05828780 | |
| 550 32.00000000 | |
| 551 31.94192377 | |
| 552 32.06521739 | |
| 553 32.18806510 | |
| 554 32.12996390 | |
| 555 32.07207207 | |
| 556 32.01438849 | |
| 557 31.95691203 | |
| 558 31.89964158 | |
| 559 31.84257603 | |
| 560 31.96428571 | |
| 561 32.08556150 | |
| 562 32.02846975 | |
| 563 31.97158082 | |
| 564 31.91489362 | |
| 565 32.03539823 | |
| 566 32.15547703 | |
| 567 32.09876543 | |
| 568 32.21830986 | |
| 569 32.33743409 | |
| 570 32.28070175 | |
| 571 32.39929947 | |
| 572 32.34265734 | |
| 573 32.28621291 | |
| 574 32.40418118 | |
| 575 32.52173913 | |
| 576 32.63888889 | |
| 577 32.75563258 | |
| 578 32.87197232 | |
| 579 32.98791019 | |
| 580 33.10344828 | |
| 581 33.04647160 | |
| 582 33.16151203 | |
| 583 33.10463122 | |
| 584 33.21917808 | |
| 585 33.16239316 | |
| 586 33.27645051 | |
| 587 33.21976150 | |
| 588 33.16326531 | |
| 589 33.10696095 | |
| 590 33.05084746 | |
| 591 32.99492386 | |
| 592 32.93918919 | |
| 593 32.88364250 | |
| 594 32.99663300 | |
| 595 32.94117647 | |
| 596 32.88590604 | |
| 597 32.83082077 | |
| 598 32.77591973 | |
| 599 32.72120200 | |
| 600 32.66666667 | |
| 601 32.77870216 | |
| 602 32.72425249 | |
| 603 32.83582090 | |
| 604 32.78145695 | |
| 605 32.72727273 | |
| 606 32.83828383 | |
| 607 32.78418451 | |
| 608 32.73026316 | |
| 609 32.84072250 | |
| 610 32.78688525 | |
| 611 32.73322422 | |
| 612 32.84313725 | |
| 613 32.78955954 | |
| 614 32.73615635 | |
| 615 32.68292683 | |
| 616 32.79220779 | |
| 617 32.90113452 | |
| 618 32.84789644 | |
| 619 32.95638126 | |
| 620 32.90322581 | |
| 621 32.85024155 | |
| 622 32.95819936 | |
| 623 32.90529695 | |
| 624 32.85256410 | |
| 625 32.80000000 | |
| 626 32.74760383 | |
| 627 32.85486443 | |
| 628 32.96178344 | |
| 629 32.90937997 | |
| 630 32.85714286 | |
| 631 32.96354992 | |
| 632 32.91139241 | |
| 633 32.85939968 | |
| 634 32.96529968 | |
| 635 32.91338583 | |
| 636 32.86163522 | |
| 637 32.81004710 | |
| 638 32.75862069 | |
| 639 32.70735524 | |
| 640 32.81250000 | |
| 641 32.76131045 | |
| 642 32.71028037 | |
| 643 32.65940902 | |
| 644 32.60869565 | |
| 645 32.71317829 | |
| 646 32.66253870 | |
| 647 32.61205564 | |
| 648 32.71604938 | |
| 649 32.66563945 | |
| 650 32.76923077 | |
| 651 32.87250384 | |
| 652 32.97546012 | |
| 653 33.07810107 | |
| 654 33.02752294 | |
| 655 32.97709924 | |
| 656 33.07926829 | |
| 657 33.18112633 | |
| 658 33.13069909 | |
| 659 33.23216995 | |
| 660 33.18181818 | |
| 661 33.13161876 | |
| 662 33.08157100 | |
| 663 33.03167421 | |
| 664 32.98192771 | |
| 665 32.93233083 | |
| 666 33.03303303 | |
| 667 33.13343328 | |
| 668 33.08383234 | |
| 669 33.03437967 | |
| 670 33.13432836 | |
| 671 33.08494784 | |
| 672 33.03571429 | |
| 673 32.98662704 | |
| 674 32.93768546 | |
| 675 33.03703704 | |
| 676 33.13609467 | |
| 677 33.08714919 | |
| 678 33.03834808 | |
| 679 32.98969072 | |
| 680 32.94117647 | |
| 681 32.89280470 | |
| 682 32.99120235 | |
| 683 32.94289898 | |
| 684 33.04093567 | |
| 685 32.99270073 | |
| 686 32.94460641 | |
| 687 32.89665211 | |
| 688 32.84883721 | |
| 689 32.94629898 | |
| 690 32.89855072 | |
| 691 32.99565847 | |
| 692 33.09248555 | |
| 693 33.04473304 | |
| 694 32.99711816 | |
| 695 32.94964029 | |
| 696 33.04597701 | |
| 697 32.99856528 | |
| 698 32.95128940 | |
| 699 32.90414878 | |
| 700 33.00000000 | |
| 701 32.95292439 | |
| 702 32.90598291 | |
| 703 32.85917496 | |
| 704 32.81250000 | |
| 705 32.76595745 | |
| 706 32.86118980 | |
| 707 32.95615276 | |
| 708 32.90960452 | |
| 709 32.86318759 | |
| 710 32.81690141 | |
| 711 32.77074543 | |
| 712 32.86516854 | |
| 713 32.81907433 | |
| 714 32.77310924 | |
| 715 32.86713287 | |
| 716 32.82122905 | |
| 717 32.91492329 | |
| 718 32.86908078 | |
| 719 32.96244784 | |
| 720 32.91666667 | |
| 721 32.87101248 | |
| 722 32.82548476 | |
| 723 32.78008299 | |
| 724 32.73480663 | |
| 725 32.68965517 | |
| 726 32.78236915 | |
| 727 32.73727648 | |
| 728 32.69230769 | |
| 729 32.78463649 | |
| 730 32.87671233 | |
| 731 32.83173735 | |
| 732 32.92349727 | |
| 733 32.87858117 | |
| 734 32.83378747 | |
| 735 32.78911565 | |
| 736 32.74456522 | |
| 737 32.83582090 | |
| 738 32.79132791 | |
| 739 32.74695535 | |
| 740 32.70270270 | |
| 741 32.65856950 | |
| 742 32.61455526 | |
| 743 32.57065949 | |
| 744 32.66129032 | |
| 745 32.61744966 | |
| 746 32.70777480 | |
| 747 32.79785810 | |
| 748 32.75401070 | |
| 749 32.84379172 | |
| 750 32.93333333 | |
| Final result: 32.9333 +/- 1.7172 | |
| Random chance: 19.8992 +/- 1.4588 | |