| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 100.00000000 | |
| 4 75.00000000 | |
| 5 80.00000000 | |
| 6 83.33333333 | |
| 7 85.71428571 | |
| 8 87.50000000 | |
| 9 77.77777778 | |
| 10 80.00000000 | |
| 11 72.72727273 | |
| 12 75.00000000 | |
| 13 69.23076923 | |
| 14 71.42857143 | |
| 15 73.33333333 | |
| 16 75.00000000 | |
| 17 76.47058824 | |
| 18 72.22222222 | |
| 19 68.42105263 | |
| 20 65.00000000 | |
| 21 66.66666667 | |
| 22 68.18181818 | |
| 23 65.21739130 | |
| 24 66.66666667 | |
| 25 64.00000000 | |
| 26 65.38461538 | |
| 27 66.66666667 | |
| 28 64.28571429 | |
| 29 65.51724138 | |
| 30 63.33333333 | |
| 31 61.29032258 | |
| 32 62.50000000 | |
| 33 60.60606061 | |
| 34 58.82352941 | |
| 35 60.00000000 | |
| 36 58.33333333 | |
| 37 59.45945946 | |
| 38 57.89473684 | |
| 39 58.97435897 | |
| 40 60.00000000 | |
| 41 60.97560976 | |
| 42 59.52380952 | |
| 43 58.13953488 | |
| 44 56.81818182 | |
| 45 57.77777778 | |
| 46 58.69565217 | |
| 47 59.57446809 | |
| 48 60.41666667 | |
| 49 61.22448980 | |
| 50 60.00000000 | |
| 51 58.82352941 | |
| 52 59.61538462 | |
| 53 58.49056604 | |
| 54 57.40740741 | |
| 55 56.36363636 | |
| 56 57.14285714 | |
| 57 57.89473684 | |
| 58 58.62068966 | |
| 59 59.32203390 | |
| 60 60.00000000 | |
| 61 60.65573770 | |
| 62 59.67741935 | |
| 63 60.31746032 | |
| 64 60.93750000 | |
| 65 60.00000000 | |
| 66 60.60606061 | |
| 67 61.19402985 | |
| 68 61.76470588 | |
| 69 60.86956522 | |
| 70 60.00000000 | |
| 71 59.15492958 | |
| 72 58.33333333 | |
| 73 58.90410959 | |
| 74 59.45945946 | |
| 75 60.00000000 | |
| 76 60.52631579 | |
| 77 61.03896104 | |
| 78 61.53846154 | |
| 79 60.75949367 | |
| 80 61.25000000 | |
| 81 60.49382716 | |
| 82 60.97560976 | |
| 83 60.24096386 | |
| 84 60.71428571 | |
| 85 61.17647059 | |
| 86 60.46511628 | |
| 87 59.77011494 | |
| 88 60.22727273 | |
| 89 60.67415730 | |
| 90 60.00000000 | |
| 91 60.43956044 | |
| 92 60.86956522 | |
| 93 60.21505376 | |
| 94 59.57446809 | |
| 95 60.00000000 | |
| 96 60.41666667 | |
| 97 59.79381443 | |
| 98 60.20408163 | |
| 99 60.60606061 | |
| 100 61.00000000 | |
| 101 60.39603960 | |
| 102 60.78431373 | |
| 103 60.19417476 | |
| 104 60.57692308 | |
| 105 60.95238095 | |
| 106 60.37735849 | |
| 107 60.74766355 | |
| 108 60.18518519 | |
| 109 59.63302752 | |
| 110 60.00000000 | |
| 111 60.36036036 | |
| 112 60.71428571 | |
| 113 61.06194690 | |
| 114 61.40350877 | |
| 115 61.73913043 | |
| 116 61.20689655 | |
| 117 60.68376068 | |
| 118 61.01694915 | |
| 119 60.50420168 | |
| 120 60.83333333 | |
| 121 60.33057851 | |
| 122 60.65573770 | |
| 123 60.97560976 | |
| 124 60.48387097 | |
| 125 60.80000000 | |
| 126 61.11111111 | |
| 127 61.41732283 | |
| 128 61.71875000 | |
| 129 61.24031008 | |
| 130 60.76923077 | |
| 131 61.06870229 | |
| 132 61.36363636 | |
| 133 61.65413534 | |
| 134 61.19402985 | |
| 135 60.74074074 | |
| 136 60.29411765 | |
| 137 60.58394161 | |
| 138 60.86956522 | |
| 139 61.15107914 | |
| 140 61.42857143 | |
| 141 60.99290780 | |
| 142 61.26760563 | |
| 143 61.53846154 | |
| 144 61.11111111 | |
| 145 61.37931034 | |
| 146 60.95890411 | |
| 147 61.22448980 | |
| 148 61.48648649 | |
| 149 61.74496644 | |
| 150 62.00000000 | |
| 151 62.25165563 | |
| 152 62.50000000 | |
| 153 62.74509804 | |
| 154 62.33766234 | |
| 155 62.58064516 | |
| 156 62.82051282 | |
| 157 63.05732484 | |
| 158 62.65822785 | |
| 159 62.89308176 | |
| 160 62.50000000 | |
| 161 62.73291925 | |
| 162 62.96296296 | |
| 163 62.57668712 | |
| 164 62.19512195 | |
| 165 62.42424242 | |
| 166 62.65060241 | |
| 167 62.27544910 | |
| 168 62.50000000 | |
| 169 62.13017751 | |
| 170 62.35294118 | |
| 171 62.57309942 | |
| 172 62.20930233 | |
| 173 62.42774566 | |
| 174 62.64367816 | |
| 175 62.28571429 | |
| 176 62.50000000 | |
| 177 62.71186441 | |
| 178 62.92134831 | |
| 179 63.12849162 | |
| 180 63.33333333 | |
| 181 62.98342541 | |
| 182 63.18681319 | |
| 183 62.84153005 | |
| 184 63.04347826 | |
| 185 63.24324324 | |
| 186 63.44086022 | |
| 187 63.63636364 | |
| 188 63.29787234 | |
| 189 62.96296296 | |
| 190 62.63157895 | |
| 191 62.82722513 | |
| 192 63.02083333 | |
| 193 63.21243523 | |
| 194 63.40206186 | |
| 195 63.58974359 | |
| 196 63.77551020 | |
| 197 63.95939086 | |
| 198 64.14141414 | |
| 199 63.81909548 | |
| 200 64.00000000 | |
| 201 64.17910448 | |
| 202 63.86138614 | |
| 203 64.03940887 | |
| 204 64.21568627 | |
| 205 64.39024390 | |
| 206 64.07766990 | |
| 207 63.76811594 | |
| 208 63.94230769 | |
| 209 64.11483254 | |
| 210 64.28571429 | |
| 211 64.45497630 | |
| 212 64.15094340 | |
| 213 64.31924883 | |
| 214 64.48598131 | |
| 215 64.65116279 | |
| 216 64.35185185 | |
| 217 64.05529954 | |
| 218 63.76146789 | |
| 219 63.92694064 | |
| 220 63.63636364 | |
| 221 63.80090498 | |
| 222 63.51351351 | |
| 223 63.22869955 | |
| 224 63.39285714 | |
| 225 63.11111111 | |
| 226 62.83185841 | |
| 227 62.99559471 | |
| 228 63.15789474 | |
| 229 63.31877729 | |
| 230 63.47826087 | |
| 231 63.20346320 | |
| 232 62.93103448 | |
| 233 63.09012876 | |
| 234 63.24786325 | |
| 235 62.97872340 | |
| 236 62.71186441 | |
| 237 62.44725738 | |
| 238 62.60504202 | |
| 239 62.76150628 | |
| 240 62.91666667 | |
| 241 63.07053942 | |
| 242 63.22314050 | |
| 243 63.37448560 | |
| 244 63.11475410 | |
| 245 63.26530612 | |
| 246 63.41463415 | |
| 247 63.56275304 | |
| 248 63.70967742 | |
| 249 63.45381526 | |
| 250 63.60000000 | |
| 251 63.74501992 | |
| 252 63.88888889 | |
| 253 64.03162055 | |
| 254 63.77952756 | |
| 255 63.92156863 | |
| 256 64.06250000 | |
| 257 63.81322957 | |
| 258 63.95348837 | |
| 259 63.70656371 | |
| 260 63.84615385 | |
| 261 63.98467433 | |
| 262 64.12213740 | |
| 263 64.25855513 | |
| 264 64.39393939 | |
| 265 64.15094340 | |
| 266 63.90977444 | |
| 267 64.04494382 | |
| 268 64.17910448 | |
| 269 64.31226766 | |
| 270 64.44444444 | |
| 271 64.57564576 | |
| 272 64.33823529 | |
| 273 64.10256410 | |
| 274 63.86861314 | |
| 275 63.63636364 | |
| 276 63.76811594 | |
| 277 63.89891697 | |
| 278 63.66906475 | |
| 279 63.79928315 | |
| 280 63.92857143 | |
| 281 64.05693950 | |
| 282 64.18439716 | |
| 283 63.95759717 | |
| 284 64.08450704 | |
| 285 64.21052632 | |
| 286 64.33566434 | |
| 287 64.45993031 | |
| 288 64.58333333 | |
| 289 64.35986159 | |
| 290 64.48275862 | |
| 291 64.60481100 | |
| 292 64.72602740 | |
| 293 64.84641638 | |
| 294 64.62585034 | |
| 295 64.74576271 | |
| 296 64.86486486 | |
| 297 64.98316498 | |
| 298 65.10067114 | |
| 299 65.21739130 | |
| 300 65.00000000 | |
| 301 64.78405316 | |
| 302 64.90066225 | |
| 303 65.01650165 | |
| 304 65.13157895 | |
| 305 65.24590164 | |
| 306 65.03267974 | |
| 307 64.82084691 | |
| 308 64.93506494 | |
| 309 65.04854369 | |
| 310 64.83870968 | |
| 311 64.95176849 | |
| 312 64.74358974 | |
| 313 64.85623003 | |
| 314 64.64968153 | |
| 315 64.76190476 | |
| 316 64.87341772 | |
| 317 64.98422713 | |
| 318 64.77987421 | |
| 319 64.57680251 | |
| 320 64.37500000 | |
| 321 64.48598131 | |
| 322 64.59627329 | |
| 323 64.70588235 | |
| 324 64.81481481 | |
| 325 64.92307692 | |
| 326 65.03067485 | |
| 327 64.83180428 | |
| 328 64.93902439 | |
| 329 65.04559271 | |
| 330 64.84848485 | |
| 331 64.95468278 | |
| 332 65.06024096 | |
| 333 65.16516517 | |
| 334 65.26946108 | |
| 335 65.07462687 | |
| 336 65.17857143 | |
| 337 65.28189911 | |
| 338 65.08875740 | |
| 339 65.19174041 | |
| 340 65.29411765 | |
| 341 65.39589443 | |
| 342 65.49707602 | |
| 343 65.59766764 | |
| 344 65.69767442 | |
| 345 65.50724638 | |
| 346 65.60693642 | |
| 347 65.41786744 | |
| 348 65.22988506 | |
| 349 65.32951289 | |
| 350 65.42857143 | |
| 351 65.52706553 | |
| 352 65.62500000 | |
| 353 65.72237960 | |
| 354 65.53672316 | |
| 355 65.35211268 | |
| 356 65.44943820 | |
| 357 65.54621849 | |
| 358 65.64245810 | |
| 359 65.45961003 | |
| 360 65.55555556 | |
| 361 65.65096953 | |
| 362 65.74585635 | |
| 363 65.84022039 | |
| 364 65.65934066 | |
| 365 65.75342466 | |
| 366 65.57377049 | |
| 367 65.39509537 | |
| 368 65.48913043 | |
| 369 65.58265583 | |
| 370 65.67567568 | |
| 371 65.76819407 | |
| 372 65.86021505 | |
| 373 65.95174263 | |
| 374 66.04278075 | |
| 375 66.13333333 | |
| 376 65.95744681 | |
| 377 65.78249337 | |
| 378 65.87301587 | |
| 379 65.69920844 | |
| 380 65.78947368 | |
| 381 65.61679790 | |
| 382 65.44502618 | |
| 383 65.27415144 | |
| 384 65.36458333 | |
| 385 65.19480519 | |
| 386 65.28497409 | |
| 387 65.37467700 | |
| 388 65.20618557 | |
| 389 65.03856041 | |
| 390 64.87179487 | |
| 391 64.96163683 | |
| 392 65.05102041 | |
| 393 65.13994911 | |
| 394 65.22842640 | |
| 395 65.31645570 | |
| 396 65.15151515 | |
| 397 65.23929471 | |
| 398 65.07537688 | |
| 399 65.16290727 | |
| 400 65.25000000 | |
| 401 65.08728180 | |
| 402 65.17412935 | |
| 403 65.26054591 | |
| 404 65.34653465 | |
| 405 65.43209877 | |
| 406 65.51724138 | |
| 407 65.60196560 | |
| 408 65.68627451 | |
| 409 65.52567237 | |
| 410 65.60975610 | |
| 411 65.69343066 | |
| 412 65.77669903 | |
| 413 65.61743341 | |
| 414 65.70048309 | |
| 415 65.54216867 | |
| 416 65.62500000 | |
| 417 65.46762590 | |
| 418 65.31100478 | |
| 419 65.15513126 | |
| 420 65.23809524 | |
| 421 65.08313539 | |
| 422 65.16587678 | |
| 423 65.24822695 | |
| 424 65.09433962 | |
| 425 65.17647059 | |
| 426 65.02347418 | |
| 427 65.10538642 | |
| 428 64.95327103 | |
| 429 65.03496503 | |
| 430 64.88372093 | |
| 431 64.96519722 | |
| 432 65.04629630 | |
| 433 64.89607390 | |
| 434 64.97695853 | |
| 435 65.05747126 | |
| 436 65.13761468 | |
| 437 65.21739130 | |
| 438 65.06849315 | |
| 439 65.14806378 | |
| 440 65.00000000 | |
| 441 64.85260771 | |
| 442 64.70588235 | |
| 443 64.55981941 | |
| 444 64.41441441 | |
| 445 64.49438202 | |
| 446 64.57399103 | |
| 447 64.65324385 | |
| 448 64.73214286 | |
| 449 64.81069042 | |
| 450 64.66666667 | |
| 451 64.74501109 | |
| 452 64.82300885 | |
| 453 64.67991170 | |
| 454 64.75770925 | |
| 455 64.83516484 | |
| 456 64.91228070 | |
| 457 64.98905908 | |
| 458 65.06550218 | |
| 459 65.14161220 | |
| 460 65.00000000 | |
| 461 65.07592191 | |
| 462 65.15151515 | |
| 463 65.01079914 | |
| 464 64.87068966 | |
| 465 64.94623656 | |
| 466 65.02145923 | |
| 467 65.09635974 | |
| 468 65.17094017 | |
| 469 65.03198294 | |
| 470 65.10638298 | |
| 471 65.18046709 | |
| 472 65.25423729 | |
| 473 65.11627907 | |
| 474 65.18987342 | |
| 475 65.26315789 | |
| 476 65.33613445 | |
| 477 65.40880503 | |
| 478 65.48117155 | |
| 479 65.55323591 | |
| 480 65.62500000 | |
| 481 65.69646570 | |
| 482 65.76763485 | |
| 483 65.83850932 | |
| 484 65.90909091 | |
| 485 65.97938144 | |
| 486 66.04938272 | |
| 487 65.91375770 | |
| 488 65.98360656 | |
| 489 66.05316973 | |
| 490 65.91836735 | |
| 491 65.78411405 | |
| 492 65.85365854 | |
| 493 65.72008114 | |
| 494 65.78947368 | |
| 495 65.65656566 | |
| 496 65.72580645 | |
| 497 65.79476861 | |
| 498 65.86345382 | |
| 499 65.93186373 | |
| 500 66.00000000 | |
| 501 65.86826347 | |
| 502 65.93625498 | |
| 503 66.00397614 | |
| 504 66.07142857 | |
| 505 66.13861386 | |
| 506 66.20553360 | |
| 507 66.07495069 | |
| 508 65.94488189 | |
| 509 65.81532417 | |
| 510 65.68627451 | |
| 511 65.55772994 | |
| 512 65.62500000 | |
| 513 65.49707602 | |
| 514 65.56420233 | |
| 515 65.63106796 | |
| 516 65.69767442 | |
| 517 65.76402321 | |
| 518 65.83011583 | |
| 519 65.70327553 | |
| 520 65.57692308 | |
| 521 65.64299424 | |
| 522 65.70881226 | |
| 523 65.58317400 | |
| 524 65.45801527 | |
| 525 65.33333333 | |
| 526 65.20912548 | |
| 527 65.08538899 | |
| 528 64.96212121 | |
| 529 64.83931947 | |
| 530 64.90566038 | |
| 531 64.97175141 | |
| 532 65.03759398 | |
| 533 65.10318949 | |
| 534 64.98127341 | |
| 535 64.85981308 | |
| 536 64.92537313 | |
| 537 64.99068901 | |
| 538 65.05576208 | |
| 539 65.12059369 | |
| 540 65.00000000 | |
| 541 64.87985213 | |
| 542 64.94464945 | |
| 543 64.82504604 | |
| 544 64.70588235 | |
| 545 64.77064220 | |
| 546 64.65201465 | |
| 547 64.53382084 | |
| 548 64.59854015 | |
| 549 64.66302368 | |
| 550 64.72727273 | |
| 551 64.60980036 | |
| 552 64.49275362 | |
| 553 64.55696203 | |
| 554 64.62093863 | |
| 555 64.68468468 | |
| 556 64.56834532 | |
| 557 64.63195691 | |
| 558 64.51612903 | |
| 559 64.57960644 | |
| 560 64.64285714 | |
| 561 64.70588235 | |
| 562 64.76868327 | |
| 563 64.83126110 | |
| 564 64.89361702 | |
| 565 64.77876106 | |
| 566 64.84098940 | |
| 567 64.72663139 | |
| 568 64.78873239 | |
| 569 64.67486819 | |
| 570 64.56140351 | |
| 571 64.62346760 | |
| 572 64.51048951 | |
| 573 64.57242583 | |
| 574 64.63414634 | |
| 575 64.69565217 | |
| 576 64.75694444 | |
| 577 64.81802426 | |
| 578 64.87889273 | |
| 579 64.93955095 | |
| 580 64.82758621 | |
| 581 64.71600688 | |
| 582 64.77663230 | |
| 583 64.83704974 | |
| 584 64.89726027 | |
| 585 64.95726496 | |
| 586 65.01706485 | |
| 587 65.07666099 | |
| 588 65.13605442 | |
| 589 65.19524618 | |
| 590 65.25423729 | |
| 591 65.31302876 | |
| 592 65.20270270 | |
| 593 65.09274874 | |
| 594 65.15151515 | |
| 595 65.04201681 | |
| 596 64.93288591 | |
| 597 64.82412060 | |
| 598 64.88294314 | |
| 599 64.94156928 | |
| 600 64.83333333 | |
| 601 64.89184692 | |
| 602 64.78405316 | |
| 603 64.67661692 | |
| 604 64.73509934 | |
| 605 64.79338843 | |
| 606 64.85148515 | |
| 607 64.90939044 | |
| 608 64.96710526 | |
| 609 64.86042693 | |
| 610 64.91803279 | |
| 611 64.81178396 | |
| 612 64.86928105 | |
| 613 64.76345840 | |
| 614 64.82084691 | |
| 615 64.71544715 | |
| 616 64.61038961 | |
| 617 64.66774716 | |
| 618 64.56310680 | |
| 619 64.45880452 | |
| 620 64.51612903 | |
| 621 64.57326892 | |
| 622 64.46945338 | |
| 623 64.36597111 | |
| 624 64.26282051 | |
| 625 64.16000000 | |
| 626 64.05750799 | |
| 627 64.11483254 | |
| 628 64.17197452 | |
| 629 64.06995231 | |
| 630 64.12698413 | |
| 631 64.18383518 | |
| 632 64.08227848 | |
| 633 63.98104265 | |
| 634 64.03785489 | |
| 635 64.09448819 | |
| 636 64.15094340 | |
| 637 64.05023548 | |
| 638 64.10658307 | |
| 639 64.16275430 | |
| 640 64.06250000 | |
| 641 63.96255850 | |
| 642 63.86292835 | |
| 643 63.76360809 | |
| 644 63.81987578 | |
| 645 63.72093023 | |
| 646 63.77708978 | |
| 647 63.83307573 | |
| 648 63.73456790 | |
| 649 63.63636364 | |
| 650 63.69230769 | |
| 651 63.74807988 | |
| 652 63.65030675 | |
| 653 63.70597243 | |
| 654 63.76146789 | |
| 655 63.66412214 | |
| 656 63.56707317 | |
| 657 63.47031963 | |
| 658 63.37386018 | |
| 659 63.27769347 | |
| 660 63.33333333 | |
| 661 63.23751891 | |
| 662 63.29305136 | |
| 663 63.34841629 | |
| 664 63.40361446 | |
| 665 63.30827068 | |
| 666 63.21321321 | |
| 667 63.26836582 | |
| 668 63.17365269 | |
| 669 63.07922272 | |
| 670 62.98507463 | |
| 671 62.89120715 | |
| 672 62.94642857 | |
| 673 62.85289747 | |
| 674 62.90801187 | |
| 675 62.81481481 | |
| 676 62.86982249 | |
| 677 62.92466765 | |
| 678 62.97935103 | |
| 679 63.03387334 | |
| 680 63.08823529 | |
| 681 62.99559471 | |
| 682 62.90322581 | |
| 683 62.81112738 | |
| 684 62.86549708 | |
| 685 62.91970803 | |
| 686 62.82798834 | |
| 687 62.73653566 | |
| 688 62.79069767 | |
| 689 62.69956459 | |
| 690 62.75362319 | |
| 691 62.80752533 | |
| 692 62.86127168 | |
| 693 62.91486291 | |
| 694 62.96829971 | |
| 695 62.87769784 | |
| 696 62.78735632 | |
| 697 62.69727403 | |
| 698 62.75071633 | |
| 699 62.80400572 | |
| 700 62.85714286 | |
| 701 62.91012839 | |
| 702 62.96296296 | |
| 703 63.01564723 | |
| 704 63.06818182 | |
| 705 63.12056738 | |
| 706 63.17280453 | |
| 707 63.22489392 | |
| 708 63.27683616 | |
| 709 63.32863188 | |
| 710 63.38028169 | |
| 711 63.43178622 | |
| 712 63.48314607 | |
| 713 63.53436185 | |
| 714 63.44537815 | |
| 715 63.35664336 | |
| 716 63.40782123 | |
| 717 63.31938633 | |
| 718 63.23119777 | |
| 719 63.14325452 | |
| 720 63.19444444 | |
| 721 63.24549237 | |
| 722 63.29639889 | |
| 723 63.34716459 | |
| 724 63.25966851 | |
| 725 63.31034483 | |
| 726 63.22314050 | |
| 727 63.13617607 | |
| 728 63.18681319 | |
| 729 63.23731139 | |
| 730 63.28767123 | |
| 731 63.33789330 | |
| 732 63.38797814 | |
| 733 63.43792633 | |
| 734 63.48773842 | |
| 735 63.53741497 | |
| 736 63.58695652 | |
| 737 63.50067843 | |
| 738 63.55013550 | |
| 739 63.46414073 | |
| 740 63.37837838 | |
| 741 63.42780027 | |
| 742 63.34231806 | |
| 743 63.39165545 | |
| 744 63.30645161 | |
| 745 63.35570470 | |
| 746 63.40482574 | |
| 747 63.31994645 | |
| 748 63.23529412 | |
| 749 63.15086782 | |
| 750 63.20000000 | |
| Final result: 63.2000 +/- 1.7621 | |
| Random chance: 25.0083 +/- 1.5824 | |