scores/Watt-Tool-8B-iq3_m.arc

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 869 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 869 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	100.00000000
2	100.00000000
3	100.00000000
4	75.00000000
5	80.00000000
6	83.33333333
7	85.71428571
8	87.50000000
9	77.77777778
10	80.00000000
11	72.72727273
12	75.00000000
13	69.23076923
14	71.42857143
15	73.33333333
16	75.00000000
17	76.47058824
18	72.22222222
19	68.42105263
20	65.00000000
21	66.66666667
22	68.18181818
23	69.56521739
24	70.83333333
25	68.00000000
26	65.38461538
27	66.66666667
28	64.28571429
29	65.51724138
30	63.33333333
31	61.29032258
32	59.37500000
33	57.57575758
34	55.88235294
35	57.14285714
36	55.55555556
37	56.75675676
38	55.26315789
39	56.41025641
40	55.00000000
41	56.09756098
42	54.76190476
43	53.48837209
44	52.27272727
45	53.33333333
46	54.34782609
47	55.31914894
48	56.25000000
49	57.14285714
50	56.00000000
51	54.90196078
52	55.76923077
53	54.71698113
54	53.70370370
55	52.72727273
56	53.57142857
57	54.38596491
58	55.17241379
59	55.93220339
60	56.66666667
61	57.37704918
62	56.45161290
63	57.14285714
64	57.81250000
65	56.92307692
66	57.57575758
67	58.20895522
68	58.82352941
69	59.42028986
70	58.57142857
71	57.74647887
72	56.94444444
73	57.53424658
74	58.10810811
75	58.66666667
76	59.21052632
77	59.74025974
78	60.25641026
79	59.49367089
80	60.00000000
81	59.25925926
82	59.75609756
83	59.03614458
84	59.52380952
85	60.00000000
86	60.46511628
87	60.91954023
88	61.36363636
89	61.79775281
90	62.22222222
91	62.63736264
92	63.04347826
93	62.36559140
94	61.70212766
95	62.10526316
96	62.50000000
97	61.85567010
98	61.22448980
99	61.61616162
100	62.00000000
101	61.38613861
102	61.76470588
103	61.16504854
104	60.57692308
105	60.95238095
106	60.37735849
107	60.74766355
108	60.18518519
109	59.63302752
110	60.00000000
111	60.36036036
112	60.71428571
113	60.17699115
114	60.52631579
115	60.86956522
116	61.20689655
117	60.68376068
118	61.01694915
119	60.50420168
120	60.83333333
121	60.33057851
122	60.65573770
123	60.97560976
124	60.48387097
125	60.80000000
126	61.11111111
127	61.41732283
128	61.71875000
129	61.24031008
130	60.76923077
131	61.06870229
132	61.36363636
133	61.65413534
134	61.19402985
135	60.74074074
136	60.29411765
137	60.58394161
138	60.86956522
139	61.15107914
140	61.42857143
141	60.99290780
142	61.26760563
143	61.53846154
144	61.11111111
145	61.37931034
146	60.95890411
147	61.22448980
148	61.48648649
149	61.74496644
150	62.00000000
151	61.58940397
152	61.84210526
153	62.09150327
154	61.68831169
155	61.29032258
156	61.53846154
157	61.78343949
158	61.39240506
159	61.63522013
160	61.25000000
161	61.49068323
162	61.72839506
163	61.34969325
164	61.58536585
165	61.81818182
166	62.04819277
167	61.67664671
168	61.90476190
169	61.53846154
170	61.76470588
171	61.98830409
172	61.62790698
173	61.84971098
174	62.06896552
175	61.71428571
176	61.93181818
177	62.14689266
178	62.35955056
179	62.56983240
180	62.77777778
181	62.43093923
182	62.63736264
183	62.29508197
184	61.95652174
185	62.16216216
186	62.36559140
187	62.56684492
188	62.23404255
189	61.90476190
190	61.57894737
191	61.78010471
192	61.97916667
193	62.17616580
194	62.37113402
195	62.56410256
196	62.75510204
197	62.94416244
198	62.62626263
199	62.31155779
200	62.50000000
201	62.68656716
202	62.37623762
203	62.56157635
204	62.74509804
205	62.43902439
206	62.13592233
207	61.83574879
208	62.01923077
209	62.20095694
210	62.38095238
211	62.55924171
212	62.26415094
213	62.44131455
214	62.61682243
215	62.79069767
216	62.50000000
217	62.67281106
218	62.38532110
219	62.55707763
220	62.27272727
221	62.44343891
222	62.16216216
223	61.88340807
224	62.05357143
225	61.77777778
226	61.50442478
227	61.67400881
228	61.84210526
229	62.00873362
230	62.17391304
231	61.90476190
232	61.63793103
233	61.80257511
234	61.53846154
235	61.27659574
236	61.01694915
237	60.75949367
238	60.92436975
239	61.08786611
240	61.25000000
241	61.41078838
242	61.57024793
243	61.72839506
244	61.47540984
245	61.63265306
246	61.38211382
247	61.53846154
248	61.69354839
249	61.44578313
250	61.60000000
251	61.75298805
252	61.90476190
253	62.05533597
254	61.81102362
255	61.96078431
256	62.10937500
257	61.86770428
258	62.01550388
259	61.77606178
260	61.92307692
261	62.06896552
262	62.21374046
263	62.35741445
264	62.50000000
265	62.64150943
266	62.40601504
267	62.54681648
268	62.31343284
269	62.45353160
270	62.59259259
271	62.73062731
272	62.50000000
273	62.27106227
274	62.04379562
275	61.81818182
276	61.95652174
277	62.09386282
278	61.87050360
279	62.00716846
280	62.14285714
281	62.27758007
282	62.41134752
283	62.19081272
284	62.32394366
285	62.45614035
286	62.58741259
287	62.36933798
288	62.50000000
289	62.28373702
290	62.41379310
291	62.54295533
292	62.67123288
293	62.79863481
294	62.58503401
295	62.71186441
296	62.83783784
297	62.96296296
298	63.08724832
299	63.21070234
300	63.33333333
301	63.12292359
302	63.24503311
303	63.36633663
304	63.15789474
305	63.27868852
306	63.07189542
307	63.19218241
308	63.31168831
309	63.43042071
310	63.54838710
311	63.66559486
312	63.78205128
313	63.89776358
314	63.69426752
315	63.49206349
316	63.60759494
317	63.72239748
318	63.52201258
319	63.32288401
320	63.12500000
321	63.23987539
322	63.35403727
323	63.46749226
324	63.27160494
325	63.38461538
326	63.49693252
327	63.30275229
328	63.41463415
329	63.52583587
330	63.33333333
331	63.44410876
332	63.55421687
333	63.66366366
334	63.77245509
335	63.58208955
336	63.69047619
337	63.79821958
338	63.60946746
339	63.71681416
340	63.82352941
341	63.92961877
342	63.74269006
343	63.84839650
344	63.95348837
345	63.76811594
346	63.87283237
347	63.68876081
348	63.50574713
349	63.61031519
350	63.42857143
351	63.53276353
352	63.63636364
353	63.73937677
354	63.55932203
355	63.38028169
356	63.48314607
357	63.58543417
358	63.68715084
359	63.50974930
360	63.61111111
361	63.71191136
362	63.81215470
363	63.63636364
364	63.46153846
365	63.56164384
366	63.38797814
367	63.21525886
368	63.31521739
369	63.41463415
370	63.51351351
371	63.34231806
372	63.17204301
373	63.27077748
374	63.36898396
375	63.46666667
376	63.29787234
377	63.12997347
378	63.22751323
379	63.06068602
380	63.15789474
381	62.99212598
382	62.82722513
383	62.66318538
384	62.76041667
385	62.85714286
386	62.95336788
387	62.79069767
388	62.62886598
389	62.46786632
390	62.30769231
391	62.40409207
392	62.50000000
393	62.59541985
394	62.69035533
395	62.78481013
396	62.62626263
397	62.72040302
398	62.56281407
399	62.65664160
400	62.75000000
401	62.59351621
402	62.43781095
403	62.53101737
404	62.62376238
405	62.71604938
406	62.80788177
407	62.89926290
408	62.99019608
409	62.83618582
410	62.92682927
411	63.01703163
412	63.10679612
413	62.95399516
414	63.04347826
415	62.89156627
416	62.98076923
417	62.82973621
418	62.91866029
419	62.76849642
420	62.85714286
421	62.70783848
422	62.79620853
423	62.88416076
424	62.73584906
425	62.82352941
426	62.67605634
427	62.76346604
428	62.61682243
429	62.70396270
430	62.55813953
431	62.64501160
432	62.73148148
433	62.58660508
434	62.67281106
435	62.75862069
436	62.84403670
437	62.92906178
438	62.78538813
439	62.87015945
440	62.72727273
441	62.58503401
442	62.44343891
443	62.30248307
444	62.16216216
445	62.24719101
446	62.33183857
447	62.41610738
448	62.50000000
449	62.58351893
450	62.44444444
451	62.30598670
452	62.38938053
453	62.25165563
454	62.33480176
455	62.19780220
456	62.28070175
457	62.36323851
458	62.44541485
459	62.52723312
460	62.60869565
461	62.68980477
462	62.77056277
463	62.63498920
464	62.50000000
465	62.58064516
466	62.66094421
467	62.74089936
468	62.60683761
469	62.47334755
470	62.55319149
471	62.63269639
472	62.50000000
473	62.36786469
474	62.44725738
475	62.52631579
476	62.60504202
477	62.68343816
478	62.76150628
479	62.83924843
480	62.91666667
481	62.78586279
482	62.86307054
483	62.93995859
484	63.01652893
485	63.09278351
486	63.16872428
487	63.03901437
488	63.11475410
489	62.98568507
490	62.85714286
491	62.72912424
492	62.80487805
493	62.67748479
494	62.75303644
495	62.62626263
496	62.70161290
497	62.57545272
498	62.65060241
499	62.72545090
500	62.80000000
501	62.67465070
502	62.74900398
503	62.82306163
504	62.89682540
505	62.97029703
506	62.84584980
507	62.72189349
508	62.59842520
509	62.47544204
510	62.35294118
511	62.23091977
512	62.30468750
513	62.18323587
514	62.25680934
515	62.13592233
516	62.20930233
517	62.28239845
518	62.35521236
519	62.42774566
520	62.30769231
521	62.38003839
522	62.45210728
523	62.33269598
524	62.21374046
525	62.09523810
526	61.97718631
527	61.85958254
528	61.74242424
529	61.62570888
530	61.69811321
531	61.58192090
532	61.65413534
533	61.72607880
534	61.61048689
535	61.49532710
536	61.56716418
537	61.63873371
538	61.71003717
539	61.78107607
540	61.66666667
541	61.73752311
542	61.80811808
543	61.69429098
544	61.76470588
545	61.83486239
546	61.72161172
547	61.60877514
548	61.67883212
549	61.74863388
550	61.81818182
551	61.70598911
552	61.59420290
553	61.66365280
554	61.73285199
555	61.80180180
556	61.69064748
557	61.75942549
558	61.64874552
559	61.71735242
560	61.78571429
561	61.85383244
562	61.92170819
563	61.98934281
564	62.05673759
565	61.94690265
566	62.01413428
567	61.90476190
568	61.97183099
569	61.86291740
570	61.75438596
571	61.82136602
572	61.71328671
573	61.78010471
574	61.84668990
575	61.73913043
576	61.80555556
577	61.87175043
578	61.93771626
579	62.00345423
580	61.89655172
581	61.79001721
582	61.68384880
583	61.74957118
584	61.81506849
585	61.88034188
586	61.94539249
587	62.01022147
588	62.07482993
589	62.13921902
590	62.20338983
591	62.26734349
592	62.16216216
593	62.05733558
594	62.12121212
595	62.01680672
596	61.91275168
597	61.80904523
598	61.70568562
599	61.76961603
600	61.66666667
601	61.73044925
602	61.62790698
603	61.52570481
604	61.42384106
605	61.48760331
606	61.55115512
607	61.61449753
608	61.67763158
609	61.57635468
610	61.63934426
611	61.53846154
612	61.60130719
613	61.50081566
614	61.56351792
615	61.46341463
616	61.36363636
617	61.42625608
618	61.48867314
619	61.38933764
620	61.45161290
621	61.51368760
622	61.41479100
623	61.31621188
624	61.21794872
625	61.12000000
626	61.02236422
627	61.08452951
628	61.14649682
629	61.04928458
630	61.11111111
631	61.17274168
632	61.07594937
633	60.97946288
634	61.04100946
635	61.10236220
636	61.16352201
637	61.06750392
638	61.12852665
639	61.18935837
640	61.09375000
641	61.15444618
642	61.05919003
643	60.96423017
644	61.02484472
645	60.93023256
646	60.99071207
647	61.05100464
648	61.11111111
649	61.01694915
650	61.07692308
651	61.13671275
652	61.19631902
653	61.10260337
654	61.16207951
655	61.06870229
656	60.97560976
657	60.88280061
658	60.79027356
659	60.69802731
660	60.75757576
661	60.66565809
662	60.72507553
663	60.78431373
664	60.84337349
665	60.90225564
666	60.81081081
667	60.86956522
668	60.77844311
669	60.68759342
670	60.74626866
671	60.65573770
672	60.71428571
673	60.62407132
674	60.53412463
675	60.44444444
676	60.35502959
677	60.41358936
678	60.47197640
679	60.53019146
680	60.58823529
681	60.49926579
682	60.41055718
683	60.32210835
684	60.38011696
685	60.43795620
686	60.34985423
687	60.26200873
688	60.31976744
689	60.23222061
690	60.28985507
691	60.34732272
692	60.40462428
693	60.46176046
694	60.51873199
695	60.43165468
696	60.34482759
697	60.25824964
698	60.17191977
699	60.22889843
700	60.28571429
701	60.34236805
702	60.25641026
703	60.31294452
704	60.36931818
705	60.42553191
706	60.48158640
707	60.53748232
708	60.59322034
709	60.64880113
710	60.70422535
711	60.75949367
712	60.81460674
713	60.86956522
714	60.78431373
715	60.69930070
716	60.75418994
717	60.66945607
718	60.58495822
719	60.50069541
720	60.55555556
721	60.61026352
722	60.66481994
723	60.71922545
724	60.63535912
725	60.68965517
726	60.60606061
727	60.52269601
728	60.57692308
729	60.63100137
730	60.68493151
731	60.73871409
732	60.79234973
733	60.84583902
734	60.89918256
735	60.95238095
736	61.00543478
737	60.92265943
738	60.97560976
739	60.89309878
740	60.81081081
741	60.86369771
742	60.78167116
743	60.83445491
744	60.75268817
745	60.80536913
746	60.85790885
747	60.77643909
748	60.69518717
749	60.61415220
750	60.66666667

Final result: 60.6667 +/- 1.7849
Random chance: 25.0083 +/- 1.5824