adds turkce_atasozleri dataset and results (#1)

- adds turkce_atasozleri dataset and results (5e62ec852c98968ce5a3779433ddc810c073f6f6)

data.py

@@ -64,6 +64,7 @@ DATASET_TASK_DICT = {
     # other generation
     'wmt-tr-en-prompt': Tasks.MACHINE_TRANSLATION,
     'gecturk_generation': Tasks.GRAMMATICAL_ERROR_CORRECTION,
+    'turkce_atasozleri': Tasks.MULTIPLE_CHOICE,
 }
 
 
@@ -90,8 +91,8 @@ DATASET_GROUPS = {
         'description': 'Turkish splits of SQuAD-like datasets XQuAD and TQUAD.',
     },
     'MCQA': {
-        'datasets': ['xcopa_tr', 'exams_tr', 'belebele_tr'] + [x for x in DATASET_TASK_DICT.keys() if x.startswith('turkish_plu')],
-        'description': 'Multiple Choice Question Answering datasets: XCOPA, Exams, Belebele and Turkish PLU.'
+        'datasets': ['xcopa_tr', 'exams_tr', 'belebele_tr', 'turkce_atasozleri'] + [x for x in DATASET_TASK_DICT.keys() if x.startswith('turkish_plu')],
+        'description': 'Multiple Choice Question Answering datasets: XCOPA, Exams, Belebele, Turkish PLU, and Turkce-Atasozleri.'
     },
     'TC': {
         'datasets': ['sts_tr', 'offenseval_tr', 'news_cat', 'ironytr', ],

data/datasets.json

@@ -181,5 +181,13 @@
         "url": "https://github.com/GGLAB-KU/gecturk",
         "hf_name": "mcemilg/GECTurk-generation",
         "generative": true
+    },
+    "turkce_atasozleri": {
+        "name": "Proverbs",
+        "task": "multiple_choice",
+        "description": "A dataset for Turkish proverbs and their definitions.",
+        "url": "https://huggingface.co/datasets/furkanunluturk/turkce-atasozleri",
+        "hf_name": "abrek/turkce-atasozleri-lm-evaluation-harness",
+        "generative": false
     }
 }

results/zero-shot/CerebrumTech__cere-llama-3-8b-tr.json

@@ -168,6 +168,12 @@
             "task": "multiple_choice",
             "acc": 0.5144444444444445,
             "acc_norm": 0.5144444444444445
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.48092485549132946,
+            "acc_norm": 0.48092485549132946
         }
     ]
 }

results/zero-shot/Llama-3.3-70B-Instruct.json

@@ -170,6 +170,12 @@
             "rouge1": 0.4242379216426208,
             "rouge2": 0.2810704842229309,
             "rougeL": 0.3500597320086401
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.9254335260115607,
+            "acc_norm": 0.9254335260115607
         }
     ]
 }

results/zero-shot/Ministral-8B-Instruct.json

@@ -167,6 +167,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0.009003803745967548
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.4046242774566474,
+            "acc_norm": 0.4046242774566474
         }
     ]
 }

results/zero-shot/Mistral-7B-Instruct-v0.3.json

@@ -167,6 +167,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0.009677885309836777
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.3,
+            "acc_norm": 0.3
         }
     ]
 }

results/zero-shot/Mistral-7B-v0.3.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.4111111111111111,
             "acc_norm": 0.4111111111111111
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.27572254335260116,
+            "acc_norm": 0.27572254335260116
         }
     ]
 }

results/zero-shot/Mixtral-8x7B-Instruct-v0.1.json

@@ -169,6 +169,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0.036255958399537776
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.5150289017341041,
+            "acc_norm": 0.5150289017341041
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B-Instruct.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.3,
             "acc_norm": 0.3
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.2832369942196532,
+            "acc_norm": 0.2832369942196532
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.29888888888888887,
             "acc_norm": 0.29888888888888887
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.20346820809248556,
+            "acc_norm": 0.20346820809248556
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B-Instruct.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.5344444444444445,
             "acc_norm": 0.5344444444444445
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.3468208092485549,
+            "acc_norm": 0.3468208092485549
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.4666666666666667,
             "acc_norm": 0.4666666666666667
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.2300578034682081,
+            "acc_norm": 0.2300578034682081
         }
     ]
 }

results/zero-shot/Qwen2.5-14B-Instruct.json

@@ -169,6 +169,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0.00130001444460494
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.7832369942196532,
+            "acc_norm": 0.7832369942196532
         }
     ]
 }

results/zero-shot/Qwen2.5-14B.json

@@ -169,6 +169,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0.003707448601280755
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.753757225433526,
+            "acc_norm": 0.753757225433526
         }
     ]
 }

results/zero-shot/Qwen2.5-3B-Instruct.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.6744444444444444,
             "acc_norm": 0.6744444444444444
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.6011560693641619,
+            "acc_norm": 0.6011560693641619
         }
     ]
 }

results/zero-shot/Qwen2.5-3B.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.6188888888888889,
             "acc_norm": 0.6188888888888889
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.4346820809248555,
+            "acc_norm": 0.4346820809248555
         }
     ]
 }

results/zero-shot/Qwen2.5-7B-Instruct.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.7344444444444445,
             "acc_norm": 0.7344444444444445
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.7121387283236994,
+            "acc_norm": 0.7121387283236994
         }
     ]
 }

results/zero-shot/Qwen2.5-7B.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.7388888888888889,
             "acc_norm": 0.7388888888888889
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.7352601156069364,
+            "acc_norm": 0.7352601156069364
         }
     ]
 }

results/zero-shot/aya-23-35B.json

@@ -169,6 +169,12 @@
             "rouge1": 0.3920969453077054,
             "rouge2": 0.25937196554017156,
             "rougeL": 0.33144850765201345
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.5687861271676301,
+            "acc_norm": 0.5687861271676301
         }
     ]
 }

results/zero-shot/aya-23-8b.json

@@ -163,6 +163,12 @@
             "rouge1": 0.2855728817569547,
             "rouge2": 0.14081555638864124,
             "rougeL": 0.23467303626936886
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.44971098265895953,
+            "acc_norm": 0.44971098265895953
         }
     ]
 }

results/zero-shot/aya-expanse-32b.json

@@ -168,6 +168,12 @@
             "name": "gecturk_generation",
             "task": "grammatical_error_correction",
             "exact_match": 0
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.8236994219653179,
+            "acc_norm": 0.8236994219653179
         }
     ]
 }

results/zero-shot/aya-expanse-8b.json

@@ -154,6 +154,12 @@
             "task": "extractive_question_answering",
             "exact_match": 0.2495798319327731,
             "f1": 0.4735125568867167
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.723121387283237,
+            "acc_norm": 0.723121387283237
         }
     ]
 }

results/zero-shot/aya101.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.2288888888888889,
             "acc_norm": 0.2288888888888889
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.009826589595375723,
+            "acc_norm": 0.009826589595375723
         }
     ]
 }

results/zero-shot/commencis-7b.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.3233333333333333,
             "acc_norm": 0.3233333333333333
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.22658959537572254,
+            "acc_norm": 0.22658959537572254
         }
     ]
 }

results/zero-shot/kanarya-2b.json

@@ -166,6 +166,12 @@
             "task": "extractive_question_answering",
             "exact_match": 0.008403361344537815,
             "f1": 0.027799180278171867
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.0,
+            "acc_norm": 0.0
         }
     ]
 }

results/zero-shot/llama-3-8b-instruct.json

@@ -162,6 +162,12 @@
             "rouge1": 0.29619456321037296,
             "rouge2": 0.13520487191226377,
             "rougeL": 0.220446635816053
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.6947976878612717,
+            "acc_norm": 0.6947976878612717
         }
     ]
 }

results/zero-shot/llama-3-8b.json

@@ -161,6 +161,12 @@
             "rouge1": 0.2615001361521869,
             "rouge2": 0.11093149007661907,
             "rougeL": 0.20321693263972507
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.44046242774566474,
+            "acc_norm": 0.44046242774566474
         }
     ]
 }

results/zero-shot/llama-3.1-8b-instruct.json

@@ -161,6 +161,12 @@
             "rouge1": 0.2805962791068744,
             "rouge2": 0.12421139697660691,
             "rougeL": 0.21080710839195932
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.7549132947976879,
+            "acc_norm": 0.7549132947976879
         }
     ]
 }

results/zero-shot/llama-3.1-8b.json

@@ -161,6 +161,12 @@
             "rouge1": 0.2805962791068744,
             "rouge2": 0.12421139697660691,
             "rougeL": 0.21080710839195932
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.5410404624277456,
+            "acc_norm": 0.5410404624277456
         }
     ]
 }

results/zero-shot/llama-3.2-1b.json

@@ -193,6 +193,12 @@
             "rouge1": 0.2429304790539497,
             "rouge2": 0.09668008744707143,
             "rougeL": 0.18327092913535944
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.21676300578034682,
+            "acc_norm": 0.21676300578034682
         }
     ]
 }

results/zero-shot/llama-3.2-3b-instruct.json

@@ -186,6 +186,12 @@
             "rouge1": 0.2616423061938248,
             "rouge2": 0.11064039063859936,
             "rougeL": 0.19686955120787036
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.010982658959537572,
+            "acc_norm": 0.010982658959537572
         }
     ]
 }

results/zero-shot/llama-3.2-3b.json

@@ -155,6 +155,12 @@
             "rouge1": 0.2718989612690207,
             "rouge2": 0.12031087767355472,
             "rougeL": 0.21504760362022454
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.1994219653179191,
+            "acc_norm": 0.1994219653179191
         }
     ]
 }

results/zero-shot/mistral-7b.json

@@ -160,6 +160,12 @@
             "rouge1": 0.02720399421152351,
             "rouge2": 0.012032606076011431,
             "rougeL": 0.02311080687545987
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.30809248554913293,
+            "acc_norm": 0.30809248554913293
         }
     ]
 }

results/zero-shot/trendyol-7b.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.3622222222222222,
             "acc_norm": 0.3622222222222222
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.0,
+            "acc_norm": 0.0
         }
     ]
 }

results/zero-shot/turna.json

@@ -167,6 +167,12 @@
             "task": "multiple_choice",
             "acc": 0.22555555555555556,
             "acc_norm": 0.22555555555555556
+        },
+        {
+            "name": "turkce_atasozleri",
+            "task": "multiple_choice",
+            "acc": 0.19248554913294796,
+            "acc_norm": 0.19248554913294796
         }
     ]
 }