pretrain dataset

scripts/prepare_pretrain_dataset.py

@@ -96,7 +96,7 @@ datasets_configs = [
         # ~3 GB, 4,976,850
         {'path': 'saillab/taco-datasets', 'data_dir': name, 'split': 'train', 'format': '{instruction} {input} {output}'}
         for name in [
-            # 'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4',
+            'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4',
             'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k',
         ]
     ],
@@ -109,11 +109,11 @@ datasets_configs = [
     #     {'path': 'jordiclive/wikipedia-summary-dataset', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['summary']}
     #     for i in range(0, 100, 20)
     # ],
-    # ~17.6 GB, ~6.41M rows
-    [
-        {'path': 'wikimedia/wikipedia', 'name': '20231101.en', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['text']}
-        for i in range(0, 100, 20)
-    ],
+    ## ~17.6 GB, ~6.41M rows
+    # [
+    #     {'path': 'wikimedia/wikipedia', 'name': '20231101.en', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['text']}
+    #     for i in range(0, 100, 20)
+    # ],
     # 65.1 MB, 7,819
     {'path': 'Sketched33/Cities_Wikipedia_Information', 'format': lambda n: n['wikipedia_content']},