:sparkles: Added test_script and a folder for scripts

test_script.py

@@ -0,0 +1,45 @@
+"""CONFIG"""
+#!/usr/bin/env python3
+from transformers import RobertaConfig
+config = RobertaConfig.from_pretrained("roberta-large")
+config.save_pretrained("./")
+
+"""TOKENIZER"""
+#!/usr/bin/env python3
+from datasets import load_dataset
+from tokenizers import ByteLevelBPETokenizer
+# load dataset
+dataset = load_dataset("large_spanish_corpus")
+# Instantiate tokenizer
+tokenizer = ByteLevelBPETokenizer()
+def batch_iterator(batch_size=1000):
+    for i in range(0, len(dataset), batch_size):
+        yield dataset[i: i + batch_size]["text"]
+# Customized training
+tokenizer.train_from_iterator(batch_iterator(), vocab_size=50265, min_frequency=2, special_tokens=[
+    "<s>",
+    "<pad>",
+    "</s>",
+    "<unk>",
+    "<mask>",
+])
+# Save files to disk
+tokenizer.save("./tokenizer.json")
+
+"""TOKENIZER"""
+#!/usr/bin/env bash
+./run_mlm_flax.py \
+    --output_dir="./" \
+    --model_type="roberta" \
+    --config_name="./" \
+    --tokenizer_name="./" \
+    --dataset_name="large_spanish_corpus" \
+    --dataset_config_name \ # I think this would be empty
+    --max_seq_length="128" \
+    --per_device_train_batch_size="4" \
+    --per_device_eval_batch_size="4" \
+    --learning_rate="3e-4" \
+    --warmup_steps="1000" \
+    --overwrite_output_dir \
+    --num_train_epochs="8" \
+    --push_to_hub