tokenizer; pretrain

scripts/requirements.in

@@ -5,8 +5,8 @@ datasets
 jinja2
 transformers
 wandb
-litgpt[all]
-# litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
+# litgpt[all]
+litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
 litdata
 grokadamw
 # bitsandbytes

scripts/train_tokenizer.py

@@ -259,7 +259,7 @@ tokenizer.normalizer = normalizers.Sequence([
     normalizers.Replace(' ', '▁'),
 ])
 
-tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+tokenizer.pre_tokenizer = None
 
 tokenizer.post_processor = processors.TemplateProcessing(
     single='$A:0',                              # $A represents the token, :0 specifies the type ID for single sequences

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|end_of_text|>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87f950ea35683d2d80e36a35aa88b4c80fe5bd6d53aaab0a3ff3380ed0bc823e
+size 19928939