adding saved tokenizer

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e2f7ab8f62e892f6b916e2746d9e5ae76d39e840e7b7ef999bcdb62ed3bec58
+oid sha256:872860cb3510b706fb519ac5684f79479eda7a379e1f8ddd564d37d40237ba71
 size 1262040407

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|"}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a189d29c5d827af709833ebbb11b15b88f0dba74e6acf1bcad13271eff79d228
+oid sha256:d560876b7930226e53041da76aa870f84489163362f6b578efd026e3a8725f29
 size 2287

vocab.json

@@ -1 +1 @@
-{"n": 0, "s": 1, "i": 2, "r": 3, "t": 4, "\u0254": 5, "\u025b": 6, "l": 7, "w": 8, "u": 9, "o": 10, "m": 11, "g": 12, "d": 13, "b": 15, "p": 16, "a": 17, "f": 18, "\u2019": 19, "y": 20, "e": 21, "k": 22, "h": 23, "|": 14, "[UNK]": 24, "[PAD]": 25}
+{"k": 0, "p": 1, "d": 2, "o": 3, "h": 4, "m": 6, "ɛ": 7, "ɔ": 8, "n": 9, "’": 10, "g": 11, "b": 12, "t": 13, "s": 14, "r": 15, "u": 16, "l": 17, "e": 18, "w": 19, "a": 20, "f": 21, "y": 22, "i": 23, "|": 5, "[UNK]": 24, "[PAD]": 25}