add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Jan25_22-00-26_72940b2e396a/1643148035.7183433/events.out.tfevents.1643148035.72940b2e396a.82.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4edaf737aa2ca6b76b777fb8fa627379ec78fe64e806fce37adb6b4b079b47cf
+size 4782

runs/Jan25_22-00-26_72940b2e396a/events.out.tfevents.1643148035.72940b2e396a.82.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:256ebf090cf599185651bba4d02054be0a61f209c9e81e902fab8cf40a330992
+size 4896

runs/Jan25_22-41-55_72940b2e396a/1643150525.1576257/events.out.tfevents.1643150525.72940b2e396a.82.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad4299def49f65dc5d3251cffd0497a5f6c9a6228dcde8a8174788448fca544b
+size 4782

runs/Jan25_22-41-55_72940b2e396a/events.out.tfevents.1643150525.72940b2e396a.82.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4783fe4ba1dc2f661329af9c7bbc1a79f728130056b2adca91c25c50723840b8
+size 4742