tangledgroup
/

tangled-llama-e-128k-v0.1

Text Generation

Inference Endpoints

Model card Files Files and versions Community

mtasic85 commited on 21 days ago

Commit

804c80b

•

1 Parent(s): 419b474

tokenizer

Files changed (1) hide show

scripts/train_tokenizer.py +2 -2

scripts/train_tokenizer.py CHANGED Viewed

@@ -255,7 +255,7 @@ bpe = BPE(unk_token='<unk>', fuse_unk=True, byte_fallback=True)
 tokenizer = Tokenizer(bpe)
 tokenizer.normalizer = normalizers.Sequence([
-    # normalizers.Prepend('▁'),
     normalizers.Replace(' ', '▁'),
 ])
@@ -271,7 +271,7 @@ tokenizer.decoder = decoders.Sequence([
     decoders.Replace('▁', ' '),
     decoders.ByteFallback(),
     decoders.Fuse(),
-    # decoders.Strip(' ', 1, 0),
 ])
 trainer = BpeTrainer(

 tokenizer = Tokenizer(bpe)
 tokenizer.normalizer = normalizers.Sequence([
+    normalizers.Prepend('▁'),
     normalizers.Replace(' ', '▁'),
 ])
     decoders.Replace('▁', ' '),
     decoders.ByteFallback(),
     decoders.Fuse(),
+    decoders.Strip(' ', 1, 0),
 ])
 trainer = BpeTrainer(