add space
Browse files- sentencepiece_ja.py +1 -0
sentencepiece_ja.py
CHANGED
|
@@ -28,6 +28,7 @@ class SentencePieceJA(PreTrainedTokenizer):
|
|
| 28 |
'unk_token': unk,
|
| 29 |
'mask_token': mask
|
| 30 |
})
|
|
|
|
| 31 |
|
| 32 |
def get_vocab(self) -> int:
|
| 33 |
return self._tokenizer.get_vocab()
|
|
|
|
| 28 |
'unk_token': unk,
|
| 29 |
'mask_token': mask
|
| 30 |
})
|
| 31 |
+
self._tokenizer.add_tokens([" ", " "])
|
| 32 |
|
| 33 |
def get_vocab(self) -> int:
|
| 34 |
return self._tokenizer.get_vocab()
|