Update distilbert_japanese_tokenizer.py

As [discussed in the community](https://huggingface.co/line-corporation/line-distilbert-base-japanese/discussions/3), current tokenizer code does not work with `transformers>=4.34`, this is because the [tokenizer refactoring](https://github.com/huggingface/transformers/pull/23909) introduced in that version.

With this change, `PreTrainedTokenizer.__init__()` starts to access `get_vocab()`, so `self.subword_tokenizer_type` needs to be initialized before `super().__init__()` of `DistilBertJapaneseTokenizer`.

This issue is already fixed in `transformers` with [2da8853](https://github.com/huggingface/transformers/commit/2da8853775b61cde0894dee17c6c713aba711688). This PR basically follows that change.

Confirmed it works with [my repository](https://huggingface.co/liwii/line-distilbert-base-japanese-fork) forked from line-corporation/line-distilbert-base-japanese.

distilbert_japanese_tokenizer.py

@@ -170,25 +170,6 @@ class DistilBertJapaneseTokenizer(PreTrainedTokenizer):
         jumanpp_kwargs=None,
         **kwargs
     ):
-        super().__init__(
-            spm_file=spm_file,
-            unk_token=unk_token,
-            sep_token=sep_token,
-            pad_token=pad_token,
-            cls_token=cls_token,
-            mask_token=mask_token,
-            do_lower_case=do_lower_case,
-            do_word_tokenize=do_word_tokenize,
-            do_subword_tokenize=do_subword_tokenize,
-            word_tokenizer_type=word_tokenizer_type,
-            subword_tokenizer_type=subword_tokenizer_type,
-            never_split=never_split,
-            mecab_kwargs=mecab_kwargs,
-            sudachi_kwargs=sudachi_kwargs,
-            jumanpp_kwargs=jumanpp_kwargs,
-            **kwargs,
-        )
-
         if subword_tokenizer_type == "sentencepiece":
             if not os.path.isfile(spm_file):
                 raise ValueError(
@@ -236,14 +217,33 @@ class DistilBertJapaneseTokenizer(PreTrainedTokenizer):
         self.subword_tokenizer_type = subword_tokenizer_type
         if do_subword_tokenize:
             if subword_tokenizer_type == "wordpiece":
-                self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token)
+                self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))
             elif subword_tokenizer_type == "character":
-                self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, unk_token=self.unk_token)
+                self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, unk_token=str(unk_token))
             elif subword_tokenizer_type == "sentencepiece":
-                self.subword_tokenizer = SentencepieceTokenizer(vocab=self.spm_file, unk_token=self.unk_token)
+                self.subword_tokenizer = SentencepieceTokenizer(vocab=self.spm_file, unk_token=str(unk_token))
             else:
                 raise ValueError(f"Invalid subword_tokenizer_type '{subword_tokenizer_type}' is specified.")
 
+        super().__init__(
+            spm_file=spm_file,
+            unk_token=unk_token,
+            sep_token=sep_token,
+            pad_token=pad_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            do_lower_case=do_lower_case,
+            do_word_tokenize=do_word_tokenize,
+            do_subword_tokenize=do_subword_tokenize,
+            word_tokenizer_type=word_tokenizer_type,
+            subword_tokenizer_type=subword_tokenizer_type,
+            never_split=never_split,
+            mecab_kwargs=mecab_kwargs,
+            sudachi_kwargs=sudachi_kwargs,
+            jumanpp_kwargs=jumanpp_kwargs,
+            **kwargs,
+        )
+
     @property
     def do_lower_case(self):
         return self.lower_case