iioSnail
/

ChineseBERT-base

feature-extraction

Model card Files Files and versions Community

iioSnail commited on Jul 10, 2023

Commit

f76a1c9

·

1 Parent(s): 78daf2e

Upload bert_tokenizer.py

Files changed (1) hide show

bert_tokenizer.py +1 -1

bert_tokenizer.py CHANGED Viewed

@@ -143,7 +143,7 @@ class ChineseBertTokenizer(BertTokenizerFast):
                 pinyin_ids.append([0] * 8)
                 continue
-            pinyin_string = pinyin(token, style=Style.TONE3, errors=lambda x: [['not chinese'] for _ in x])[0]
             if pinyin_string == "not chinese":
                 pinyin_ids.append([0] * 8)

                 pinyin_ids.append([0] * 8)
                 continue
+            pinyin_string = pinyin(token, style=Style.TONE3, errors=lambda x: [['not chinese'] for _ in x])[0][0]
             if pinyin_string == "not chinese":
                 pinyin_ids.append([0] * 8)