Spaces:
Running
on
Zero
Running
on
Zero
File size: 2,623 Bytes
568e264 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# Copyright (c) 2023 Wenet Community. (authors: Dinghao Zhou)
# (authors: Xingchen Song)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from wenet.text.base_tokenizer import BaseTokenizer
from wenet.text.bpe_tokenizer import BpeTokenizer
from wenet.text.char_tokenizer import CharTokenizer
from wenet.text.hugging_face_tokenizer import HuggingFaceTokenizer
from wenet.text.paraformer_tokenizer import ParaformerTokenizer
from wenet.text.whisper_tokenizer import WhisperTokenizer
def init_tokenizer(configs) -> BaseTokenizer:
# TODO(xcsong): Forcefully read the 'tokenizer' attribute.
tokenizer_type = configs.get("tokenizer", "char")
if tokenizer_type == "whisper":
tokenizer = WhisperTokenizer(
multilingual=configs['tokenizer_conf']['is_multilingual'],
num_languages=configs['tokenizer_conf']['num_languages'])
elif tokenizer_type == "char":
tokenizer = CharTokenizer(
configs['tokenizer_conf']['symbol_table_path'],
configs['tokenizer_conf']['non_lang_syms_path'],
split_with_space=configs['tokenizer_conf'].get(
'split_with_space', False),
connect_symbol=configs['tokenizer_conf'].get('connect_symbol', ''))
elif tokenizer_type == "bpe":
tokenizer = BpeTokenizer(
configs['tokenizer_conf']['bpe_path'],
configs['tokenizer_conf']['symbol_table_path'],
configs['tokenizer_conf']['non_lang_syms_path'],
split_with_space=configs['tokenizer_conf'].get(
'split_with_space', False))
elif tokenizer_type == 'paraformer':
tokenizer = ParaformerTokenizer(
symbol_table=configs['tokenizer_conf']['symbol_table_path'],
seg_dict=configs['tokenizer_conf']['seg_dict_path'])
elif tokenizer_type == 'huggingface':
tokenizer = HuggingFaceTokenizer(
model=configs['tokenizer_conf']['llm_path'])
else:
raise NotImplementedError
logging.info("use {} tokenizer".format(configs["tokenizer"]))
return tokenizer
|