|
from transformers import AutoModelForCTC, AutoProcessor |
|
model_id = "." |
|
model = AutoModelForCTC.from_pretrained(model_id) |
|
processor = AutoProcessor.from_pretrained(model_id) |
|
|
|
vocab_dict = processor.tokenizer.get_vocab() |
|
|
|
sorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])} |
|
print("the sorted vocab list is", sorted_vocab_dict) |
|
|
|
from pyctcdecode import build_ctcdecoder |
|
|
|
decoder = build_ctcdecoder( |
|
labels=list(sorted_vocab_dict.keys()), |
|
kenlm_model_path="5gram_correct.arpa", |
|
) |
|
|
|
from transformers import Wav2Vec2ProcessorWithLM |
|
|
|
processor_with_lm = Wav2Vec2ProcessorWithLM( |
|
feature_extractor=processor.feature_extractor, |
|
tokenizer=processor.tokenizer, |
|
decoder=decoder |
|
) |
|
|
|
processor_with_lm.save_pretrained(".") |
