''' Defines the set of symbols used in text input to the model. ''' # cjehd_cleaners: _pad = '_' _punctuation = ',.!?…~' _letters = ' #*=AEINOQU^`abdefghijklmnopqrstuvwxyzãæçéðøĭŋœɐɑɔəɛɡɥɦɪɫɯɱɸɹɽɾʀʁʃʊʏʑʒʔʦʧʰˀˈˌːˑ̩̯̃͜͡βθχ⁼↑→↓šđǩḱ-ă,ś' # German_cleaners: _pad = '_' _punctuation =',.!?…~;:' _letters ="'*^_abdefghijklmnopstuvxyzçõøĭŋɐɘəɚɱɹɽɾʀʁʃʋʏʔʥʰʷˌːˑχ↓ⱼ" # Export all symbols: symbols = [_pad] + list(_punctuation) + list(_letters) # Special symbol ids SPACE_ID = symbols.index(" ")