Hugging Face's logo

phonemetransformers
/
CHILDES-Romanian-phoneme-tokenizer

Transformers
Inference Endpoints
Model card Files Community
codebyzeb commited on
Commit
be85522
·
verified ·
1 Parent(s): 5ded8f0

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +59 -55
  2. vocab.json +1 -1
tokenizer.json CHANGED
@@ -106,61 +106,65 @@
106
  "PAD": 1,
107
  "WORD_BOUNDARY": 2,
108
  "UTT_BOUNDARY": 3,
109
- "a": 4,
110
- "ʃ": 5,
111
- "m": 6,
112
- "e": 7,
113
- "h": 8,
114
- "d": 9,
115
- "s": 10,
116
- "f": 11,
117
- "t̠ʃ": 12,
118
- "l": 13,
119
- "ə": 14,
120
- "p": 15,
121
- "i": 16,
122
- "k": 17,
123
- "ɨ": 18,
124
- "n": 19,
125
- "uɪ": 20,
126
- "t": 21,
127
- "t̠ʃʲ": 22,
128
- "w": 23,
129
- "u": 24,
130
- "ts": 25,
131
- "aɪ": 26,
132
- "r": 27,
133
- "o": 28,
134
- "j": 29,
135
- "b": 30,
136
- "v": 31,
137
- "eɪ": 32,
138
- "ɔa": 33,
139
- "ɡ": 34,
140
- "z": 35,
141
- "ɾ": 36,
142
- "ea": 37,
143
- "iɪ": 38,
144
- "aʊ": 39,
145
- "ʒ": 40,
146
- "əɪ": 41,
147
- "d̠ʒ": 42,
148
- "eʊ": 43,
149
- "iʊ": 44,
150
- "tʲ": 45,
151
- "ŋ": 46,
152
- "tsʲ": 47,
153
- "kʲ": 48,
154
- "eo": 49,
155
- "d̠ʒʲ": 50,
156
- "rʲ": 51,
157
- "əʊ": 52,
158
- "oʊ": 53,
159
- "sʲ": 54,
160
- "ɾʲ": 55,
161
- "ɔ": 56,
162
- "nʲ": 57,
163
- "oɪ": 58
 
 
 
 
164
  },
165
  "unk_token": "UNK"
166
  }
 
106
  "PAD": 1,
107
  "WORD_BOUNDARY": 2,
108
  "UTT_BOUNDARY": 3,
109
+ "m": 4,
110
+ "ä": 5,
111
+ "n̪": 6,
112
+ "d̠ʒ": 7,
113
+ "i": 8,
114
+ "v": 9,
115
+ "e̞": 10,
116
+ "h": 11,
117
+ "u": 12,
118
+ "ʒ": 13,
119
+ "d̪": 14,
120
+ "o̞": 15,
121
+ "l": 16,
122
+ "ɾ̪": 17,
123
+ "t̠ʃ": 18,
124
+ "p": 19,
125
+ "j": 20,
126
+ "s̪": 21,
127
+ "oʊ": 22,
128
+ "t̪": 23,
129
+ "aɪ": 24,
130
+ "k": 25,
131
+ "w": 26,
132
+ "ɡ": 27,
133
+ "b": 28,
134
+ "t̠ʃʲ": 29,
135
+ "e̯ä": 30,
136
+ "ʃ": 31,
137
+ "ʃʲ": 32,
138
+ "ə": 33,
139
+ "o̯ä": 34,
140
+ "ɨ": 35,
141
+ "uɪ": 36,
142
+ "f": 37,
143
+ "t̪s̪": 38,
144
+ "z̪": 39,
145
+ "əɪ": 40,
146
+ "eɪ": 41,
147
+ "tsʲ": 42,
148
+ "zʲ": 43,
149
+ "iɪ": 44,
150
+ "aʊ": 45,
151
+ "tʲ": 46,
152
+ "nʲ": 47,
153
+ "eʊ": 48,
154
+ "iʊ": 49,
155
+ "ɾʲ": 50,
156
+ "mʲ": 51,
157
+ "bʲ": 52,
158
+ "sʲ": 53,
159
+ "kʲ": 54,
160
+ "lʲ": 55,
161
+ "eo": 56,
162
+ "d̠ʒʲ": 57,
163
+ "dʲ": 58,
164
+ "pʲ": 59,
165
+ "əʊ": 60,
166
+ "fʲ": 61,
167
+ "oɪ": 62
168
  },
169
  "unk_token": "UNK"
170
  }
vocab.json CHANGED
@@ -1 +1 @@
1
- {"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"a":4,"ʃ":5,"m":6,"e":7,"h":8,"d":9,"s":10,"f":11,"t̠ʃ":12,"l":13,"ə":14,"p":15,"i":16,"k":17,"ɨ":18,"n":19,"uɪ":20,"t":21,"t̠ʃʲ":22,"w":23,"u":24,"ts":25,"aɪ":26,"r":27,"o":28,"j":29,"b":30,"v":31,"eɪ":32,"ɔa":33,"ɡ":34,"z":35,"ɾ":36,"ea":37,"iɪ":38,"aʊ":39,"ʒ":40,"əɪ":41,"d̠ʒ":42,"eʊ":43,"iʊ":44,"tʲ":45,"ŋ":46,"tsʲ":47,"kʲ":48,"eo":49,"d̠ʒʲ":50,"rʲ":51,"əʊ":52,"oʊ":53,"sʲ":54,"ɾʲ":55,"ɔ":56,"nʲ":57,"oɪ":58}
 
1
+ {"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"m":4,"ä":5,"n̪":6,"d̠ʒ":7,"i":8,"v":9,"e̞":10,"h":11,"u":12,"ʒ":13,"d̪":14,"o̞":15,"l":16,"ɾ̪":17,"t̠ʃ":18,"p":19,"j":20,"s̪":21,"oʊ":22,"t̪":23,"aɪ":24,"k":25,"w":26,"ɡ":27,"b":28,"t̠ʃʲ":29,"e̯ä":30,"ʃ":31,"ʃʲ":32,"ə":33,"o̯ä":34,"ɨ":35,"uɪ":36,"f":37,"t̪s̪":38,"z̪":39,"əɪ":40,"eɪ":41,"tsʲ":42,"zʲ":43,"iɪ":44,"aʊ":45,"tʲ":46,"nʲ":47,"eʊ":48,"iʊ":49,"ɾʲ":50,"mʲ":51,"bʲ":52,"sʲ":53,"kʲ":54,"lʲ":55,"eo":56,"d̠ʒʲ":57,"dʲ":58,"pʲ":59,"əʊ":60,"fʲ":61,"oɪ":62}