Hugging Face's logo

phonemetransformers
/
CHILDES-PortugueseBr-phoneme-tokenizer

Transformers
Inference Endpoints
Model card Files Community
codebyzeb commited on
Commit
cfa3f65
·
verified ·
1 Parent(s): 26ee18d

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +9 -0
  2. tokenizer_config.json +8 -0
tokenizer.json CHANGED
@@ -21,6 +21,15 @@
21
  "normalized": false,
22
  "special": true
23
  },
 
 
 
 
 
 
 
 
 
24
  {
25
  "id": 3,
26
  "content": "UTT_BOUNDARY",
 
21
  "normalized": false,
22
  "special": true
23
  },
24
+ {
25
+ "id": 2,
26
+ "content": "WORD_BOUNDARY",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
  {
34
  "id": 3,
35
  "content": "UTT_BOUNDARY",
tokenizer_config.json CHANGED
@@ -17,6 +17,14 @@
17
  "single_word": false,
18
  "special": true
19
  },
 
 
 
 
 
 
 
 
20
  "3": {
21
  "content": "UTT_BOUNDARY",
22
  "lstrip": false,
 
17
  "single_word": false,
18
  "special": true
19
  },
20
+ "2": {
21
+ "content": "WORD_BOUNDARY",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
  "3": {
29
  "content": "UTT_BOUNDARY",
30
  "lstrip": false,