tmae commited on
Commit
990cf60
·
1 Parent(s): 5ed6bca

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +0 -12
tokenizer.json CHANGED
@@ -97,18 +97,6 @@
97
  },
98
  "pre_tokenizer": null,
99
  "post_processor": null,
100
- "special_tokens": {
101
- "<EOD|LLM-jp>": {
102
- "id": "<EOD|LLM-jp>",
103
- "ids": [
104
- 7
105
- ],
106
- "tokens": [
107
- "<EOD|LLM-jp>"
108
- ]
109
- }
110
- }
111
- },
112
  "decoder": {
113
  "type": "Sequence",
114
  "decoders": [
 
97
  },
98
  "pre_tokenizer": null,
99
  "post_processor": null,
 
 
 
 
 
 
 
 
 
 
 
 
100
  "decoder": {
101
  "type": "Sequence",
102
  "decoders": [