fenguhao commited on
Commit
747c7bb
1 Parent(s): f7f5a43

Upload tokenizer

Browse files
Files changed (4) hide show
  1. added_tokens.json +2 -1
  2. tokenizer.json +4 -4
  3. tokenizer_config.json +2 -2
  4. vocab.json +1 -1
added_tokens.json CHANGED
@@ -1,3 +1,4 @@
1
  {
2
- "1": 1
 
3
  }
 
1
  {
2
+ "0": 1,
3
+ "1": 2
4
  }
tokenizer.json CHANGED
@@ -4,7 +4,7 @@
4
  "padding": null,
5
  "added_tokens": [
6
  {
7
- "id": 0,
8
  "content": "0",
9
  "single_word": false,
10
  "lstrip": false,
@@ -13,7 +13,7 @@
13
  "special": true
14
  },
15
  {
16
- "id": 1,
17
  "content": "1",
18
  "single_word": false,
19
  "lstrip": false,
@@ -46,12 +46,12 @@
46
  "dropout": null,
47
  "unk_token": null,
48
  "continuing_subword_prefix": "",
49
- "end_of_word_suffix": " ",
50
  "fuse_unk": false,
51
  "byte_fallback": false,
52
  "ignore_merges": false,
53
  "vocab": {
54
- "0": 0
55
  },
56
  "merges": []
57
  }
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
7
+ "id": 1,
8
  "content": "0",
9
  "single_word": false,
10
  "lstrip": false,
 
13
  "special": true
14
  },
15
  {
16
+ "id": 2,
17
  "content": "1",
18
  "single_word": false,
19
  "lstrip": false,
 
46
  "dropout": null,
47
  "unk_token": null,
48
  "continuing_subword_prefix": "",
49
+ "end_of_word_suffix": "",
50
  "fuse_unk": false,
51
  "byte_fallback": false,
52
  "ignore_merges": false,
53
  "vocab": {
54
+ "0 ": 0
55
  },
56
  "merges": []
57
  }
tokenizer_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
- "0": {
5
  "content": "0",
6
  "lstrip": false,
7
  "normalized": false,
@@ -9,7 +9,7 @@
9
  "single_word": false,
10
  "special": true
11
  },
12
- "1": {
13
  "content": "1",
14
  "lstrip": false,
15
  "normalized": false,
 
1
  {
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
+ "1": {
5
  "content": "0",
6
  "lstrip": false,
7
  "normalized": false,
 
9
  "single_word": false,
10
  "special": true
11
  },
12
+ "2": {
13
  "content": "1",
14
  "lstrip": false,
15
  "normalized": false,
vocab.json CHANGED
@@ -1 +1 @@
1
- {"0":0}
 
1
+ {"0 ":0}