nagrajn commited on
Commit
2fdeaeb
1 Parent(s): 4a593b0

Upload tokenizer

Browse files
added_tokens.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "<|a|>": 50259,
3
- "<|c|>": 50257,
4
- "<|f|>": 50260,
5
- "<|u|>": 50258
6
  }
 
1
  {
2
+ "[assist]": 50259,
3
+ "[context]": 50257,
4
+ "[fin]": 50260,
5
+ "[user]": 50258
6
  }
special_tokens_map.json CHANGED
@@ -1,28 +1,28 @@
1
  {
2
  "additional_special_tokens": [
3
  {
4
- "content": "<|c|>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
- "content": "<|u|>",
12
  "lstrip": false,
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
  },
17
  {
18
- "content": "<|a|>",
19
  "lstrip": false,
20
  "normalized": false,
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
  {
25
- "content": "<|f|>",
26
  "lstrip": false,
27
  "normalized": false,
28
  "rstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
  {
4
+ "content": "[context]",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
+ "content": "[user]",
12
  "lstrip": false,
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
  },
17
  {
18
+ "content": "[assist]",
19
  "lstrip": false,
20
  "normalized": false,
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
  {
25
+ "content": "[fin]",
26
  "lstrip": false,
27
  "normalized": false,
28
  "rstrip": false,
tokenizer.json CHANGED
@@ -14,7 +14,7 @@
14
  },
15
  {
16
  "id": 50257,
17
- "content": "<|c|>",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
@@ -23,7 +23,7 @@
23
  },
24
  {
25
  "id": 50258,
26
- "content": "<|u|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
@@ -32,7 +32,7 @@
32
  },
33
  {
34
  "id": 50259,
35
- "content": "<|a|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
@@ -41,7 +41,7 @@
41
  },
42
  {
43
  "id": 50260,
44
- "content": "<|f|>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
 
14
  },
15
  {
16
  "id": 50257,
17
+ "content": "[context]",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
 
23
  },
24
  {
25
  "id": 50258,
26
+ "content": "[user]",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
 
32
  },
33
  {
34
  "id": 50259,
35
+ "content": "[assist]",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
 
41
  },
42
  {
43
  "id": 50260,
44
+ "content": "[fin]",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -10,7 +10,7 @@
10
  "special": true
11
  },
12
  "50257": {
13
- "content": "<|c|>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
@@ -18,7 +18,7 @@
18
  "special": true
19
  },
20
  "50258": {
21
- "content": "<|u|>",
22
  "lstrip": false,
23
  "normalized": false,
24
  "rstrip": false,
@@ -26,7 +26,7 @@
26
  "special": true
27
  },
28
  "50259": {
29
- "content": "<|a|>",
30
  "lstrip": false,
31
  "normalized": false,
32
  "rstrip": false,
@@ -34,7 +34,7 @@
34
  "special": true
35
  },
36
  "50260": {
37
- "content": "<|f|>",
38
  "lstrip": false,
39
  "normalized": false,
40
  "rstrip": false,
@@ -43,10 +43,10 @@
43
  }
44
  },
45
  "additional_special_tokens": [
46
- "<|c|>",
47
- "<|u|>",
48
- "<|a|>",
49
- "<|f|>"
50
  ],
51
  "bos_token": "<|endoftext|>",
52
  "clean_up_tokenization_spaces": true,
 
10
  "special": true
11
  },
12
  "50257": {
13
+ "content": "[context]",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
 
18
  "special": true
19
  },
20
  "50258": {
21
+ "content": "[user]",
22
  "lstrip": false,
23
  "normalized": false,
24
  "rstrip": false,
 
26
  "special": true
27
  },
28
  "50259": {
29
+ "content": "[assist]",
30
  "lstrip": false,
31
  "normalized": false,
32
  "rstrip": false,
 
34
  "special": true
35
  },
36
  "50260": {
37
+ "content": "[fin]",
38
  "lstrip": false,
39
  "normalized": false,
40
  "rstrip": false,
 
43
  }
44
  },
45
  "additional_special_tokens": [
46
+ "[context]",
47
+ "[user]",
48
+ "[assist]",
49
+ "[fin]"
50
  ],
51
  "bos_token": "<|endoftext|>",
52
  "clean_up_tokenization_spaces": true,