MRNH commited on
Commit
e43b505
·
verified ·
1 Parent(s): f06e88f

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -7,6 +7,20 @@
7
  "rstrip": false,
8
  "single_word": false
9
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  {
11
  "content": "<answer>",
12
  "lstrip": true,
@@ -14,6 +28,20 @@
14
  "rstrip": false,
15
  "single_word": false
16
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  {
18
  "content": "<context>",
19
  "lstrip": true,
 
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
+ {
11
+ "content": "<stopTMP>",
12
+ "lstrip": true,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ {
18
+ "content": "</question>",
19
+ "lstrip": true,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
  {
25
  "content": "<answer>",
26
  "lstrip": true,
 
28
  "rstrip": false,
29
  "single_word": false
30
  },
31
+ {
32
+ "content": "</answer>",
33
+ "lstrip": true,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ },
38
+ {
39
+ "content": "</context>",
40
+ "lstrip": true,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false
44
+ },
45
  {
46
  "content": "<context>",
47
  "lstrip": true,
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
The diff for this file is too large to render. See raw diff