Update tokenization_xgen.py
Browse files- tokenization_xgen.py +1 -0
tokenization_xgen.py
CHANGED
|
@@ -115,6 +115,7 @@ class XgenTokenizer(PreTrainedTokenizer):
|
|
| 115 |
def __init__(
|
| 116 |
self,
|
| 117 |
pad_token=None,
|
|
|
|
| 118 |
add_eos_token=False,
|
| 119 |
add_special_tokens=True,
|
| 120 |
**kwargs,
|
|
|
|
| 115 |
def __init__(
|
| 116 |
self,
|
| 117 |
pad_token=None,
|
| 118 |
+
eos_token=<|endoftext|>",
|
| 119 |
add_eos_token=False,
|
| 120 |
add_special_tokens=True,
|
| 121 |
**kwargs,
|