Upload tokenizer
Browse files- tokenization_vnsabsa.py +6 -0
tokenization_vnsabsa.py
CHANGED
|
@@ -11,10 +11,16 @@ import os
|
|
| 11 |
import requests
|
| 12 |
|
| 13 |
class VnSmartphoneAbsaTokenizer(PreTrainedTokenizer):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
pretrained_vocab_files_map = {
|
| 15 |
"vocab_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/vocab.txt",
|
| 16 |
"merge_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/merge.txt"
|
| 17 |
}
|
|
|
|
|
|
|
| 18 |
def __init__(
|
| 19 |
self,
|
| 20 |
vocab_file,
|
|
|
|
| 11 |
import requests
|
| 12 |
|
| 13 |
class VnSmartphoneAbsaTokenizer(PreTrainedTokenizer):
|
| 14 |
+
vocab_files_names = {
|
| 15 |
+
"vocab_file": "vocab.txt",
|
| 16 |
+
"merges_file": "merge.txt",
|
| 17 |
+
}
|
| 18 |
pretrained_vocab_files_map = {
|
| 19 |
"vocab_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/vocab.txt",
|
| 20 |
"merge_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/merge.txt"
|
| 21 |
}
|
| 22 |
+
model_input_names = ["input_ids", "attention_mask"]
|
| 23 |
+
|
| 24 |
def __init__(
|
| 25 |
self,
|
| 26 |
vocab_file,
|