Shitao commited on
Commit
0c5962b
·
verified ·
1 Parent(s): 7b03a43

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenization_qwen.py +250 -0
  2. tokenizer_config.json +1 -1
tokenization_qwen.py ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Copied from https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct/blob/main/tokenization_qwen.py
3
+ """
4
+ from typing import List, Optional
5
+ from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer as OriginalQwen2Tokenizer
6
+ from transformers.models.qwen2.tokenization_qwen2_fast import Qwen2TokenizerFast as OriginalQwen2TokenizerFast
7
+ from tokenizers import processors
8
+
9
+ VOCAB_FILES_NAMES = {
10
+ "vocab_file": "vocab.json",
11
+ "merges_file": "merges.txt",
12
+ "tokenizer_file": "tokenizer.json",
13
+ }
14
+
15
+ class Qwen2Tokenizer(OriginalQwen2Tokenizer):
16
+ """
17
+ Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
18
+ Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
19
+ be encoded differently whether it is at the beginning of the sentence (without space) or not:
20
+ ```python
21
+ >>> from transformers import Qwen2Tokenizer
22
+ >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
23
+ >>> tokenizer("Hello world")["input_ids"]
24
+ [9707, 1879]
25
+ >>> tokenizer(" Hello world")["input_ids"]
26
+ [21927, 1879]
27
+ ```
28
+ This is expected.
29
+ You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
30
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
31
+ this superclass for more information regarding those methods.
32
+ Args:
33
+ vocab_file (`str`):
34
+ Path to the vocabulary file.
35
+ merges_file (`str`):
36
+ Path to the merges file.
37
+ errors (`str`, *optional*, defaults to `"replace"`):
38
+ Paradigm to follow when decoding bytes to UTF-8. See
39
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
40
+ unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
41
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
42
+ token instead.
43
+ bos_token (`str`, *optional*):
44
+ The beginning of sequence token. Not applicable for this tokenizer.
45
+ eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
46
+ The end of sequence token.
47
+ pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
48
+ The token used for padding, for example when batching sequences of different lengths.
49
+ clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
50
+ Whether or not the model should cleanup the spaces that were added when splitting the input text during the
51
+ tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
52
+ split_special_tokens (`bool`, *optional*, defaults to `False`):
53
+ Whether or not the special tokens should be split during the tokenization process. The default behavior is
54
+ to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
55
+ ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
56
+ '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
57
+ add_eos_token (`bool`, *optional*, defaults to `False`):
58
+ Whether or not to add an `eos_token` at the end of sequences.
59
+ """
60
+
61
+ def __init__(
62
+ self,
63
+ vocab_file,
64
+ merges_file,
65
+ errors="replace",
66
+ unk_token="<|endoftext|>",
67
+ bos_token=None,
68
+ eos_token="<|endoftext|>",
69
+ pad_token="<|endoftext|>",
70
+ clean_up_tokenization_spaces=False,
71
+ split_special_tokens=False,
72
+ add_eos_token=False,
73
+ **kwargs,
74
+ ):
75
+ # The add_eos_token code was inspired by the LlamaTokenizer
76
+ self.add_eos_token = add_eos_token
77
+
78
+ super().__init__(
79
+ vocab_file=vocab_file,
80
+ merges_file=merges_file,
81
+ errors=errors,
82
+ unk_token=unk_token,
83
+ bos_token=bos_token,
84
+ eos_token=eos_token,
85
+ pad_token=pad_token,
86
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
87
+ split_special_tokens=split_special_tokens,
88
+ add_eos_token=add_eos_token,
89
+ **kwargs,
90
+ )
91
+
92
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
93
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
94
+
95
+ output = token_ids_0 + eos_token_id
96
+
97
+ if token_ids_1 is not None:
98
+ output = output + token_ids_1 + eos_token_id
99
+
100
+ return output
101
+
102
+ def get_special_tokens_mask(
103
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
104
+ ) -> List[int]:
105
+ """
106
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
107
+ special tokens using the tokenizer `prepare_for_model` method.
108
+ Args:
109
+ token_ids_0 (`List[int]`):
110
+ List of IDs.
111
+ token_ids_1 (`List[int]`, *optional*):
112
+ Optional second list of IDs for sequence pairs.
113
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
114
+ Whether or not the token list is already formatted with special tokens for the model.
115
+ Returns:
116
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
117
+ """
118
+ if already_has_special_tokens:
119
+ return super().get_special_tokens_mask(
120
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
121
+ )
122
+
123
+ eos_token_id = [1] if self.add_eos_token else []
124
+
125
+ if token_ids_1 is None:
126
+ return ([0] * len(token_ids_0)) + eos_token_id
127
+ return (
128
+ ([0] * len(token_ids_0))
129
+ + eos_token_id
130
+ + ([0] * len(token_ids_1))
131
+ + eos_token_id
132
+ )
133
+
134
+ def create_token_type_ids_from_sequences(
135
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
136
+ ) -> List[int]:
137
+ """
138
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
139
+ sequence pair mask has the following format:
140
+ ```
141
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
142
+ | first sequence | second sequence |
143
+ ```
144
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
145
+ Args:
146
+ token_ids_0 (`List[int]`):
147
+ List of ids.
148
+ token_ids_1 (`List[int]`, *optional*):
149
+ Optional second list of IDs for sequence pairs.
150
+ Returns:
151
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
152
+ """
153
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
154
+
155
+ output = [0] * len(token_ids_0 + eos_token_id)
156
+
157
+ if token_ids_1 is not None:
158
+ output += [1] * len(token_ids_1 + eos_token_id)
159
+
160
+ return output
161
+
162
+ class Qwen2TokenizerFast(OriginalQwen2TokenizerFast):
163
+ """
164
+ Construct a "fast" Qwen2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
165
+ Byte-Pair-Encoding.
166
+ Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
167
+ be encoded differently whether it is at the beginning of the sentence (without space) or not:
168
+ ```python
169
+ >>> from transformers import Qwen2TokenizerFast
170
+ >>> tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen-tokenizer")
171
+ >>> tokenizer("Hello world")["input_ids"]
172
+ [9707, 1879]
173
+ >>> tokenizer(" Hello world")["input_ids"]
174
+ [21927, 1879]
175
+ ```
176
+ This is expected.
177
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
178
+ refer to this superclass for more information regarding those methods.
179
+ Args:
180
+ vocab_file (`str`, *optional*):
181
+ Path to the vocabulary file.
182
+ merges_file (`str`, *optional*):
183
+ Path to the merges file.
184
+ tokenizer_file (`str`, *optional*):
185
+ Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
186
+ contains everything needed to load the tokenizer.
187
+ unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
188
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
189
+ token instead. Not applicable to this tokenizer.
190
+ bos_token (`str`, *optional*):
191
+ The beginning of sequence token. Not applicable for this tokenizer.
192
+ eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
193
+ The end of sequence token.
194
+ pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
195
+ The token used for padding, for example when batching sequences of different lengths.
196
+ add_eos_token (`bool`, *optional*, defaults to `False`):
197
+ Whether or not to add an `eos_token` at the end of sequences.
198
+ """
199
+
200
+ slow_tokenizer_class = Qwen2Tokenizer
201
+ padding_side = "left"
202
+
203
+ def __init__(
204
+ self,
205
+ vocab_file=None,
206
+ merges_file=None,
207
+ tokenizer_file=None,
208
+ unk_token="<|endoftext|>",
209
+ bos_token=None,
210
+ eos_token="<|endoftext|>",
211
+ pad_token="<|endoftext|>",
212
+ add_eos_token=False,
213
+ **kwargs,
214
+ ):
215
+ super().__init__(
216
+ vocab_file=vocab_file,
217
+ merges_file=merges_file,
218
+ tokenizer_file=tokenizer_file,
219
+ unk_token=unk_token,
220
+ bos_token=bos_token,
221
+ eos_token=eos_token,
222
+ pad_token=pad_token,
223
+ **kwargs,
224
+ )
225
+
226
+ self._add_eos_token = add_eos_token
227
+ self.update_post_processor()
228
+
229
+ def update_post_processor(self):
230
+ """
231
+ Updates the underlying post processor with the current `eos_token`.
232
+ """
233
+ eos = self.eos_token
234
+ eos_token_id = self.eos_token_id
235
+ if eos is None and self.add_eos_token:
236
+ raise ValueError("add_eos_token = True but eos_token = None")
237
+
238
+ single = f"$A:0{(' '+eos+':0') if self.add_eos_token else ''}"
239
+ pair = f"{single} $B:1{(' '+eos+':1') if self.add_eos_token else ''}"
240
+
241
+ special_tokens = []
242
+ if self.add_eos_token:
243
+ special_tokens.append((eos, eos_token_id))
244
+ self._tokenizer.post_processor = processors.TemplateProcessing(
245
+ single=single, pair=pair, special_tokens=special_tokens
246
+ )
247
+
248
+ @property
249
+ def add_eos_token(self):
250
+ return self._add_eos_token
tokenizer_config.json CHANGED
@@ -202,7 +202,7 @@
202
  ],
203
  "auto_map": {
204
  "AutoTokenizer": [
205
- "/share_2/chaofan/models/CodeR-full--tokenization_qwen.Qwen2Tokenizer",
206
  null
207
  ]
208
  },
 
202
  ],
203
  "auto_map": {
204
  "AutoTokenizer": [
205
+ "tokenization_qwen.Qwen2Tokenizer",
206
  null
207
  ]
208
  },