Llama-3.1-8B-DALv0.1
/
venv
/lib
/python3.12
/site-packages
/transformers
/models
/rembert
/tokenization_rembert.py
| # coding=utf-8 | |
| # Copyright The HuggingFace Team and The HuggingFace Inc. team. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Tokenization classes for RemBERT.""" | |
| import os | |
| from shutil import copyfile | |
| from typing import List, Optional, Tuple | |
| import sentencepiece as spm | |
| from ...tokenization_utils import AddedToken, PreTrainedTokenizer | |
| from ...utils import logging | |
| logger = logging.get_logger(__name__) | |
| VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.model"} | |
| class RemBertTokenizer(PreTrainedTokenizer): | |
| """ | |
| Construct a RemBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). | |
| This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to | |
| this superclass for more information regarding those methods. | |
| Args: | |
| vocab_file (`str`): | |
| [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that | |
| contains the vocabulary necessary to instantiate a tokenizer. | |
| bos_token (`str`, *optional*, defaults to `"[CLS]"`): | |
| The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. | |
| <Tip> | |
| When building a sequence using special tokens, this is not the token that is used for the beginning of | |
| sequence. The token used is the `cls_token`. | |
| </Tip> | |
| eos_token (`str`, *optional*, defaults to `"[SEP]"`): | |
| The end of sequence token. | |
| <Tip> | |
| When building a sequence using special tokens, this is not the token that is used for the end of sequence. | |
| The token used is the `sep_token`. | |
| </Tip> | |
| unk_token (`str`, *optional*, defaults to `"<unk>"`): | |
| The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this | |
| token instead. | |
| sep_token (`str`, *optional*, defaults to `"[SEP]"`): | |
| The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for | |
| sequence classification or for a text and a question for question answering. It is also used as the last | |
| token of a sequence built with special tokens. | |
| pad_token (`str`, *optional*, defaults to `"<pad>"`): | |
| The token used for padding, for example when batching sequences of different lengths. | |
| cls_token (`str`, *optional*, defaults to `"[CLS]"`): | |
| The classifier token which is used when doing sequence classification (classification of the whole sequence | |
| instead of per-token classification). It is the first token of the sequence when built with special tokens. | |
| mask_token (`str`, *optional*, defaults to `"[MASK]"`): | |
| The token used for masking values. This is the token used when training this model with masked language | |
| modeling. This is the token which the model will try to predict. | |
| Attributes: | |
| sp_model (`SentencePieceProcessor`): | |
| The *SentencePiece* processor that is used for every conversion (string, tokens and IDs). | |
| """ | |
| vocab_files_names = VOCAB_FILES_NAMES | |
| def __init__( | |
| self, | |
| vocab_file, | |
| do_lower_case=False, | |
| remove_space=True, | |
| keep_accents=True, | |
| bos_token="[CLS]", | |
| eos_token="[SEP]", | |
| unk_token="[UNK]", | |
| sep_token="[SEP]", | |
| pad_token="[PAD]", | |
| cls_token="[CLS]", | |
| mask_token="[MASK]", | |
| **kwargs, | |
| ): | |
| # Mask token behave like a normal word, i.e. include the space before it | |
| mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token | |
| self.do_lower_case = do_lower_case | |
| self.remove_space = remove_space | |
| self.keep_accents = keep_accents | |
| self.vocab_file = vocab_file | |
| self.sp_model = spm.SentencePieceProcessor() | |
| self.sp_model.Load(vocab_file) | |
| super().__init__( | |
| do_lower_case=do_lower_case, | |
| remove_space=remove_space, | |
| keep_accents=keep_accents, | |
| bos_token=bos_token, | |
| eos_token=eos_token, | |
| unk_token=unk_token, | |
| sep_token=sep_token, | |
| pad_token=pad_token, | |
| cls_token=cls_token, | |
| mask_token=mask_token, | |
| **kwargs, | |
| ) | |
| def vocab_size(self): | |
| return len(self.sp_model) | |
| def get_vocab(self): | |
| vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} | |
| vocab.update(self.added_tokens_encoder) | |
| return vocab | |
| def __getstate__(self): | |
| state = self.__dict__.copy() | |
| state["sp_model"] = None | |
| return state | |
| def __setstate__(self, d): | |
| self.__dict__ = d | |
| self.sp_model = spm.SentencePieceProcessor() | |
| self.sp_model.Load(self.vocab_file) | |
| def _tokenize(self, text, sample=False): | |
| """Tokenize a string.""" | |
| pieces = self.sp_model.EncodeAsPieces(text) | |
| return pieces | |
| def _convert_token_to_id(self, token): | |
| """Converts a token (str) in an id using the vocab.""" | |
| return self.sp_model.PieceToId(token) | |
| def _convert_id_to_token(self, index): | |
| """Converts an index (integer) in a token (str) using the vocab.""" | |
| return self.sp_model.IdToPiece(index) | |
| def convert_tokens_to_string(self, tokens): | |
| out_string = self.sp_model.decode_pieces(tokens) | |
| return out_string | |
| def build_inputs_with_special_tokens( | |
| self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None | |
| ) -> List[int]: | |
| """ | |
| Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and | |
| adding special tokens. A REMBERT sequence has the following format: | |
| - single sequence: `[CLS] X [SEP]` | |
| - pair of sequences: `[CLS] A [SEP] B [SEP]` | |
| Args: | |
| token_ids_0 (`List[int]`): | |
| List of IDs to which the special tokens will be added. | |
| token_ids_1 (`List[int]`, *optional*): | |
| Optional second list of IDs for sequence pairs. | |
| Returns: | |
| `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. | |
| """ | |
| sep = [self.sep_token_id] | |
| cls = [self.cls_token_id] | |
| if token_ids_1 is None: | |
| return cls + token_ids_0 + sep | |
| return cls + token_ids_0 + sep + token_ids_1 + sep | |
| def get_special_tokens_mask( | |
| self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False | |
| ) -> List[int]: | |
| """ | |
| Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding | |
| special tokens using the tokenizer `prepare_for_model` method. | |
| Args: | |
| token_ids_0 (`List[int]`): | |
| List of IDs. | |
| token_ids_1 (`List[int]`, *optional*): | |
| Optional second list of IDs for sequence pairs. | |
| already_has_special_tokens (`bool`, *optional*, defaults to `False`): | |
| Whether or not the token list is already formatted with special tokens for the model. | |
| Returns: | |
| `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. | |
| """ | |
| if already_has_special_tokens: | |
| if token_ids_1 is not None: | |
| raise ValueError( | |
| "You should not supply a second sequence if the provided sequence of " | |
| "ids is already formatted with special tokens for the model." | |
| ) | |
| return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_0] | |
| if token_ids_1 is not None: | |
| return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] | |
| return [1] + ([0] * len(token_ids_0)) + [1] | |
| def create_token_type_ids_from_sequences( | |
| self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None | |
| ) -> List[int]: | |
| """ | |
| Create a mask from the two sequences passed to be used in a sequence-pair classification task. A RemBERT | |
| sequence pair mask has the following format: | |
| ``` | |
| 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 | |
| | first sequence | second sequence | | |
| ``` | |
| If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). | |
| Args: | |
| token_ids_0 (`List[int]`): | |
| List of IDs. | |
| token_ids_1 (`List[int]`, *optional*): | |
| Optional second list of IDs for sequence pairs. | |
| Returns: | |
| `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). | |
| """ | |
| sep = [self.sep_token_id] | |
| cls = [self.cls_token_id] | |
| if token_ids_1 is None: | |
| return len(cls + token_ids_0 + sep) * [0] | |
| return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] | |
| def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: | |
| if not os.path.isdir(save_directory): | |
| logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) | |
| return | |
| out_vocab_file = os.path.join( | |
| save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] | |
| ) | |
| if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file): | |
| copyfile(self.vocab_file, out_vocab_file) | |
| elif not os.path.isfile(self.vocab_file): | |
| with open(out_vocab_file, "wb") as fi: | |
| content_spiece_model = self.sp_model.serialized_model_proto() | |
| fi.write(content_spiece_model) | |
| return (out_vocab_file,) | |