tokenization_hlm.py

import os
import json
import unicodedata
from typing import Any, Dict, List, Optional, Tuple, Union
from collections.abc import Mapping
from collections import Counter
import itertools
import torch

from transformers.tokenization_utils import PreTrainedTokenizer, PaddingStrategy, TruncationStrategy, TensorType, BatchEncoding
from transformers.utils import logging, is_torch_tensor

TextInput = str
PreTokenizedInput = List[str]
EncodedInput = List[List[int]]
TextInputPair = Tuple[TextInput, TextInput]
PreTokenizedInputPair = Tuple[PreTokenizedInput, PreTokenizedInput]
EncodedInputPair = Tuple[EncodedInput, EncodedInput]

logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json"}

# TODO: add support for return_offsets_mapping

class HLMTokenizer(PreTrainedTokenizer):
    r"""
    Constructs a HLM tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).

    Args:
        vocab_file (`str`):
            Path to .json vocab file.
        bos_token (`string`, *optional*, defaults to `"[CLS]"`):
            The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token.
            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.
        eos_token (`string`, *optional*, defaults to `"[SEP]"`):
            The end of sequence token. When building a sequence using special tokens, this is not the token that is
            used for the end of sequence. The token used is the `sep_token`.
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        word_cls_token (`str`, *optional*, defaults to `"[WORD_CLS]"`):
            The classifier token which is used for word representations and word classification.
            It is the first token of each word when built with special tokens.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    model_input_names: List[str] = ["input_ids", "char_input_mask", "word_input_mask", "word_type_ids"]

    def __init__(
        self,
        vocab_file,
        split_by_punct=False,
        bos_token="[CLS]",
        eos_token="[SEP]",
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        word_cls_token="[WORD_CLS]",
        max_word_length=None,
        model_max_length=None,
        **kwargs,
    ) -> None:
        if not os.path.isfile(vocab_file):
            raise ValueError(
                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a pretrained"
                " model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
            )

        if max_word_length is not None:
            self.max_word_length = max_word_length
        else:
            try:
                with open(os.path.dirname(vocab_file) + "/config.json", "r") as f:
                    config = json.load(f)
                    self.max_word_length = config["max_word_length"]
                    if model_max_length is None:
                        model_max_length = config.get("max_seq_length", None)
            except:
                raise ValueError("Failed to load max_word_length from config.json. Please specify max_word_length.")

        self.split_by_punct = split_by_punct
        self.vocab_file = vocab_file
        with open(vocab_file, 'r', encoding='utf-8') as f:
            vocab_data = json.load(f)
            self.vocab = vocab_data["vocab"]
            self.inv_vocab = {v: k for k, v in self.vocab.items()}

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            split_by_punct=split_by_punct,
            model_max_length=model_max_length,
            **kwargs,
        )
        self.unk_id = self.vocab["[UNK]"]
        self.word_cls_token = word_cls_token
        self.word_cls_token_id = self._convert_token_to_id(word_cls_token)
        self.label_pad_token_id = -100
        self.special_ids = [self._convert_token_to_id(token) for token in vocab_data["special_tokens"]]

        #self.pad_word = [[self.word_cls_token_id] + [0]*(self.max_word_length-1)]
        #self.pad_mask_word = [[1] + [0]*(self.max_word_length-1)]
        self.pad_word = [[0] + [0]*(self.max_word_length-1)]
        self.pad_mask_word = [[0] + [0]*(self.max_word_length-1)]

    @staticmethod
    def train(files: List[Union[str, os.PathLike]], output_dir: Union[str, os.PathLike], vocab_size: int=512, max_lines_to_consider=2_000_000):
        char_maps = []
        # Each input file is weighted equally, regardless of size
        # This is to prevent one language from dominating the character distribution
        for file in files:
            print('Loading char counts from', file)
            counter = Counter()
            line_count = 0
            with open(file, "r", encoding="utf-8") as file:
                while line_count < max_lines_to_consider:
                    lines = file.readlines(100*1024)
                    if len(lines) == 0:
                        break
                    for line in lines:
                        line = unicodedata.normalize('NFKC', line)
                        line_count += 1
                        counter.update(line)
            d = {}
            total = counter.total()
            for char, count in counter.items():
                d[char] = count / total
            char_maps.append(d)

        char_map = {}
        for d in char_maps:
            for char, freq in d.items():
                if not char.isspace():
                    char_map[char] = char_map.get(char, 0) + freq

        special_tokens = ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '[WORD_CLS]']
        chars_to_keep = sorted(list(char_map.keys()), key=lambda c: char_map[c], reverse=True)[:vocab_size-len(special_tokens)]
        vocab_entries = [*special_tokens, *chars_to_keep]

        vocab = {
            'special_tokens': special_tokens,
            'vocab': { key: i for i, key in enumerate(vocab_entries) }
        }

        assert(len(vocab_entries) == vocab_size)

        filename = os.path.join(output_dir, VOCAB_FILES_NAMES["vocab_file"])
        os.makedirs(output_dir, exist_ok=True)
        print("Saving vocab to", filename)
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(vocab, f, ensure_ascii=False, indent=4)

        return filename

    @property
    def vocab_size(self):
        return len(self.vocab)

    def get_vocab(self):
        return self.vocab

    def _convert_token_to_id(self, token):
        """Converts a token (str) to an id using the vocab."""
        return self.vocab.get(token, self.unk_id)

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.inv_vocab[index] if index < self.vocab_size else self.unk_token

    def convert_tokens_to_ids(self, tokens: Union[str, List[str], List[List[str]]]):
        if isinstance(tokens, str):
            return self._convert_token_to_id(tokens)
        if len(tokens) > 0 and isinstance(tokens[0], str):
            return [self._convert_token_to_id(token) for token in tokens]
        return [[self._convert_token_to_id(token) for token in word] for word in tokens]

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        raise NotImplementedError

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        if token_ids_1 is None:
            return [[self.cls_token_id]] + token_ids_0 + [[self.eos_token_id]]
        return [[self.cls_token_id]] + token_ids_0 + [[self.eos_token_id], [self.cls_token_id]] + token_ids_1 + [[self.eos_token_id]]

    def num_special_tokens_to_add(self, pair: bool = False) -> int:
        return 3 if pair else 2

    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
        raise NotImplementedError

    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None, has_special_tokens=False):
        if has_special_tokens:
            return [0] * (len(token_ids_0)+2) + ([1] * (len(token_ids_1)+2) if token_ids_1 is not None else [])
        else:
            return [0] * len(token_ids_0) + ([1] * len(token_ids_1) if token_ids_1 is not None else [])

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        filename = VOCAB_FILES_NAMES["vocab_file"]
        if filename_prefix is not None:
            filename = filename_prefix + "-" + filename
        full_path = os.path.join(save_directory, filename)
        with open(full_path, "w", encoding="utf-8") as f:
            json.dump({
                "special_tokens": self.all_special_tokens,
                "vocab": self.get_vocab(),
            }, f, ensure_ascii=False, indent=4)
        return (full_path,)

    def encode(
        self,
        text: Union[TextInput, PreTokenizedInput, EncodedInput],
        text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
        is_split_into_words: bool = False,
        add_special_tokens: bool = False,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> List[int]:
        def get_input_ids(text):
            if isinstance(text, str):
                tokens = self.tokenize(text, **kwargs)
                return self.convert_tokens_to_ids(tokens)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):
                if is_split_into_words:
                    tokens = list(
                        itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))
                    )
                    return self.convert_tokens_to_ids(tokens)
                else:
                    return self.convert_tokens_to_ids(text)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], List[int]):
                return text
            else:
                raise ValueError(
                    f"Input {text} is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.")

        first_ids = get_input_ids(text)
        second_ids = get_input_ids(text_pair) if text_pair is not None else None

        if add_special_tokens:
            sequence = self.build_inputs_with_special_tokens(first_ids, second_ids)
        else:
            sequence = first_ids

        return sequence

    def prepare_for_model(
        self,
        ids: List[List[int]],
        pair_ids: Optional[List[List[int]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: bool = True,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        add_word_cls: bool = True,
        prepend_batch_axis: bool = False,
        **kwargs,
    ) -> BatchEncoding:
        """
        Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
        adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
        manages a moving window (with user defined stride) for overflowing tokens.

        Args:
            ids (`List[List[int]]`):
                Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize` and
                `convert_tokens_to_ids` methods.
            pair_ids (`List[List[int]]`, *optional*):
                Tokenized input ids of the second sequence. Can be obtained from a string by chaining the `tokenize`
                and `convert_tokens_to_ids` methods.
        """

        # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        pair = bool(pair_ids is not None)
        len_pair_ids = len(pair_ids) if pair else 0

        if return_token_type_ids and not add_special_tokens:
            raise ValueError(
                "Asking to return token_type_ids while setting add_special_tokens to False "
                "results in an undefined behavior. Please set add_special_tokens to True or "
                "set return_token_type_ids to None."
            )

        if (
            return_overflowing_tokens
            and truncation_strategy == TruncationStrategy.LONGEST_FIRST
            and pair_ids is not None
        ):
            raise ValueError(
                "Not possible to return overflowing tokens for pair of sequences with the "
                "`longest_first`. Please select another truncation strategy than `longest_first`, "
                "for instance `only_second` or `only_first`."
            )

        encoded_inputs = {}

        # Compute the total size of the returned encodings
        total_len = len(ids) + len_pair_ids + (self.num_special_tokens_to_add(pair=pair) if add_special_tokens else 0)

        # Truncation: Handle max sequence length
        overflowing_tokens = []
        if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and total_len > max_length:
            ids, pair_ids, overflowing_tokens = self.truncate_sequences(
                ids,
                pair_ids=pair_ids,
                num_tokens_to_remove=total_len - max_length,
                truncation_strategy=truncation_strategy,
                stride=stride,
            )

        if return_overflowing_tokens:
            encoded_inputs["overflowing_tokens"] = overflowing_tokens
            encoded_inputs["num_truncated_tokens"] = total_len - max_length

        if add_special_tokens:
            sequence = self.build_inputs_with_special_tokens(ids, pair_ids)
        else:
            sequence = ids + pair_ids if pair else ids

        if add_word_cls:
            for word in sequence:
                word.insert(0, self.word_cls_token_id)

        # Build output dictionary
        encoded_inputs["input_ids"] = sequence
        encoded_inputs["char_input_mask"] = [[1]*len(word)+[0]*(self.max_word_length-len(word)) for word in sequence]
        encoded_inputs["word_input_mask"] = [1]*len(sequence)
        if return_token_type_ids or pair:
            encoded_inputs["word_type_ids"] = self.create_token_type_ids_from_sequences(ids, pair_ids, add_special_tokens)
            assert len(encoded_inputs["word_type_ids"]) == len(encoded_inputs["word_input_mask"])

        # Always pad words
        for word in encoded_inputs["input_ids"]:
            if len(word) < self.max_word_length:
                word.extend([self.pad_token_id] * (self.max_word_length - len(word)))

        # Padding
        if padding_strategy != PaddingStrategy.DO_NOT_PAD or return_attention_mask:
            encoded_inputs = self.pad(
                encoded_inputs,
                max_length=max_length,
                padding=padding_strategy.value,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
            )

        batch_outputs = BatchEncoding(
            encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis
        )

        return batch_outputs

    def _encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput, EncodedInput],
        text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        add_word_cls: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        def get_input_ids(text):
            if isinstance(text, str):
                tokens = self.tokenize(text, **kwargs)
                return self.convert_tokens_to_ids(tokens)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):
                if is_split_into_words:
                    tokens = list(
                        itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))
                    )
                    return self.convert_tokens_to_ids(tokens)
                else:
                    return self.convert_tokens_to_ids(text)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], List[int]):
                return text
            else:
                raise ValueError(
                    f"Input {text} is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.")

        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast. "
                "More information on available tokenizers at "
                "https://github.com/huggingface/transformers/pull/2674"
            )

        first_ids = get_input_ids(text)
        second_ids = get_input_ids(text_pair) if text_pair is not None else None

        return self.prepare_for_model(
            first_ids,
            pair_ids=second_ids,
            add_special_tokens=add_special_tokens,
            padding=padding_strategy.value,
            truncation=truncation_strategy.value,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            prepend_batch_axis=True,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            verbose=verbose,
            add_word_cls=add_word_cls,
        )

    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
            List[PreTokenizedInputPair],
            List[EncodedInput],
            List[EncodedInputPair],
        ],
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        def get_input_ids(text):
            if isinstance(text, str):
                tokens = self.tokenize(text, **kwargs)
                return self.convert_tokens_to_ids(tokens)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):
                if is_split_into_words:
                    tokens = list(
                        itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))
                    )
                    return self.convert_tokens_to_ids(tokens)
                else:
                    return self.convert_tokens_to_ids(text)
            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], List[int]):
                return text
            else:
                raise ValueError(
                    "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
                )

        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast."
            )

        input_ids = []
        for ids_or_pair_ids in batch_text_or_text_pairs:
            if not isinstance(ids_or_pair_ids, (list, tuple)):
                ids, pair_ids = ids_or_pair_ids, None
            elif is_split_into_words and not isinstance(ids_or_pair_ids[0], (list, tuple)):
                ids, pair_ids = ids_or_pair_ids, None
            else:
                ids, pair_ids = ids_or_pair_ids

            first_ids = get_input_ids(ids)
            second_ids = get_input_ids(pair_ids) if pair_ids is not None else None
            input_ids.append((first_ids, second_ids))

        batch_outputs = self._batch_prepare_for_model(
            input_ids,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            return_tensors=return_tensors,
            verbose=verbose,
        )

        return BatchEncoding(batch_outputs)

    def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, split_long_words: bool = True, **kwargs) -> List[List[str]]:
        text = unicodedata.normalize('NFKC', text)
        if split_long_words:
            tokenized_text = []
            for token in text.split():
                tokens = [char for char in token]
                tokenized_text.extend(
                    tokens[i: i + self.max_word_length - 1] for i in range(0, len(tokens), self.max_word_length - 1))
            return tokenized_text
        else:
            return [[char for char in token] for token in text.split()]

    def pad(
        self,
        encoded_inputs: Union[
            BatchEncoding,
            List[BatchEncoding],
            Dict[str, EncodedInput],
            Dict[str, List[EncodedInput]],
            List[Dict[str, EncodedInput]],
        ],
        padding: Union[bool, str, PaddingStrategy] = True,
        max_length: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None, # TODO: add support for pad_to_multiple_of
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        #label_pad_token_id=-100,
        verbose: bool = True,
        **kwargs
    ) -> BatchEncoding:
        # If we have a list of dicts, let's convert it in a dict of lists
        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
        if isinstance(encoded_inputs, (list, tuple)) and isinstance(encoded_inputs[0], Mapping):
            encoded_inputs = {key: [example[key] for example in encoded_inputs] for key in encoded_inputs[0].keys()}

        # The model's main input name, usually `input_ids`, has be passed for padding
        #if self.model_input_names[0] not in encoded_inputs:
        #    raise ValueError(
        #        "You should supply an encoding or a list of encodings to this method "
        #        f"that includes {self.model_input_names[0]}, but you provided {list(encoded_inputs.keys())}"
        #    )

        required_input = encoded_inputs["input_ids"]

        #if required_input is None or (isinstance(required_input, Sized) and len(required_input) == 0):
        #    if return_attention_mask:
        #        encoded_inputs["char_input_mask"] = []
        #        encoded_inputs["word_input_mask"] = []
        #    return encoded_inputs

        # If we have PyTorch/TF/NumPy tensors/arrays as inputs, we cast them as python objects
        # and rebuild them afterwards if no return_tensors is specified
        # Note that we lose the specific device the tensor may be on for PyTorch

        #first_element = required_input[0]
        ## At this state, if `first_element` is still a list/tuple, it's an empty one so there is nothing to do.
        #if not isinstance(first_element, (int, list, tuple)):
        #    if is_torch_tensor(first_element):
        #        return_tensors = "pt" if return_tensors is None else return_tensors

        #    for key, value in encoded_inputs.items():
        #        encoded_inputs[key] = to_py_obj(value)

        # Convert padding_strategy in PaddingStrategy
        padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
            padding=padding, max_length=max_length, verbose=verbose)

        if padding_strategy == PaddingStrategy.DO_NOT_PAD:
            return encoded_inputs

        assert (padding_strategy == PaddingStrategy.LONGEST)

        longest_in_batch = max(len(f) for f in required_input)
        batch_outputs = {}
        batch_outputs["input_ids"] = [f + self.pad_word*(longest_in_batch - len(f)) for f in encoded_inputs["input_ids"]]
        batch_outputs["char_input_mask"] = [f + self.pad_mask_word*(longest_in_batch - len(f)) for f in encoded_inputs["char_input_mask"]]

        batch_outputs["word_input_mask"] = \
            [f + [0]*(longest_in_batch - len(f)) for f in encoded_inputs['word_input_mask']]

        if "word_type_ids" in encoded_inputs:
            batch_outputs["word_type_ids"] = [f + [0]*(longest_in_batch - len(f)) for f in encoded_inputs["word_type_ids"]]

        batch_outputs["char_input_mask"] = torch.tensor(batch_outputs["char_input_mask"], dtype=torch.bool)
        batch_outputs["word_input_mask"] = torch.tensor(batch_outputs["word_input_mask"], dtype=torch.bool)

        # TODO: move label names elsewhere
        label_fields = ('labels', 'upos', 'feats', 'heads', 'deprels', 'lemmas')
        label_names = [feature for feature in encoded_inputs.keys() if feature in label_fields]

        if len(label_names) > 0:
            def to_list(tensor_or_iterable):
                if is_torch_tensor(tensor_or_iterable):
                    return tensor_or_iterable.tolist()
                return list(tensor_or_iterable)

            for label_name in label_names:
                if label_name not in encoded_inputs:
                    continue
                labels = encoded_inputs[label_name]
                label_pad_word = [[self.label_pad_token_id]*self.max_word_length]
                batch_outputs[label_name] = [
                    to_list(label) + label_pad_word * (longest_in_batch - len(label)) for label in labels
                ]

        return BatchEncoding(batch_outputs, tensor_type=return_tensors)