|
|
|
|
|
class PostProcessor: |
|
|
""" |
|
|
Base class for all post-processors |
|
|
|
|
|
This class is not supposed to be instantiated directly. Instead, any implementation of |
|
|
a PostProcessor will return an instance of this class when instantiated. |
|
|
""" |
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
|
|
class BertProcessing(PostProcessor): |
|
|
""" |
|
|
This post-processor takes care of adding the special tokens needed by |
|
|
a Bert model: |
|
|
|
|
|
- a SEP token |
|
|
- a CLS token |
|
|
|
|
|
Args: |
|
|
sep (:obj:`Tuple[str, int]`): |
|
|
A tuple with the string representation of the SEP token, and its id |
|
|
|
|
|
cls (:obj:`Tuple[str, int]`): |
|
|
A tuple with the string representation of the CLS token, and its id |
|
|
""" |
|
|
def __init__(self, sep, cls): |
|
|
pass |
|
|
|
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
|
|
class ByteLevel(PostProcessor): |
|
|
""" |
|
|
This post-processor takes care of trimming the offsets. |
|
|
|
|
|
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't |
|
|
want the offsets to include these whitespaces, then this PostProcessor must be used. |
|
|
|
|
|
Args: |
|
|
trim_offsets (:obj:`bool`): |
|
|
Whether to trim the whitespaces from the produced offsets. |
|
|
""" |
|
|
def __init__(self, trim_offsets=True): |
|
|
pass |
|
|
|
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
|
|
class RobertaProcessing(PostProcessor): |
|
|
""" |
|
|
This post-processor takes care of adding the special tokens needed by |
|
|
a Roberta model: |
|
|
|
|
|
- a SEP token |
|
|
- a CLS token |
|
|
|
|
|
It also takes care of trimming the offsets. |
|
|
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't |
|
|
want the offsets to include these whitespaces, then this PostProcessor should be initialized |
|
|
with :obj:`trim_offsets=True` |
|
|
|
|
|
Args: |
|
|
sep (:obj:`Tuple[str, int]`): |
|
|
A tuple with the string representation of the SEP token, and its id |
|
|
|
|
|
cls (:obj:`Tuple[str, int]`): |
|
|
A tuple with the string representation of the CLS token, and its id |
|
|
|
|
|
trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`): |
|
|
Whether to trim the whitespaces from the produced offsets. |
|
|
|
|
|
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): |
|
|
Whether the add_prefix_space option was enabled during pre-tokenization. This |
|
|
is relevant because it defines the way the offsets are trimmed out. |
|
|
""" |
|
|
def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True): |
|
|
pass |
|
|
|
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
|
|
class Sequence(PostProcessor): |
|
|
""" |
|
|
Sequence Processor |
|
|
|
|
|
Args: |
|
|
processors (:obj:`List[PostProcessor]`) |
|
|
The processors that need to be chained |
|
|
""" |
|
|
def __init__(self, processors): |
|
|
pass |
|
|
|
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
|
|
class TemplateProcessing(PostProcessor): |
|
|
""" |
|
|
Provides a way to specify templates in order to add the special tokens to each |
|
|
input sequence as relevant. |
|
|
|
|
|
Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to |
|
|
delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first |
|
|
sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair |
|
|
sequences. The final result looks like this: |
|
|
|
|
|
- Single sequence: :obj:`[CLS] Hello there [SEP]` |
|
|
- Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]` |
|
|
|
|
|
With the type ids as following:: |
|
|
|
|
|
[CLS] ... [SEP] ... [SEP] |
|
|
0 0 0 1 1 |
|
|
|
|
|
You can achieve such behavior using a TemplateProcessing:: |
|
|
|
|
|
TemplateProcessing( |
|
|
single="[CLS] $0 [SEP]", |
|
|
pair="[CLS] $A [SEP] $B:1 [SEP]:1", |
|
|
special_tokens=[("[CLS]", 1), ("[SEP]", 0)], |
|
|
) |
|
|
|
|
|
In this example, each input sequence is identified using a ``$`` construct. This identifier |
|
|
lets us specify each input sequence, and the type_id to use. When nothing is specified, |
|
|
it uses the default values. Here are the different ways to specify it: |
|
|
|
|
|
- Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B`` |
|
|
- Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ... |
|
|
- Specifying both: ``$A:0``, ``$B:1``, ... |
|
|
|
|
|
The same construct is used for special tokens: ``<identifier>(:<type_id>)?``. |
|
|
|
|
|
**Warning**: You must ensure that you are giving the correct tokens/ids as these |
|
|
will be added to the Encoding without any further check. If the given ids correspond |
|
|
to something totally different in a `Tokenizer` using this `PostProcessor`, it |
|
|
might lead to unexpected results. |
|
|
|
|
|
Args: |
|
|
single (:obj:`Template`): |
|
|
The template used for single sequences |
|
|
|
|
|
pair (:obj:`Template`): |
|
|
The template used when both sequences are specified |
|
|
|
|
|
special_tokens (:obj:`Tokens`): |
|
|
The list of special tokens used in each sequences |
|
|
|
|
|
Types: |
|
|
|
|
|
Template (:obj:`str` or :obj:`List`): |
|
|
- If a :obj:`str` is provided, the whitespace is used as delimiter between tokens |
|
|
- If a :obj:`List[str]` is provided, a list of tokens |
|
|
|
|
|
Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`): |
|
|
- A :obj:`Tuple` with both a token and its associated ID, in any order |
|
|
- A :obj:`dict` with the following keys: |
|
|
- "id": :obj:`str` => The special token id, as specified in the Template |
|
|
- "ids": :obj:`List[int]` => The associated IDs |
|
|
- "tokens": :obj:`List[str]` => The associated tokens |
|
|
|
|
|
The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have |
|
|
the same length. |
|
|
""" |
|
|
def __init__(self, single, pair, special_tokens): |
|
|
pass |
|
|
|
|
|
def num_special_tokens_to_add(self, is_pair): |
|
|
""" |
|
|
Return the number of special tokens that would be added for single/pair sentences. |
|
|
|
|
|
Args: |
|
|
is_pair (:obj:`bool`): |
|
|
Whether the input would be a pair of sequences |
|
|
|
|
|
Returns: |
|
|
:obj:`int`: The number of tokens to add |
|
|
""" |
|
|
pass |
|
|
|
|
|
def process(self, encoding, pair=None, add_special_tokens=True): |
|
|
""" |
|
|
Post-process the given encodings, generating the final one |
|
|
|
|
|
Args: |
|
|
encoding (:class:`~tokenizers.Encoding`): |
|
|
The encoding for the first sequence |
|
|
|
|
|
pair (:class:`~tokenizers.Encoding`, `optional`): |
|
|
The encoding for the pair sequence |
|
|
|
|
|
add_special_tokens (:obj:`bool`): |
|
|
Whether to add the special tokens |
|
|
|
|
|
Return: |
|
|
:class:`~tokenizers.Encoding`: The final encoding |
|
|
""" |
|
|
pass |
|
|
|
