Spaces:
Running
Running
File size: 691 Bytes
224a33f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
from typing import Protocol, runtime_checkable
@runtime_checkable
class EsmTokenizerBase(Protocol):
def encode(self, *args, **kwargs):
...
def decode(self, *args, **kwargs):
...
@property
def mask_token(self) -> str:
...
@property
def mask_token_id(self) -> int:
...
@property
def bos_token(self) -> str:
...
@property
def bos_token_id(self) -> int:
...
@property
def eos_token(self) -> str:
...
@property
def eos_token_id(self) -> int:
...
@property
def pad_token(self) -> str:
...
@property
def pad_token_id(self) -> int:
...
|