File size: 347 Bytes
ad16788 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
from abc import ABC
from abc import abstractmethod
from typing import Iterable
from typing import List
class AbsTokenizer(ABC):
@abstractmethod
def text2tokens(self, line: str) -> List[str]:
raise NotImplementedError
@abstractmethod
def tokens2text(self, tokens: Iterable[str]) -> str:
raise NotImplementedError
|