1
2
3
4
5
6
7
8
9
10
11
12
13
14
| from abc import ABC
| from abc import abstractmethod
| from typing import Iterable
| from typing import List
|
|
| class AbsTokenizer(ABC):
| @abstractmethod
| def text2tokens(self, line: str) -> List[str]:
| raise NotImplementedError
|
| @abstractmethod
| def tokens2text(self, tokens: Iterable[str]) -> str:
| raise NotImplementedError
|
|
