| | |
|---|
| | | import yaml |
|---|
| | | from onnxruntime import (GraphOptimizationLevel, InferenceSession, |
|---|
| | | SessionOptions, get_available_providers, get_device) |
|---|
| | | from typeguard import check_argument_types |
|---|
| | | |
|---|
| | | import warnings |
|---|
| | | |
|---|
| | |
|---|
| | | class TokenIDConverter(): |
|---|
| | | def __init__(self, token_list: Union[List, str], |
|---|
| | | ): |
|---|
| | | check_argument_types() |
|---|
| | | |
|---|
| | | self.token_list = token_list |
|---|
| | | self.unk_symbol = token_list[-1] |
|---|
| | |
|---|
| | | space_symbol: str = "<space>", |
|---|
| | | remove_non_linguistic_symbols: bool = False, |
|---|
| | | ): |
|---|
| | | check_argument_types() |
|---|
| | | |
|---|
| | | self.space_symbol = space_symbol |
|---|
| | | self.non_linguistic_symbols = self.load_symbols(symbol_value) |
|---|
