| | |
|---|
| | | sentences.append(words[sentence_len * word_limit:]) |
|---|
| | | return sentences |
|---|
| | | |
|---|
| | | def code_mix_split_words(text: str): |
|---|
| | | words = [] |
|---|
| | | segs = text.split() |
|---|
| | | for seg in segs: |
|---|
| | | # There is no space in seg. |
|---|
| | | current_word = "" |
|---|
| | | for c in seg: |
|---|
| | | if len(c.encode()) == 1: |
|---|
| | | # This is an ASCII char. |
|---|
| | | current_word += c |
|---|
| | | else: |
|---|
| | | # This is a Chinese char. |
|---|
| | | if len(current_word) > 0: |
|---|
| | | words.append(current_word) |
|---|
| | | current_word = "" |
|---|
| | | words.append(c) |
|---|
| | | if len(current_word) > 0: |
|---|
| | | words.append(current_word) |
|---|
| | | return words |
|---|
| | | |
|---|
| | | def read_yaml(yaml_path: Union[str, Path]) -> Dict: |
|---|
| | | if not Path(yaml_path).exists(): |
|---|
| | |
|---|
| | | |
|---|
| | | |
|---|
| | | @functools.lru_cache() |
|---|
| | | def get_logger(name='rapdi_paraformer'): |
|---|
| | | def get_logger(name='funasr_onnx'): |
|---|
| | | """Initialize and get a logger by name. |
|---|
| | | If the logger has not been initialized, this method will initialize the |
|---|
| | | logger by adding one or two handlers, otherwise the initialized logger will |
|---|
