From c2f174cd7811a7a11e6227ecb371887f97fd66d3 Mon Sep 17 00:00:00 2001
From: 夜雨飘零 <yeyupiaoling@foxmail.com>
Date: 星期五, 02 二月 2024 23:06:13 +0800
Subject: [PATCH] Use ffmpeg read data (#1349)
---
funasr/tokenizer/char_tokenizer.py | 12 +++++++-----
1 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 6c9a5a5..0635fd7 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -4,17 +4,19 @@
from typing import Union
import warnings
+from funasr.tokenizer.abs_tokenizer import BaseTokenizer
+from funasr.register import tables
-from funasr.tokenizer.abs_tokenizer import AbsTokenizer
-
-
-class CharTokenizer(AbsTokenizer):
+@tables.register("tokenizer_classes", "CharTokenizer")
+class CharTokenizer(BaseTokenizer):
def __init__(
self,
non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
space_symbol: str = "<space>",
remove_non_linguistic_symbols: bool = False,
+ **kwargs,
):
+ super().__init__(**kwargs)
self.space_symbol = space_symbol
if non_linguistic_symbols is None:
self.non_linguistic_symbols = set()
@@ -57,4 +59,4 @@
def tokens2text(self, tokens: Iterable[str]) -> str:
tokens = [t if t != self.space_symbol else " " for t in tokens]
- return "".join(tokens)
+ return "".join(tokens)
\ No newline at end of file
--
Gitblit v1.9.1