update inverse_text_normalization/zh into fun_text_processing, update setup.py

onlybetheone

2022-12-28 0fa815632101cc73ea78363d80bf4a51facb8a7f

update inverse_text_normalization/zh into fun_text_processing, update setup.py

 fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py

@@ -28,6 +28,7 @@
    GraphFst,
    delete_extra_space,
    delete_space,
    insert_space,
    generator_main,
)
from pynini.lib import pynutil
@@ -94,10 +95,10 @@
            punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }")
            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
            token_plus_punct = (
                pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct)
                pynini.closure(punct + insert_space) + token + pynini.closure(insert_space + punct)
            )

            graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
            graph = token_plus_punct + pynini.closure(insert_space + token_plus_punct)
            graph = delete_space + graph + delete_space

            self.fst = graph.optimize()

 fun_text_processing/inverse_text_normalization/zh/taggers/word.py

@@ -11,7 +11,7 @@
# limitations under the License.

import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, GraphFst
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, DAMO_CHAR, GraphFst
from pynini.lib import pynutil


@@ -23,5 +23,5 @@

    def __init__(self):
        super().__init__(name="word", kind="classify")
        word = pynutil.insert("name: \"") + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert("\"")
        word = pynutil.insert("name: \"") + DAMO_NOT_SPACE + pynutil.insert("\"")
        self.fst = word.optimize()

 fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py

@@ -37,5 +37,5 @@
            + delete_space
            + pynutil.delete("}")
        )
        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
        graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space
        self.fst = graph

 setup.py

@@ -124,7 +124,7 @@
    long_description=open(os.path.join(dirname, "README.md"), encoding="utf-8").read(),
    long_description_content_type="text/markdown",
    license="The MIT License",
    packages=find_packages(include=["funasr*"]),
    packages=find_packages(include=["funasr*", "fun_text_processing*"]),
    package_data={"funasr": ["version.txt"]},
    install_requires=install_requires,
    setup_requires=setup_requires,