|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
DAMO_NOT_QUOTE,
|
DAMO_NOT_SPACE,
|
DAMO_SIGMA,
|
TO_UPPER,
|
GraphFst,
|
delete_extra_space,
|
delete_space,
|
insert_space,
|
)
|
from fun_text_processing.text_normalization.en.utils import get_abs_path
|
from pynini.examples import plurals
|
from pynini.lib import pynutil
|
|
|
class ElectronicFst(GraphFst):
|
"""
|
Finite state transducer for verbalizing electronic
|
e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u
|
|
Args:
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, deterministic: bool = True):
|
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
graph_digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize()
|
graph_zero = pynini.cross("0", "zero")
|
|
if not deterministic:
|
graph_zero |= pynini.cross("0", "o") | pynini.cross("0", "oh")
|
|
graph_digit = graph_digit_no_zero | graph_zero
|
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize()
|
|
default_chars_symbols = pynini.cdrewrite(
|
pynutil.insert(" ") + (graph_symbols | graph_digit) + pynutil.insert(" "), "", "", DAMO_SIGMA
|
)
|
default_chars_symbols = pynini.compose(
|
pynini.closure(DAMO_NOT_SPACE), default_chars_symbols.optimize()
|
).optimize()
|
|
user_name = (
|
pynutil.delete("username:")
|
+ delete_space
|
+ pynutil.delete("\"")
|
+ default_chars_symbols
|
+ pynutil.delete("\"")
|
)
|
|
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
|
domain = (
|
default_chars_symbols
|
+ insert_space
|
+ plurals._priority_union(
|
domain_common, pynutil.add_weight(pynini.cross(".", "dot"), weight=0.0001), DAMO_SIGMA
|
)
|
+ pynini.closure(
|
insert_space + (pynini.cdrewrite(TO_UPPER, "", "", DAMO_SIGMA) @ default_chars_symbols), 0, 1
|
)
|
)
|
domain = (
|
pynutil.delete("domain:")
|
+ delete_space
|
+ pynutil.delete("\"")
|
+ domain
|
+ delete_space
|
+ pynutil.delete("\"")
|
).optimize()
|
|
protocol = pynutil.delete("protocol: \"") + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete("\"")
|
graph = (
|
pynini.closure(protocol + delete_space, 0, 1)
|
+ pynini.closure(user_name + delete_space + pynutil.insert(" at ") + delete_space, 0, 1)
|
+ domain
|
+ delete_space
|
).optimize() @ pynini.cdrewrite(delete_extra_space, "", "", DAMO_SIGMA)
|
|
delete_tokens = self.delete_tokens(graph)
|
self.fst = delete_tokens.optimize()
|
