import pynini
|
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
DAMO_ALPHA,
|
DAMO_DIGIT,
|
GraphFst,
|
insert_space,
|
)
|
from pynini.lib import pynutil
|
|
|
class ElectronicFst(GraphFst):
|
"""
|
Finite state transducer for classifying electronic: email addresses
|
e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true }
|
e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true }
|
|
Args:
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, deterministic: bool = True):
|
super().__init__(name="electronic", kind="classify", deterministic=deterministic)
|
|
dot = pynini.accep(".")
|
accepted_common_domains = [
|
x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))
|
]
|
accepted_common_domains = pynini.union(*accepted_common_domains)
|
accepted_symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))]
|
accepted_symbols = pynini.union(*accepted_symbols) - dot
|
accepted_characters = pynini.closure(DAMO_ALPHA | DAMO_DIGIT | accepted_symbols)
|
|
# email
|
username = (
|
pynutil.insert('username: "')
|
+ accepted_characters
|
+ pynutil.insert('"')
|
+ pynini.cross("@", " ")
|
)
|
domain_graph = accepted_characters + dot + accepted_characters
|
domain_graph = pynutil.insert('domain: "') + domain_graph + pynutil.insert('"')
|
domain_common_graph = (
|
pynutil.insert('domain: "')
|
+ accepted_characters
|
+ accepted_common_domains
|
+ pynini.closure(
|
(accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1
|
)
|
+ pynutil.insert('"')
|
)
|
graph = (username + domain_graph) | domain_common_graph
|
|
# url
|
protocol_start = pynini.accep("https://") | pynini.accep("http://")
|
protocol_end = pynini.accep("www.")
|
protocol = protocol_start | protocol_end | (protocol_start + protocol_end)
|
protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
|
graph |= protocol + insert_space + (domain_graph | domain_common_graph)
|
self.graph = graph
|
|
final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
|
self.fst = final_graph.optimize()
|
