|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
DAMO_ALPHA,
|
DAMO_DIGIT,
|
DAMO_SIGMA,
|
GraphFst,
|
delete_extra_space,
|
delete_space,
|
insert_space,
|
plurals,
|
)
|
from fun_text_processing.text_normalization.en.utils import get_abs_path
|
from pynini.lib import pynutil
|
|
|
class TelephoneFst(GraphFst):
|
"""
|
Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension
|
country code optional: +***
|
number part: ***-***-****, or (***) ***-****
|
extension optional: 1-9999
|
E.g
|
+1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" }
|
1-800-GO-U-HAUL -> telephone { country_code: "one" number_part: "one, eight hundred GO U HAUL" }
|
Args:
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, deterministic: bool = True):
|
super().__init__(name="telephone", kind="classify", deterministic=deterministic)
|
|
add_separator = pynutil.insert(", ") # between components
|
zero = pynini.cross("0", "zero")
|
if not deterministic:
|
zero |= pynini.cross("0", pynini.union("o", "oh"))
|
digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() | zero
|
|
telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv"))
|
country_code = (
|
pynini.closure(telephone_prompts + delete_extra_space, 0, 1)
|
+ pynini.closure(pynini.cross("+", "plus "), 0, 1)
|
+ pynini.closure(digit + insert_space, 0, 2)
|
+ digit
|
+ pynutil.insert(",")
|
)
|
country_code |= telephone_prompts
|
country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"")
|
country_code = country_code + pynini.closure(pynutil.delete("-"), 0, 1) + delete_space + insert_space
|
|
area_part_default = pynini.closure(digit + insert_space, 2, 2) + digit
|
area_part = pynini.cross("800", "eight hundred") | pynini.compose(
|
pynini.difference(DAMO_SIGMA, "800"), area_part_default
|
)
|
|
area_part = (
|
(area_part + (pynutil.delete("-") | pynutil.delete(".")))
|
| (
|
pynutil.delete("(")
|
+ area_part
|
+ ((pynutil.delete(")") + pynini.closure(pynutil.delete(" "), 0, 1)) | pynutil.delete(")-"))
|
)
|
) + add_separator
|
|
del_separator = pynini.closure(pynini.union("-", " ", "."), 0, 1)
|
number_length = ((DAMO_DIGIT + del_separator) | (DAMO_ALPHA + del_separator)) ** 7
|
number_words = pynini.closure(
|
(DAMO_DIGIT @ digit) + (insert_space | (pynini.cross("-", ', ')))
|
| DAMO_ALPHA
|
| (DAMO_ALPHA + pynini.cross("-", ' '))
|
)
|
number_words |= pynini.closure(
|
(DAMO_DIGIT @ digit) + (insert_space | (pynini.cross(".", ', ')))
|
| DAMO_ALPHA
|
| (DAMO_ALPHA + pynini.cross(".", ' '))
|
)
|
number_words = pynini.compose(number_length, number_words)
|
number_part = area_part + number_words
|
number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"")
|
extension = (
|
pynutil.insert("extension: \"") + pynini.closure(digit + insert_space, 0, 3) + digit + pynutil.insert("\"")
|
)
|
extension = pynini.closure(insert_space + extension, 0, 1)
|
|
graph = plurals._priority_union(country_code + number_part, number_part, DAMO_SIGMA).optimize()
|
graph = plurals._priority_union(country_code + number_part + extension, graph, DAMO_SIGMA).optimize()
|
graph = plurals._priority_union(number_part + extension, graph, DAMO_SIGMA).optimize()
|
|
# ip
|
ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv"))
|
digit_to_str_graph = digit + pynini.closure(pynutil.insert(" ") + digit, 0, 2)
|
ip_graph = digit_to_str_graph + (pynini.cross(".", " dot ") + digit_to_str_graph) ** 3
|
graph |= (
|
pynini.closure(
|
pynutil.insert("country_code: \"") + ip_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1
|
)
|
+ pynutil.insert("number_part: \"")
|
+ ip_graph.optimize()
|
+ pynutil.insert("\"")
|
)
|
# ssn
|
ssn_prompts = pynini.string_file(get_abs_path("data/telephone/ssn_prompt.tsv"))
|
three_digit_part = digit + (pynutil.insert(" ") + digit) ** 2
|
two_digit_part = digit + pynutil.insert(" ") + digit
|
four_digit_part = digit + (pynutil.insert(" ") + digit) ** 3
|
ssn_separator = pynini.cross("-", ", ")
|
ssn_graph = three_digit_part + ssn_separator + two_digit_part + ssn_separator + four_digit_part
|
|
graph |= (
|
pynini.closure(
|
pynutil.insert("country_code: \"") + ssn_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1
|
)
|
+ pynutil.insert("number_part: \"")
|
+ ssn_graph.optimize()
|
+ pynutil.insert("\"")
|
)
|
|
final_graph = self.add_tokens(graph)
|
self.fst = final_graph.optimize()
|
