import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
DAMO_ALPHA,
|
DAMO_DIGIT,
|
DAMO_SIGMA,
|
DAMO_SPACE,
|
DAMO_WHITE_SPACE,
|
GraphFst,
|
delete_space,
|
insert_space,
|
)
|
from fun_text_processing.text_normalization.es.graph_utils import cardinal_separator
|
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
from pynini.lib import pynutil
|
|
zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv")))
|
digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")))
|
teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv")))
|
ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv")))
|
twenties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")))
|
hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")))
|
|
|
def filter_punctuation(fst: "pynini.FstLike") -> "pynini.FstLike":
|
"""
|
Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils)
|
and converts to a string of digits:
|
"1 000" -> "1000"
|
"1.000.000" -> "1000000"
|
Args:
|
fst: Any pynini.FstLike object. Function composes fst onto string parser fst
|
|
Returns:
|
fst: A pynini.FstLike object
|
"""
|
exactly_three_digits = DAMO_DIGIT**3 # for blocks of three
|
up_to_three_digits = pynini.closure(DAMO_DIGIT, 1, 3) # for start of string
|
|
cardinal_string = pynini.closure(
|
DAMO_DIGIT, 1
|
) # For string w/o punctuation (used for page numbers, thousand series)
|
|
cardinal_string |= (
|
up_to_three_digits
|
+ pynutil.delete(cardinal_separator)
|
+ pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator))
|
+ exactly_three_digits
|
)
|
|
return cardinal_string @ fst
|
|
|
class CardinalFst(GraphFst):
|
"""
|
Finite state transducer for classifying cardinals, e.g.
|
"1000" -> cardinal { integer: "mil" }
|
"2.000.000" -> cardinal { integer: "dos millones" }
|
|
Args:
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, deterministic: bool = True):
|
super().__init__(name="cardinal", kind="classify", deterministic=deterministic)
|
|
# Any single digit
|
graph_digit = digit
|
digits_no_one = (DAMO_DIGIT - "1") @ graph_digit
|
|
# Any double digit
|
graph_tens = teen
|
graph_tens |= ties + (pynutil.delete("0") | (pynutil.insert(" y ") + graph_digit))
|
graph_tens |= twenties
|
|
self.tens = graph_tens.optimize()
|
|
self.two_digit_non_zero = pynini.union(
|
graph_digit, graph_tens, (pynini.cross("0", DAMO_SPACE) + graph_digit)
|
).optimize()
|
|
# Three digit strings
|
graph_hundreds = hundreds + pynini.union(
|
pynutil.delete("00"),
|
(insert_space + graph_tens),
|
(pynini.cross("0", DAMO_SPACE) + graph_digit),
|
)
|
graph_hundreds |= pynini.cross("100", "cien")
|
graph_hundreds |= (
|
pynini.cross("1", "ciento")
|
+ insert_space
|
+ pynini.union(graph_tens, pynutil.delete("0") + graph_digit)
|
)
|
|
self.hundreds = graph_hundreds.optimize()
|
|
# For all three digit strings with leading zeroes (graph appends '0's to manage place in string)
|
graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens)
|
|
graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | (
|
pynutil.delete("00") + graph_digit
|
)
|
graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | (
|
pynutil.delete("00") + digits_no_one
|
)
|
|
graph_thousands_component_at_least_one_none_zero_digit = pynini.union(
|
pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit,
|
graph_hundreds_component_at_least_one_none_zero_digit_no_one
|
+ pynutil.insert(" mil")
|
+ (
|
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
| pynutil.delete("000")
|
),
|
pynini.cross("001", "mil")
|
+ (
|
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
| pynutil.delete("000")
|
),
|
)
|
|
graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union(
|
pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one,
|
graph_hundreds_component_at_least_one_none_zero_digit_no_one
|
+ pynutil.insert(" mil")
|
+ (
|
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
| pynutil.delete("000")
|
),
|
pynini.cross("001", "mil")
|
+ (
|
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
| pynutil.delete("000")
|
),
|
)
|
|
graph_million = pynutil.add_weight(pynini.cross("000001", "un millón"), -0.001)
|
graph_million |= (
|
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
+ pynutil.insert(" millones")
|
)
|
graph_million |= pynutil.delete("000000")
|
graph_million += insert_space
|
|
graph_billion = pynutil.add_weight(pynini.cross("000001", "un billón"), -0.001)
|
graph_billion |= (
|
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
+ pynutil.insert(" billones")
|
)
|
graph_billion |= pynutil.delete("000000")
|
graph_billion += insert_space
|
|
graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trillón"), -0.001)
|
graph_trillion |= (
|
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
+ pynutil.insert(" trillones")
|
)
|
graph_trillion |= pynutil.delete("000000")
|
graph_trillion += insert_space
|
|
graph = (
|
graph_trillion
|
+ graph_billion
|
+ graph_million
|
+ (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000"))
|
)
|
|
self.graph = (
|
((DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT, 0))
|
@ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
|
@ DAMO_DIGIT**24
|
@ graph
|
@ pynini.cdrewrite(delete_space, "[BOS]", "", DAMO_SIGMA)
|
@ pynini.cdrewrite(delete_space, "", "[EOS]", DAMO_SIGMA)
|
@ pynini.cdrewrite(
|
pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 2), DAMO_SPACE),
|
DAMO_ALPHA,
|
DAMO_ALPHA,
|
DAMO_SIGMA,
|
)
|
)
|
self.graph |= zero
|
|
self.graph = filter_punctuation(self.graph).optimize()
|
|
optional_minus_graph = pynini.closure(
|
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
)
|
|
final_graph = (
|
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
)
|
|
final_graph = self.add_tokens(final_graph)
|
self.fst = final_graph.optimize()
|
