import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
DAMO_CHAR,
|
DAMO_DIGIT,
|
DAMO_SIGMA,
|
DAMO_SPACE,
|
GraphFst,
|
)
|
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
from pynini.lib import pynutil
|
|
ordinal_exceptions = pynini.string_file(get_abs_path("data/fractions/ordinal_exceptions.tsv"))
|
higher_powers_of_ten = pynini.string_file(get_abs_path("data/fractions/powers_of_ten.tsv"))
|
|
|
class FractionFst(GraphFst):
|
"""
|
Finite state transducer for classifying fraction
|
"23 4/5" ->
|
tokens { fraction { integer: "veintitrés" numerator: "cuatro" denominator: "quinto" mophosyntactic_features: "ordinal" } }
|
|
Args:
|
cardinal: CardinalFst
|
ordinal: OrdinalFst
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True):
|
super().__init__(name="fraction", kind="classify", deterministic=deterministic)
|
cardinal_graph = cardinal.graph
|
ordinal_graph = ordinal.graph
|
|
# 2-10 are all ordinals
|
three_to_ten = pynini.string_map(
|
[
|
"2",
|
"3",
|
"4",
|
"5",
|
"6",
|
"7",
|
"8",
|
"9",
|
"10",
|
]
|
)
|
block_three_to_ten = pynutil.delete(three_to_ten) # To block cardinal productions
|
if not deterministic: # Multiples of tens are sometimes rendered as ordinals
|
three_to_ten |= pynini.string_map(
|
[
|
"20",
|
"30",
|
"40",
|
"50",
|
"60",
|
"70",
|
"80",
|
"90",
|
]
|
)
|
graph_three_to_ten = three_to_ten @ ordinal_graph
|
graph_three_to_ten @= pynini.cdrewrite(ordinal_exceptions, "", "", DAMO_SIGMA)
|
|
# Higher powers of tens (and multiples) are converted to ordinals.
|
hundreds = pynini.string_map(
|
[
|
"100",
|
"200",
|
"300",
|
"400",
|
"500",
|
"600",
|
"700",
|
"800",
|
"900",
|
]
|
)
|
graph_hundreds = hundreds @ ordinal_graph
|
|
multiples_of_thousand = ordinal.multiples_of_thousand # So we can have X milésimos
|
|
graph_higher_powers_of_ten = (
|
pynini.closure(ordinal.one_to_one_thousand + DAMO_SPACE, 0, 1)
|
+ pynini.closure("mil ", 0, 1)
|
+ pynini.closure(ordinal.one_to_one_thousand + DAMO_SPACE, 0, 1)
|
) # x millones / x mil millones / x mil z millones
|
graph_higher_powers_of_ten += higher_powers_of_ten
|
graph_higher_powers_of_ten = cardinal_graph @ graph_higher_powers_of_ten
|
graph_higher_powers_of_ten @= pynini.cdrewrite(
|
pynutil.delete("un "),
|
pynini.accep("[BOS]"),
|
pynini.project(higher_powers_of_ten, "output"),
|
DAMO_SIGMA,
|
) # we drop 'un' from these ordinals (millionths, not one-millionths)
|
|
graph_higher_powers_of_ten = (
|
multiples_of_thousand | graph_hundreds | graph_higher_powers_of_ten
|
)
|
block_higher_powers_of_ten = pynutil.delete(
|
pynini.project(graph_higher_powers_of_ten, "input")
|
) # For cardinal graph
|
|
graph_fractions_ordinals = graph_higher_powers_of_ten | graph_three_to_ten
|
graph_fractions_ordinals += pynutil.insert(
|
'" morphosyntactic_features: "ordinal"'
|
) # We note the root for processing later
|
|
# Blocking the digits and hundreds from Cardinal graph
|
graph_fractions_cardinals = pynini.cdrewrite(
|
block_three_to_ten | block_higher_powers_of_ten,
|
pynini.accep("[BOS]"),
|
pynini.accep("[EOS]"),
|
DAMO_SIGMA,
|
)
|
graph_fractions_cardinals @= DAMO_CHAR.plus @ pynini.cdrewrite(
|
pynutil.delete("0"), pynini.accep("[BOS]"), pynini.accep("[EOS]"), DAMO_SIGMA
|
) # Empty characters become '0' for DAMO_CHAR fst, so need to block
|
graph_fractions_cardinals @= cardinal_graph
|
graph_fractions_cardinals += pynutil.insert(
|
'" morphosyntactic_features: "add_root"'
|
) # blocking these entries to reduce erroneous possibilities in debugging
|
|
if deterministic:
|
graph_fractions_cardinals = (
|
pynini.closure(DAMO_DIGIT, 1, 2) @ graph_fractions_cardinals
|
) # Past hundreds the conventional scheme can be hard to read. For determinism we stop here
|
|
graph_denominator = pynini.union(
|
graph_fractions_ordinals,
|
graph_fractions_cardinals,
|
pynutil.add_weight(cardinal_graph + pynutil.insert('"'), 0.001),
|
) # Last form is simply recording the cardinal. Weighting so last resort
|
|
integer = (
|
pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + DAMO_SPACE
|
)
|
numerator = (
|
pynutil.insert('numerator: "')
|
+ cardinal_graph
|
+ (pynini.cross("/", '" ') | pynini.cross(" / ", '" '))
|
)
|
denominator = pynutil.insert('denominator: "') + graph_denominator
|
|
self.graph = pynini.closure(integer, 0, 1) + numerator + denominator
|
|
final_graph = self.add_tokens(self.graph)
|
self.fst = final_graph.optimize()
|
