|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, TO_UPPER, GraphFst, get_abs_path
|
from pynini.lib import pynutil
|
|
delete_space = pynutil.delete(" ")
|
quantities = pynini.string_file(get_abs_path("data/number/thousand.tsv"))
|
quantities_abbr = pynini.string_file(get_abs_path("data/number/quantity_abbr.tsv"))
|
quantities_abbr |= TO_UPPER @ quantities_abbr
|
|
|
def get_quantity(
|
decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike', include_abbr: bool
|
) -> 'pynini.FstLike':
|
"""
|
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
|
e.g. 1 million -> integer_part: "one" quantity: "million"
|
e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million"
|
|
Args:
|
decimal: decimal FST
|
cardinal_up_to_hundred: cardinal FST
|
"""
|
quantity_wo_thousand = pynini.project(quantities, "input") - pynini.union("k", "K", "thousand")
|
if include_abbr:
|
quantity_wo_thousand |= pynini.project(quantities_abbr, "input") - pynini.union("k", "K", "thousand")
|
res = (
|
pynutil.insert("integer_part: \"")
|
+ cardinal_up_to_hundred
|
+ pynutil.insert("\"")
|
+ pynini.closure(pynutil.delete(" "), 0, 1)
|
+ pynutil.insert(" quantity: \"")
|
+ (quantity_wo_thousand @ (quantities | quantities_abbr))
|
+ pynutil.insert("\"")
|
)
|
if include_abbr:
|
quantity = quantities | quantities_abbr
|
else:
|
quantity = quantities
|
res |= (
|
decimal
|
+ pynini.closure(pynutil.delete(" "), 0, 1)
|
+ pynutil.insert("quantity: \"")
|
+ quantity
|
+ pynutil.insert("\"")
|
)
|
return res
|
|
|
class DecimalFst(GraphFst):
|
"""
|
Finite state transducer for classifying decimal, e.g.
|
-12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" }
|
1 billion -> decimal { integer_part: "one" quantity: "billion" }
|
|
cardinal: CardinalFst
|
"""
|
|
def __init__(self, cardinal: GraphFst, deterministic: bool):
|
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
|
cardinal_graph = cardinal.graph_with_and
|
cardinal_graph_hundred_component_at_least_one_none_zero_digit = (
|
cardinal.graph_hundred_component_at_least_one_none_zero_digit
|
)
|
|
self.graph = cardinal.single_digits_graph.optimize()
|
|
if not deterministic:
|
self.graph = self.graph | cardinal_graph
|
|
point = pynutil.delete(".")
|
optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1)
|
|
self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"")
|
self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"")
|
final_graph_wo_sign = (
|
pynini.closure(self.graph_integer + pynutil.insert(" "), 0, 1)
|
+ point
|
+ pynutil.insert(" ")
|
+ self.graph_fractional
|
)
|
|
quantity_w_abbr = get_quantity(
|
final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=True
|
)
|
quantity_wo_abbr = get_quantity(
|
final_graph_wo_sign, cardinal_graph_hundred_component_at_least_one_none_zero_digit, include_abbr=False
|
)
|
self.final_graph_wo_negative_w_abbr = final_graph_wo_sign | quantity_w_abbr
|
self.final_graph_wo_negative = final_graph_wo_sign | quantity_wo_abbr
|
|
# reduce options for non_deterministic and allow either "oh" or "zero", but not combination
|
if not deterministic:
|
no_oh_zero = pynini.difference(
|
DAMO_SIGMA,
|
(DAMO_SIGMA + "oh" + DAMO_SIGMA + "zero" + DAMO_SIGMA)
|
| (DAMO_SIGMA + "zero" + DAMO_SIGMA + "oh" + DAMO_SIGMA),
|
).optimize()
|
no_zero_oh = pynini.difference(
|
DAMO_SIGMA, DAMO_SIGMA + pynini.accep("zero") + DAMO_SIGMA + pynini.accep("oh") + DAMO_SIGMA
|
).optimize()
|
|
self.final_graph_wo_negative |= pynini.compose(
|
self.final_graph_wo_negative,
|
pynini.cdrewrite(
|
pynini.cross("integer_part: \"zero\"", "integer_part: \"oh\""), DAMO_SIGMA, DAMO_SIGMA, DAMO_SIGMA
|
),
|
)
|
self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_oh_zero).optimize()
|
self.final_graph_wo_negative = pynini.compose(self.final_graph_wo_negative, no_zero_oh).optimize()
|
|
final_graph = optional_graph_negative + self.final_graph_wo_negative
|
|
final_graph = self.add_tokens(final_graph)
|
self.fst = final_graph.optimize()
|
