|
from collections import defaultdict
|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, DAMO_SPACE, GraphFst, insert_space
|
from fun_text_processing.text_normalization.en.utils import load_labels
|
from fun_text_processing.text_normalization.ru.utils import get_abs_path
|
from pynini.lib import pynutil
|
|
delete_space = pynutil.delete(" ")
|
|
|
def prepare_labels_for_insertion(file_path: str):
|
"""
|
Read the file and creates a union insertion graph
|
|
Args:
|
file_path: path to a file (3 columns: a label type e.g.
|
"@@decimal_delimiter@@", a label e.g. "целого", and a weight e.g. "0.1").
|
|
Returns dictionary mapping from label type to an fst that inserts the labels with the specified weights.
|
|
"""
|
labels = load_labels(file_path)
|
mapping = defaultdict(list)
|
for k, v, w in labels:
|
mapping[k].append((v, w))
|
|
for k in mapping:
|
mapping[k] = (
|
insert_space
|
+ pynini.union(*[pynutil.add_weight(pynutil.insert(end), weight) for end, weight in mapping[k]])
|
).optimize()
|
return mapping
|
|
|
class DecimalFst(GraphFst):
|
"""
|
Finite state transducer for classifying decimal, e.g.
|
"1,08" -> tokens { decimal { integer_part: "одно целая" fractional_part: "восемь сотых} }
|
|
Args:
|
cardinal: CardinalFst
|
deterministic: if True will provide a single transduction option,
|
for False multiple transduction are generated (used for audio-based normalization)
|
"""
|
|
def __init__(self, cardinal: GraphFst, deterministic: bool = False):
|
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
|
integer_part = cardinal.cardinal_numbers_default
|
cardinal_numbers_with_leading_zeros = cardinal.cardinal_numbers_with_leading_zeros
|
|
delimiter_map = prepare_labels_for_insertion(get_abs_path("data/numbers/decimal_delimiter.tsv"))
|
delimiter = (
|
pynini.cross(",", "")
|
+ delimiter_map['@@decimal_delimiter@@']
|
+ pynini.closure(pynutil.add_weight(pynutil.insert(" и"), 0.5), 0, 1)
|
).optimize()
|
|
decimal_endings_map = prepare_labels_for_insertion(get_abs_path("data/numbers/decimal_endings.tsv"))
|
|
self.integer_part = integer_part + delimiter
|
graph_integer = pynutil.insert("integer_part: \"") + self.integer_part + pynutil.insert("\"")
|
|
graph_fractional = DAMO_DIGIT @ cardinal_numbers_with_leading_zeros + decimal_endings_map['10']
|
graph_fractional |= (DAMO_DIGIT + DAMO_DIGIT) @ cardinal_numbers_with_leading_zeros + decimal_endings_map[
|
'100'
|
]
|
graph_fractional |= (
|
DAMO_DIGIT + DAMO_DIGIT + DAMO_DIGIT
|
) @ cardinal_numbers_with_leading_zeros + decimal_endings_map['1000']
|
graph_fractional |= (
|
DAMO_DIGIT + DAMO_DIGIT + DAMO_DIGIT + DAMO_DIGIT
|
) @ cardinal_numbers_with_leading_zeros + decimal_endings_map['10000']
|
|
self.optional_quantity = pynini.string_file(get_abs_path("data/numbers/quantity.tsv")).optimize()
|
|
self.graph_fractional = graph_fractional
|
graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"")
|
optional_quantity = pynini.closure(
|
(pynutil.add_weight(pynini.accep(DAMO_SPACE), -0.1) | insert_space)
|
+ pynutil.insert("quantity: \"")
|
+ self.optional_quantity
|
+ pynutil.insert("\""),
|
0,
|
1,
|
)
|
self.final_graph = (
|
cardinal.optional_graph_negative + graph_integer + insert_space + graph_fractional + optional_quantity
|
)
|
|
self.final_graph = self.add_tokens(self.final_graph)
|
self.fst = self.final_graph.optimize()
|
