export model

游雁

2023-02-14 1d4ab65c8bfebaecbcb0eec0064bae9a321cad75

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, insert_space
from pynini.lib import pynutil
 
 
class TelephoneFst(GraphFst):
    """
    Finite state transducer for classifying telephone, which includes country code, number part and extension 
 
    E.g 
    "+49 1234-1233" -> telephone { country_code: "plus neun und vierzig" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
    "(012) 1234-1233" -> telephone { country_code: "null eins zwei" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
    (0**)
 
    Args:
        cardinal: cardinal GraphFst
        deterministic: if True will provide a single transduction option,
            for False multiple transduction are generated (used for audio-based normalization)
    """
 
    def __init__(self, cardinal: GraphFst, deterministic: bool = True):
        super().__init__(name="telephone", kind="classify", deterministic=deterministic)
 
        graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize()
        graph_digit_no_zero = pynini.invert(
            pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
        ).optimize() | pynini.cross("1", "eins")
        graph_digit = graph_digit_no_zero | graph_zero
 
        numbers_with_single_digits = pynini.closure(graph_digit + insert_space) + graph_digit
 
        two_digit_and_zero = (DAMO_DIGIT ** 2 @ cardinal.two_digit_non_zero) | graph_zero
        # def add_space_after_two_digit():
        #     return pynini.closure(two_digit_and_zero + insert_space) + (
        #         two_digit_and_zero
        #     )
 
        country_code = pynini.closure(pynini.cross("+", "plus "), 0, 1) + two_digit_and_zero
        country_code |= (
            pynutil.delete("(") + graph_zero + insert_space + numbers_with_single_digits + pynutil.delete(")")
        )
        country_code |= graph_zero + insert_space + numbers_with_single_digits
 
        country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"")
 
        del_separator = pynini.cross(pynini.union("-", " "), " ")
        # numbers_with_two_digits = pynini.closure(graph_digit + insert_space) + add_space_after_two_digit() + pynini.closure(insert_space + graph_digit)
        # numbers = numbers_with_two_digits + pynini.closure(del_separator + numbers_with_two_digits, 0, 1)
        numbers = numbers_with_single_digits + pynini.closure(del_separator + numbers_with_single_digits, 0, 1)
        number_length = pynini.closure((DAMO_DIGIT | pynini.union("-", " ", ")", "(")), 7)
        number_part = pynini.compose(number_length, numbers)
        number = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"")
 
        graph = country_code + pynini.accep(" ") + number
        self.graph = graph
        final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
        self.fst = final_graph.optimize()