1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
| import pynini
| from fun_text_processing.inverse_text_normalization.id.graph_utils import (
| DAMO_CHAR,
| DAMO_SIGMA,
| GraphFst,
| delete_space,
| )
| from pynini.lib import pynutil
|
|
| class WhiteListFst(GraphFst):
| """
| Finite state transducer for verbalizing whitelist
| e.g. tokens { name: "mrs." } -> mrs.
| """
|
| def __init__(self):
| super().__init__(name="whitelist", kind="verbalize")
| graph = (
| pynutil.delete("name:")
| + delete_space
| + pynutil.delete('"')
| + pynini.closure(DAMO_CHAR - " ", 1)
| + pynutil.delete('"')
| )
| graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
| self.fst = graph.optimize()
|
|
