|
|
|
import os
|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
MIN_NEG_WEIGHT,
|
DAMO_ALPHA,
|
DAMO_CHAR,
|
DAMO_SIGMA,
|
DAMO_SPACE,
|
generator_main,
|
)
|
from fun_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
|
from pynini.lib import pynutil
|
|
import logging
|
|
|
class PostProcessingFst:
|
"""
|
Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g.
|
removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)"
|
|
Args:
|
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
overwrite_cache: set to True to overwrite .far files
|
"""
|
|
def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
|
|
far_file = None
|
if cache_dir is not None and cache_dir != "None":
|
os.makedirs(cache_dir, exist_ok=True)
|
far_file = os.path.join(cache_dir, "en_tn_post_processing.far")
|
if not overwrite_cache and far_file and os.path.exists(far_file):
|
self.fst = pynini.Far(far_file, mode="r")["post_process_graph"]
|
logging.info(f'Post processing graph was restored from {far_file}.')
|
else:
|
self.set_punct_dict()
|
self.fst = self.get_punct_postprocess_graph()
|
|
if far_file:
|
generator_main(far_file, {"post_process_graph": self.fst})
|
|
def set_punct_dict(self):
|
self.punct_marks = {
|
"'": [
|
"'",
|
'´',
|
'ʹ',
|
'ʻ',
|
'ʼ',
|
'ʽ',
|
'ʾ',
|
'ˈ',
|
'ˊ',
|
'ˋ',
|
'˴',
|
'ʹ',
|
'΄',
|
'՚',
|
'՝',
|
'י',
|
'׳',
|
'ߴ',
|
'ߵ',
|
'ᑊ',
|
'ᛌ',
|
'᾽',
|
'᾿',
|
'`',
|
'´',
|
'῾',
|
'‘',
|
'’',
|
'‛',
|
'′',
|
'‵',
|
'ꞌ',
|
'＇',
|
'｀',
|
'𖽑',
|
'𖽒',
|
],
|
}
|
|
def get_punct_postprocess_graph(self):
|
"""
|
Returns graph to post process punctuation marks.
|
|
{``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept.
|
By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks.
|
"""
|
punct_marks_all = PunctuationFst().punct_marks
|
|
# no_space_before_punct assume no space before them
|
quotes = ["'", "\"", "``", "«"]
|
dashes = ["-", "—"]
|
brackets = ["<", "{", "("]
|
open_close_single_quotes = [
|
("`", "`"),
|
]
|
|
open_close_double_quotes = [('"', '"'), ("``", "``"), ("“", "”")]
|
open_close_symbols = open_close_single_quotes + open_close_double_quotes
|
allow_space_before_punct = ["&"] + quotes + dashes + brackets + [k[0] for k in open_close_symbols]
|
|
no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct]
|
no_space_before_punct = pynini.union(*no_space_before_punct)
|
no_space_after_punct = pynini.union(*brackets)
|
delete_space = pynutil.delete(" ")
|
delete_space_optional = pynini.closure(delete_space, 0, 1)
|
|
# non_punct allows space
|
# delete space before no_space_before_punct marks, if present
|
non_punct = pynini.difference(DAMO_CHAR, no_space_before_punct).optimize()
|
graph = (
|
pynini.closure(non_punct)
|
+ pynini.closure(
|
no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT)
|
)
|
+ pynini.closure(non_punct)
|
)
|
graph = pynini.closure(graph).optimize()
|
graph = pynini.compose(
|
graph, pynini.cdrewrite(pynini.cross("``", '"'), "", "", DAMO_SIGMA).optimize()
|
).optimize()
|
|
# remove space after no_space_after_punct (even if there are no matching closing brackets)
|
no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, DAMO_SIGMA, DAMO_SIGMA).optimize()
|
graph = pynini.compose(graph, no_space_after_punct).optimize()
|
|
# remove space around text in quotes
|
single_quote = pynutil.add_weight(pynini.accep("`"), MIN_NEG_WEIGHT)
|
double_quotes = pynutil.add_weight(pynini.accep('"'), MIN_NEG_WEIGHT)
|
quotes_graph = (
|
single_quote + delete_space_optional + DAMO_ALPHA + DAMO_SIGMA + delete_space_optional + single_quote
|
).optimize()
|
|
# this is to make sure multiple quotes are tagged from right to left without skipping any quotes in the left
|
not_alpha = pynini.difference(DAMO_CHAR, DAMO_ALPHA).optimize() | pynutil.add_weight(
|
DAMO_SPACE, MIN_NEG_WEIGHT
|
)
|
end = pynini.closure(pynutil.add_weight(not_alpha, MIN_NEG_WEIGHT))
|
quotes_graph |= (
|
double_quotes
|
+ delete_space_optional
|
+ DAMO_ALPHA
|
+ DAMO_SIGMA
|
+ delete_space_optional
|
+ double_quotes
|
+ end
|
)
|
|
quotes_graph = pynutil.add_weight(quotes_graph, MIN_NEG_WEIGHT)
|
quotes_graph = DAMO_SIGMA + pynini.closure(DAMO_SIGMA + quotes_graph + DAMO_SIGMA)
|
|
graph = pynini.compose(graph, quotes_graph).optimize()
|
|
# remove space between a word and a single quote followed by s
|
remove_space_around_single_quote = pynini.cdrewrite(
|
delete_space_optional + pynini.union(*self.punct_marks["'"]) + delete_space,
|
DAMO_ALPHA,
|
pynini.union("s ", "s[EOS]"),
|
DAMO_SIGMA,
|
)
|
|
graph = pynini.compose(graph, remove_space_around_single_quote).optimize()
|
return graph
|
