|
|
import pynini
|
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, DAMO_SPACE
|
from fun_text_processing.text_normalization.es import LOCALIZATION
|
from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels
|
from pynini.lib import pynutil
|
|
digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input")
|
tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input")
|
teens = pynini.project(pynini.string_file(get_abs_path("data/numbers/teen.tsv")), "input")
|
twenties = pynini.project(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")), "input")
|
hundreds = pynini.project(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")), "input")
|
|
accents = pynini.string_map([("á", "a"), ("é", "e"), ("í", "i"), ("ó", "o"), ("ú", "u")])
|
|
if LOCALIZATION == "am": # Setting localization for central and northern america formatting
|
cardinal_separator = pynini.string_map([",", DAMO_SPACE])
|
decimal_separator = pynini.accep(".")
|
else:
|
cardinal_separator = pynini.string_map([".", DAMO_SPACE])
|
decimal_separator = pynini.accep(",")
|
|
ones = pynini.union("un", "ún")
|
fem_ones = pynini.union(pynini.cross("un", "una"), pynini.cross("ún", "una"), pynini.cross("uno", "una"))
|
one_to_one_hundred = pynini.union(digits, "uno", tens, teens, twenties, tens + pynini.accep(" y ") + digits)
|
fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", DAMO_SIGMA)
|
|
|
def strip_accent(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Converts all accented vowels to non-accented equivalents
|
|
Args:
|
fst: Any fst. Composes vowel conversion onto fst's output strings
|
"""
|
return fst @ pynini.cdrewrite(accents, "", "", DAMO_SIGMA)
|
|
|
def shift_cardinal_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and
|
Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies
|
to value place for <1000 and multiple of 1000. (e.g. "doscientos mil doscientos" -> "doscientas mil doscientas".) For place values greater than the thousands, there
|
is no gender shift as the higher powers of ten ("millones", "billones") are masculine nouns and any conversion would be formally
|
ungrammatical.
|
e.g.
|
"doscientos" -> "doscientas"
|
"doscientos mil" -> "doscientas mil"
|
"doscientos millones" -> "doscientos millones"
|
"doscientos mil millones" -> "doscientos mil millones"
|
"doscientos millones doscientos mil doscientos" -> "doscientos millones doscientas mil doscientas"
|
|
Args:
|
fst: Any fst. Composes conversion onto fst's output strings
|
"""
|
before_mil = (
|
DAMO_SPACE
|
+ (pynini.accep("mil") | pynini.accep("milésimo"))
|
+ pynini.closure(DAMO_SPACE + hundreds, 0, 1)
|
+ pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1)
|
+ pynini.union(pynini.accep("[EOS]"), pynini.accep("\""), decimal_separator)
|
)
|
before_double_digits = pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union(
|
pynini.accep("[EOS]"), pynini.accep("\"")
|
)
|
|
fem_allign = pynini.cdrewrite(fem_hundreds, "", before_mil, DAMO_SIGMA) # doscientas mil dosciento
|
fem_allign @= pynini.cdrewrite(fem_hundreds, "", before_double_digits, DAMO_SIGMA) # doscientas mil doscienta
|
|
fem_allign @= pynini.cdrewrite(
|
fem_ones, "", pynini.union("[EOS]", "\"", decimal_separator), DAMO_SIGMA
|
) # If before a quote or EOS, we know it's the end of a string
|
|
return fst @ fem_allign
|
|
|
def shift_number_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to
|
feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una".
|
This has no boundary restriction and will perform shift across all values in output string.
|
e.g.
|
"doscientos" -> "doscientas"
|
"doscientos millones" -> "doscientas millones"
|
"doscientos millones doscientos" -> "doscientas millones doscientas"
|
|
Args:
|
fst: Any fst. Composes conversion onto fst's output strings
|
"""
|
fem_allign = pynini.cdrewrite(fem_hundreds, "", "", DAMO_SIGMA)
|
fem_allign @= pynini.cdrewrite(
|
fem_ones, "", pynini.union(DAMO_SPACE, pynini.accep("[EOS]"), pynini.accep("\"")), DAMO_SIGMA
|
) # If before a quote or EOS, we know it's the end of a string
|
|
return fst @ fem_allign
|
|
|
def strip_cardinal_apocope(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only
|
affects strings where the final value is a variation of "un".
|
e.g.
|
"un" -> "uno"
|
"veintiún" -> "veintiuno"
|
|
Args:
|
fst: Any fst. Composes conversion onto fst's output strings
|
"""
|
# Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one
|
strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno")
|
strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", "\""), DAMO_SIGMA)
|
return fst @ strip
|
|
|
def add_cardinal_apocope_fem(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable
|
with text string, so is included for non-deterministic cases.
|
e.g.
|
"una" -> "un"
|
"veintiuna" -> "veintiun"
|
|
Args:
|
fst: Any fst. Composes conversion onto fst's output strings
|
"""
|
# Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one
|
strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún")
|
strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", "\""), DAMO_SIGMA)
|
return fst @ strip
|
|
|
def roman_to_int(fst: 'pynini.FstLike') -> 'pynini.FstLike':
|
"""
|
Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000.
|
e.g.
|
"V" -> "5"
|
"i" -> "1"
|
|
Args:
|
fst: Any fst. Composes fst onto Roman conversion outputs.
|
"""
|
|
def _load_roman(file: str):
|
roman = load_labels(get_abs_path(file))
|
roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman]
|
return pynini.string_map(roman_numerals)
|
|
digit = _load_roman("data/roman/digit.tsv")
|
ties = _load_roman("data/roman/ties.tsv")
|
hundreds = _load_roman("data/roman/hundreds.tsv")
|
|
graph = (
|
digit
|
| ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
| (
|
hundreds
|
+ (ties | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
+ (digit | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
)
|
).optimize()
|
|
return graph @ fst
|
