#!/usr/bin/python # -*- coding: utf-8 -*- from argparse import ArgumentParser from time import perf_counter from typing import List from fun_text_processing.text_normalization.data_loader_utils import load_file, write_file from fun_text_processing.text_normalization.normalize import Normalizer from fun_text_processing.text_normalization.token_parser import TokenParser class InverseNormalizer(Normalizer): """ Inverse normalizer that converts text from spoken to written form. Useful for ASR postprocessing. Input is expected to have no punctuation outside of approstrophe (') and dash (-) and be lower cased. Args: lang: language specifying the ITN cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. overwrite_cache: set to True to overwrite .far files """ def __init__(self, lang: str = 'en', cache_dir: str = None, overwrite_cache: bool = False, enable_standalone_number: bool = True, enable_0_to_9: bool = True): if lang == 'en': from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'id': from fun_text_processing.inverse_text_normalization.id.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.id.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'ja': from fun_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'es': from fun_text_processing.inverse_text_normalization.es.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.es.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'pt': from fun_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.pt.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'ru': from fun_text_processing.inverse_text_normalization.ru.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.ru.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'de': from fun_text_processing.inverse_text_normalization.de.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.de.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'fr': from fun_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'vi': from fun_text_processing.inverse_text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.vi.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'ko': from fun_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'zh': from fun_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) elif lang == 'tl': from fun_text_processing.inverse_text_normalization.tl.taggers.tokenize_and_classify import ClassifyFst from fun_text_processing.inverse_text_normalization.tl.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) self.tagger = ClassifyFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) self.verbalizer = VerbalizeFinalFst() self.parser = TokenParser() self.lang = lang self.convert_number = enable_standalone_number self.enable_0_to_9 = enable_0_to_9 def inverse_normalize_list(self, texts: List[str], verbose=False) -> List[str]: """ NeMo inverse text normalizer Args: texts: list of input strings verbose: whether to print intermediate meta information Returns converted list of input strings """ # print(texts) return self.normalize_list(texts=texts, verbose=verbose) def inverse_normalize(self, text: str, verbose: bool) -> str: """ Main function. Inverse normalizes tokens from spoken to written form e.g. twelve kilograms -> 12 kg Args: text: string that may include semiotic classes verbose: whether to print intermediate meta information Returns: written form """ print(text) return self.normalize(text=text, verbose=verbose) def str2bool(s, default=False): s = s.lower() if s == 'true': return True elif s == 'false': return False else: return default def parse_args(): parser = ArgumentParser() input = parser.add_mutually_exclusive_group() input.add_argument("--text", dest="input_string", help="input string", type=str) input.add_argument("--input_file", dest="input_file", help="input file path", type=str) parser.add_argument('--output_file', dest="output_file", help="output file path", type=str) parser.add_argument( "--language", help="language", choices=['en', 'id', 'ja', 'de', 'es', 'pt', 'ru', 'fr', 'vi', 'ko', 'zh', 'tl'], default="en", type=str ) parser.add_argument("--verbose", help="print info for debugging", action='store_true') parser.add_argument("--overwrite_cache", help="set to True to re-create .far grammar files", action="store_true") parser.add_argument( "--cache_dir", help="path to a dir with .far grammar file. Set to None to avoid using cache", default=None, type=str, ) parser.add_argument('--enable_standalone_number', type=str, default='True', help='enable standalone number') parser.add_argument('--enable_0_to_9', type=str, default='True', help='enable convert number 0 to 9') return parser.parse_args() if __name__ == "__main__": args = parse_args() start_time = perf_counter() if args.language == 'ja': inverse_normalizer = InverseNormalizer(lang=args.language, cache_dir=args.cache_dir, overwrite_cache=args.overwrite_cache, enable_standalone_number=str2bool(args.enable_standalone_number), enable_0_to_9=str2bool(args.enable_0_to_9)) else: inverse_normalizer = InverseNormalizer( lang=args.language, cache_dir=args.cache_dir, overwrite_cache=args.overwrite_cache ) print(f'Time to generate graph: {round(perf_counter() - start_time, 2)} sec') if args.input_string: print(inverse_normalizer.inverse_normalize(args.input_string, verbose=args.verbose)) elif args.input_file: print("Loading data: " + args.input_file) data = load_file(args.input_file) print("- Data: " + str(len(data)) + " sentences") prediction = inverse_normalizer.inverse_normalize_list(data, verbose=args.verbose) if args.output_file: write_file(args.output_file, prediction) print(f"- Denormalized. Writing out to {args.output_file}") else: print(prediction)