From 12a7adfdf3dd4f80b5d3a51cfc4eecc84eaa7c64 Mon Sep 17 00:00:00 2001
From: jmwang66 <wangjiaming.wjm@alibaba-inc.com>
Date: 星期一, 16 一月 2023 18:46:40 +0800
Subject: [PATCH] update version 0.1.6
---
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/finetune.py | 36
fun_text_processing/inverse_text_normalization/tl/utils.py | 13
egs/aishell/tranformer/utils/fix_data.sh | 4
fun_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py | 13
fun_text_processing/text_normalization/en/taggers/electronic.py | 14
fun_text_processing/inverse_text_normalization/es/verbalizers/electronic.py | 13
funasr/bin/vad_inference.py | 430
fun_text_processing/text_normalization/de/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/pt/taggers/cardinal.py | 13
egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/data/punc_example.txt | 3
fun_text_processing/inverse_text_normalization/en/__init__.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py | 11
fun_text_processing/inverse_text_normalization/zh/taggers/.cardinal.py.swp | 0
fun_text_processing/text_normalization/de/taggers/fraction.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/infer.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/infer.py | 13
fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py | 13
funasr/bin/punc_inference_launch.py | 118
fun_text_processing/text_normalization/en/taggers/whitelist.py | 14
fun_text_processing/inverse_text_normalization/en/verbalizers/fraction.py | 13
fun_text_processing/inverse_text_normalization/es/__init__.py | 13
fun_text_processing/text_normalization/ru/taggers/money.py | 13
fun_text_processing/inverse_text_normalization/id/graph_utils.py | 14
fun_text_processing/inverse_text_normalization/ru/taggers/ordinal.py | 14
funasr/models/e2e_vad.py | 567
fun_text_processing/inverse_text_normalization/fr/verbalizers/date.py | 13
funasr/bin/build_trainer.py | 116
fun_text_processing/inverse_text_normalization/en/verbalizers/date.py | 14
egs/aishell2/paraformer/run.sh | 251
egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py | 35
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/finetune.py | 35
fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py | 14
fun_text_processing/inverse_text_normalization/ja/graph_utils.py | 14
fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py | 14
fun_text_processing/__init__.py | 1
fun_text_processing/inverse_text_normalization/id/verbalizers/time.py | 12
fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py | 11
fun_text_processing/inverse_text_normalization/zh/utils.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py | 11
fun_text_processing/text_normalization/en/graph_utils.py | 13
fun_text_processing/num2words/num2words/lang_KZ.py | 118
fun_text_processing/num2words/num2words/lang_TR.py | 846 +
docs/images/wechat.png | 0
egs/aishell2/transformer/utils/textnorm_zh.py | 0
fun_text_processing/inverse_text_normalization/en/taggers/decimal.py | 13
fun_text_processing/text_normalization/en/taggers/roman.py | 14
fun_text_processing/inverse_text_normalization/en/taggers/money.py | 13
fun_text_processing/inverse_text_normalization/tl/taggers/punctuation.py | 11
fun_text_processing/text_normalization/es/verbalizers/time.py | 14
fun_text_processing/text_normalization/zh/taggers/cardinal.py | 14
fun_text_processing/inverse_text_normalization/ja/utils.py | 14
fun_text_processing/num2words/num2words/lang_FI.py | 721 +
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/infer.py | 13
fun_text_processing/text_normalization/es/utils.py | 14
fun_text_processing/num2words/num2words/lang_KO.py | 133
fun_text_processing/text_normalization/en/utils.py | 14
fun_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py | 13
fun_text_processing/inverse_text_normalization/id/taggers/punctuation.py | 12
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/infer.py | 13
fun_text_processing/num2words/num2words/lang_FR.py | 93
fun_text_processing/text_normalization/en/taggers/measure.py | 14
fun_text_processing/inverse_text_normalization/tl/taggers/decimal.py | 11
egs/aishell2/transformer/path.sh | 6
fun_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py | 14
fun_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py | 13
fun_text_processing/text_normalization/en/verbalizers/verbalize_final.py | 14
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer.py | 88
fun_text_processing/inverse_text_normalization/ru/taggers/telephone.py | 14
fun_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py | 12
fun_text_processing/inverse_text_normalization/tl/verbalizers/money.py | 11
fun_text_processing/inverse_text_normalization/fr/__init__.py | 13
fun_text_processing/inverse_text_normalization/id/taggers/electronic.py | 12
fun_text_processing/num2words/num2words/lang_BN.py | 108
fun_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py | 13
fun_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py | 13
fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py | 12
fun_text_processing/text_normalization/en/verbalizers/word.py | 13
funasr/models/decoder/sv_decoder.py | 37
fun_text_processing/inverse_text_normalization/de/__init__.py | 14
fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py | 12
fun_text_processing/inverse_text_normalization/id/verbalizers/cardinal.py | 12
fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py | 13
fun_text_processing/num2words/num2words/lang_DE.py | 142
egs/aishell2/transformer/utils/compute_wer.py | 0
fun_text_processing/inverse_text_normalization/tl/taggers/date.py | 11
fun_text_processing/text_normalization/zh/verbalizers/math_symbol.py | 14
fun_text_processing/text_normalization/en/__init__.py | 14
fun_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py | 13
fun_text_processing/inverse_text_normalization/id/id_unit_test.tsv | 29
egs/cnceleb/resnet34/sv.yaml | 3515 ++++
fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize.py | 13
egs/aishell2/paraformer/conf/train_asr_paraformer_conformer_20e_1280_320_6d_1280_320.yaml | 96
docs/images/dingding.jpg | 0
fun_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py | 13
fun_text_processing/inverse_text_normalization/ko/taggers/time.py | 11
fun_text_processing/inverse_text_normalization/id/taggers/whitelist.py | 12
funasr/utils/wav_utils.py | 112
fun_text_processing/inverse_text_normalization/id/verbalizers/telephone.py | 12
fun_text_processing/num2words/num2words/lang_NL.py | 140
fun_text_processing/text_normalization/de/verbalizers/time.py | 13
egs/aishell2/transformer/utils/proce_text.py | 0
egs/aishell/paraformerbert/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml | 9
fun_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py | 13
fun_text_processing/num2words/num2words/lang_LO.py | 110
fun_text_processing/num2words/num2words/lang_SR.py | 204
egs/cnceleb/resnet34/configuration.json | 20
fun_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py | 14
fun_text_processing/inverse_text_normalization/zh/verbalizers/electronic.py | 11
fun_text_processing/inverse_text_normalization/tl/verbalizers/ordinal.py | 11
fun_text_processing/num2words/num2words/lang_TE.py | 169
egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py | 35
fun_text_processing/text_normalization/es/verbalizers/date.py | 14
fun_text_processing/text_normalization/es/verbalizers/ordinal.py | 14
fun_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py | 13
egs/aishell2/transformer/utils/apply_cmvn.py | 0
egs/aishell2/transformer/utils/fix_data_feat.sh | 0
fun_text_processing/inverse_text_normalization/de/taggers/cardinal.py | 13
fun_text_processing/inverse_text_normalization/tl/taggers/cardinal.py | 11
fun_text_processing/num2words/num2words/lang_KH.py | 136
fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/ordinal.py | 12
fun_text_processing/text_normalization/es/taggers/fraction.py | 14
fun_text_processing/inverse_text_normalization/en/verbalizers/telephone.py | 13
egs/aishell2/paraformerbert/local/prepare_data.sh | 54
fun_text_processing/text_normalization/de/taggers/telephone.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py | 12
fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py | 11
fun_text_processing/inverse_text_normalization/tl/taggers/electronic.py | 11
fun_text_processing/inverse_text_normalization/vi/verbalizers/date.py | 14
egs/cnceleb/resnet34/sid/calc_trial_scores.py | 160
fun_text_processing/text_normalization/en/verbalizers/cardinal.py | 14
egs/aishell2/transformer/utils/apply_cmvn.sh | 0
fun_text_processing/text_normalization/zh/verbalizers/fraction.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/word.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/fr/taggers/ordinal.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/infer.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/infer.py | 13
fun_text_processing/inverse_text_normalization/tl/taggers/tokenize_and_classify.py | 11
fun_text_processing/num2words/num2words/lang_VI.py | 86
egs/aishell2/transformer/utils/combine_cmvn_file.py | 0
fun_text_processing/inverse_text_normalization/ja/ja_itn_test_input.txt | 20
fun_text_processing/inverse_text_normalization/es/taggers/word.py | 13
fun_text_processing/inverse_text_normalization/fr/verbalizers/time.py | 13
fun_text_processing/inverse_text_normalization/ja/ja_unit_test.tsv | 20
funasr/modules/data2vec/data_utils.py | 147
fun_text_processing/inverse_text_normalization/ja/ja_itn_test_expected.txt | 20
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/finetune.py | 37
fun_text_processing/inverse_text_normalization/tl/verbalizers/fraction.py | 11
egs/cnceleb/resnet34/sid/speaker_verification.py | 167
fun_text_processing/inverse_text_normalization/es/taggers/ordinal.py | 13
fun_text_processing/inverse_text_normalization/vi/taggers/whitelist.py | 13
fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py | 14
egs/aishell2/transformer/utils/split_data.py | 0
fun_text_processing/text_normalization/en/data/number/cardinal_number_name.far | 0
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md | 30
fun_text_processing/inverse_text_normalization/id/id_itn_test_input.txt | 29
egs/aishell2/transformer/utils/error_rate_zh | 0
fun_text_processing/inverse_text_normalization/vi/taggers/measure.py | 13
fun_text_processing/inverse_text_normalization/vi/verbalizers/word.py | 13
fun_text_processing/inverse_text_normalization/fr/verbalizers/measure.py | 13
fun_text_processing/text_normalization/es/verbalizers/fraction.py | 14
fun_text_processing/text_normalization/ru/taggers/whitelist.py | 14
fun_text_processing/text_normalization/es/taggers/telephone.py | 14
fun_text_processing/inverse_text_normalization/es/taggers/whitelist.py | 13
fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py | 11
fun_text_processing/inverse_text_normalization/ko/__init__.py | 13
fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py | 11
fun_text_processing/inverse_text_normalization/pt/taggers/money.py | 14
README.md | 53
fun_text_processing/num2words/num2words/lang_ID.py | 189
egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py | 36
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py | 15
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/infer.py | 13
fun_text_processing/num2words/num2words/lang_PL.py | 245
fun_text_processing/inverse_text_normalization/zh/taggers/whitelist.py | 11
fun_text_processing/num2words/num2words/lang_UR.py | 232
fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py | 11
fun_text_processing/inverse_text_normalization/es/taggers/punctuation.py | 13
fun_text_processing/inverse_text_normalization/vi/taggers/electronic.py | 13
funasr/tasks/sv.py | 437
fun_text_processing/inverse_text_normalization/ru/verbalizers/money.py | 13
fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/finetune.py | 35
egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py | 13
fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py | 12
fun_text_processing/text_normalization/normalize.py | 14
fun_text_processing/inverse_text_normalization/pt/verbalizers/word.py | 15
funasr/models/frontend/wav_frontend.py | 222
fun_text_processing/inverse_text_normalization/tl/taggers/whitelist.py | 11
fun_text_processing/text_normalization/ru/verbalizers/electronic.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/README.md | 30
funasr/utils/compute_wer.py | 0
funasr/datasets/ms_dataset.py | 33
fun_text_processing/text_normalization/es/taggers/money.py | 14
egs/aishell2/transformer/utils/proc_conf_oss.py | 0
fun_text_processing/inverse_text_normalization/es/taggers/decimal.py | 13
egs/aishell2/transformer/utils/apply_lfr_and_cmvn.sh | 0
egs/aishell2/paraformerbert/conf/decode_asr_transformer.yaml | 6
fun_text_processing/inverse_text_normalization/vi/taggers/decimal.py | 13
fun_text_processing/num2words/num2words/lang_DK.py | 117
fun_text_processing/inverse_text_normalization/zh/taggers/cardinal.py | 11
fun_text_processing/text_normalization/de/verbalizers/ordinal.py | 13
fun_text_processing/inverse_text_normalization/tl/verbalizers/word.py | 11
fun_text_processing/num2words/num2words/lang_FR_DZ.py | 18
fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py | 13
fun_text_processing/inverse_text_normalization/ja/taggers/date.py | 12
fun_text_processing/text_normalization/zh/utils.py | 14
fun_text_processing/inverse_text_normalization/de/taggers/date.py | 13
fun_text_processing/inverse_text_normalization/ru/taggers/time.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/finetune.py | 35
fun_text_processing/text_normalization/es/verbalizers/money.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/infer.py | 13
funasr/models/encoder/conformer_encoder.py | 10
fun_text_processing/inverse_text_normalization/fr/verbalizers/money.py | 13
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/local_infer.py | 88
egs/aishell2/conformer/utils | 1
fun_text_processing/inverse_text_normalization/fr/verbalizers/word.py | 13
egs/cnceleb/resnet34/local/make_cnceleb2.sh | 32
fun_text_processing/inverse_text_normalization/tl/taggers/telephone.py | 11
fun_text_processing/inverse_text_normalization/de/taggers/money.py | 13
fun_text_processing/text_normalization/ru/utils.py | 14
funasr/bin/asr_inference_uniasr.py | 441
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md | 30
fun_text_processing/num2words/num2words/lang_MN.py | 98
fun_text_processing/text_normalization/ru/verbalizers/money.py | 14
fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize.py | 13
egs/aishell2/paraformerbert/path.sh | 6
fun_text_processing/num2words/num2words/lang_MY.py | 97
fun_text_processing/inverse_text_normalization/en/utils.py | 14
funasr/modules/data2vec/ema_module.py | 132
fun_text_processing/num2words/num2words/base.py | 300
egs/aishell2/transformer/utils/filter_scp.pl | 0
fun_text_processing/num2words/num2words/lang_NE.py | 250
fun_text_processing/inverse_text_normalization/de/verbalizers/measure.py | 13
fun_text_processing/num2words/num2words/lang_SV.py | 102
fun_text_processing/inverse_text_normalization/ko/taggers/money.py | 11
fun_text_processing/inverse_text_normalization/tl/taggers/money.py | 11
egs/aishell2/conformer/conf/decode_asr_transformer.yaml | 6
fun_text_processing/inverse_text_normalization/fr/taggers/fraction.py | 13
fun_text_processing/text_normalization/en/verbalizers/money.py | 14
fun_text_processing/num2words/num2words/lang_BG.py | 91
fun_text_processing/inverse_text_normalization/ru/taggers/whitelist.py | 14
egs/aishell2/transformer/utils/compute_fbank.py | 0
egs/aishell2/transformer/utils/text_tokenize.py | 0
fun_text_processing/inverse_text_normalization/es/taggers/cardinal.py | 14
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer.py | 88
fun_text_processing/text_normalization/en/verbalizers/roman.py | 14
fun_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py | 11
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py | 36
egs/aishell2/transformer/utils/apply_lfr_and_cmvn.py | 0
fun_text_processing/text_normalization/ru/verbalizers/date.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py | 11
fun_text_processing/inverse_text_normalization/es/verbalizers/time.py | 13
fun_text_processing/text_normalization/de/utils.py | 13
fun_text_processing/text_normalization/es/verbalizers/electronic.py | 14
funasr/datasets/large_datasets/build_dataloader.py | 23
fun_text_processing/text_normalization/en/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py | 11
fun_text_processing/inverse_text_normalization/zh/taggers/word.py | 15
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/infer.py | 14
fun_text_processing/inverse_text_normalization/vi/taggers/punctuation.py | 13
fun_text_processing/inverse_text_normalization/es/taggers/measure.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/infer.py | 14
fun_text_processing/text_normalization/en/clean_eval_data.py | 13
fun_text_processing/inverse_text_normalization/es/verbalizers/telephone.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/infer.py | 13
fun_text_processing/inverse_text_normalization/fr/taggers/money.py | 13
fun_text_processing/inverse_text_normalization/vi/taggers/cardinal.py | 14
fun_text_processing/text_normalization/de/taggers/electronic.py | 14
fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py | 11
fun_text_processing/text_normalization/ru/taggers/electronic.py | 13
egs/aishell2/transformer/utils/text_tokenize.sh | 0
fun_text_processing/inverse_text_normalization/en/verbalizers/money.py | 13
fun_text_processing/text_normalization/es/taggers/decimals.py | 14
funasr/bin/asr_inference_paraformer_timestamp.py | 639
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/README.md | 30
egs/aishell2/paraformer/path.sh | 6
fun_text_processing/inverse_text_normalization/id/id_itn_test_expected.txt | 29
fun_text_processing/inverse_text_normalization/de/verbalizers/decimal.py | 13
fun_text_processing/inverse_text_normalization/vi/graph_utils.py | 13
fun_text_processing/num2words/num2words/lang_AR.py | 352
funasr/samplers/build_batch_sampler.py | 3
fun_text_processing/num2words/num2words/lang_PT.py | 226
fun_text_processing/text_normalization/en/taggers/abbreviation.py | 14
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer_after_finetune.py | 52
fun_text_processing/text_normalization/de/verbalizers/measure.py | 13
fun_text_processing/inverse_text_normalization/de/taggers/whitelist.py | 13
egs/aishell2/transformer/utils/compute_fbank.sh | 0
fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py | 11
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/infer.py | 13
fun_text_processing/inverse_text_normalization/zh/__init__.py | 14
egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md | 30
fun_text_processing/text_normalization/de/verbalizers/date.py | 13
fun_text_processing/inverse_text_normalization/vi/taggers/money.py | 13
fun_text_processing/text_normalization/de/taggers/money.py | 13
fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py | 11
fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py | 11
egs/cnceleb/resnet34/sid/compute_min_dcf.py | 151
funasr/modules/data2vec/multihead_attention.py | 671
fun_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py | 14
fun_text_processing/inverse_text_normalization/de/taggers/fraction.py | 13
fun_text_processing/text_normalization/data_loader_utils.py | 14
funasr/models/predictor/cif.py | 477
fun_text_processing/inverse_text_normalization/ja/taggers/time.py | 12
fun_text_processing/inverse_text_normalization/zh/taggers/time.py | 11
fun_text_processing/inverse_text_normalization/zh/verbalizers/word.py | 11
fun_text_processing/text_normalization/ru/verbalizers/time.py | 14
fun_text_processing/num2words/num2words/lang_SI.py | 108
fun_text_processing/num2words/num2words/lang_JA.py | 574
fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py | 12
fun_text_processing/inverse_text_normalization/tl/graph_utils.py | 13
fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py | 11
fun_text_processing/text_normalization/ru/verbalizers/measure.py | 14
funasr/tasks/vad.py | 342
egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py | 13
funasr/tasks/punctuation.py | 228
funasr/bin/asr_inference_paraformer.py | 510
fun_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py | 12
fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py | 11
fun_text_processing/text_normalization/token_parser.py | 14
egs/aishell2/transformer/utils/shuffle_list.pl | 0
egs/aishell2/transformer/utils/print_args.py | 0
funasr/tasks/abs_task.py | 193
fun_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py | 13
funasr/train/distributed_utils.py | 4
fun_text_processing/inverse_text_normalization/tl/verbalizers/whitelist.py | 11
fun_text_processing/inverse_text_normalization/pt/__init__.py | 14
funasr/models/encoder/data2vec_encoder.py | 577
fun_text_processing/inverse_text_normalization/id/taggers/money.py | 12
funasr/modules/streaming_utils/utils.py | 46
fun_text_processing/text_normalization/en/taggers/decimal.py | 14
fun_text_processing/text_normalization/zh/verbalizers/verbalize.py | 14
fun_text_processing/inverse_text_normalization/es/verbalizers/word.py | 13
egs/aishell2/transformer/conf/train_asr_transformer.yaml | 80
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/infer.py | 13
egs/aishell2/paraformer/conf/decode_asr_transformer.yaml | 6
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/en/taggers/measure.py | 13
fun_text_processing/text_normalization/ru/verbalizers/ordinal.py | 14
funasr/models/encoder/fsmn_encoder.py | 316
egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml | 12
fun_text_processing/inverse_text_normalization/es/taggers/electronic.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/infer.py | 13
fun_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py | 14
fun_text_processing/inverse_text_normalization/tl/verbalizers/cardinal.py | 12
fun_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py | 13
funasr/models/e2e_asr_paraformer.py | 1632 +
fun_text_processing/num2words/num2words/lang_UK.py | 837 +
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/finetune.py | 35
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer_after_finetune.py | 53
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer.py | 87
fun_text_processing/inverse_text_normalization/de/taggers/measure.py | 13
egs/aishell2/transformer/run.sh | 254
fun_text_processing/num2words/tox.ini | 32
fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py | 14
fun_text_processing/inverse_text_normalization/en/taggers/ordinal.py | 13
fun_text_processing/text_normalization/en/verbalizers/whitelist.py | 13
fun_text_processing/text_normalization/ru/data/utils/g.fst | 0
fun_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py | 14
egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py | 36
fun_text_processing/inverse_text_normalization/zh/taggers/fraction.py | 11
fun_text_processing/inverse_text_normalization/fr/taggers/cardinal.py | 13
fun_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py | 13
fun_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py | 13
egs/aishell/paraformerbert/run.sh | 22
fun_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py | 13
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py | 13
fun_text_processing/text_normalization/de/taggers/cardinal.py | 13
fun_text_processing/text_normalization/zh/verbalizers/char.py | 14
fun_text_processing/text_normalization/de/taggers/decimal.py | 13
fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py | 14
fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py | 11
fun_text_processing/inverse_text_normalization/es/verbalizers/measure.py | 13
egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md | 30
fun_text_processing/inverse_text_normalization/de/taggers/ordinal.py | 13
egs/aishell2/transformer/utils/split_scp.pl | 0
fun_text_processing/inverse_text_normalization/tl/verbalizers/decimal.py | 12
fun_text_processing/inverse_text_normalization/zh/verbalizers/time.py | 11
fun_text_processing/inverse_text_normalization/export_models.py | 29
fun_text_processing/README.md | 35
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/infer.py | 13
fun_text_processing/inverse_text_normalization/pt/verbalizers/measure.py | 14
fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py | 12
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/finetune.py | 35
fun_text_processing/text_normalization/en/verbalizers/post_processing.py | 14
egs/aishell2/transformer/utils/run.pl | 0
fun_text_processing/inverse_text_normalization/id/taggers/decimal.py | 12
fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py | 11
fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py | 12
fun_text_processing/text_normalization/zh/taggers/money.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/punctuation.py | 13
egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/eval_eer.py | 69
fun_text_processing/text_normalization/es/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/zh/taggers/money.py | 11
fun_text_processing/inverse_text_normalization/zh/verbalizers/date.py | 11
fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py | 13
fun_text_processing/inverse_text_normalization/en/taggers/telephone.py | 13
funasr/utils/compute_eer.py | 59
fun_text_processing/text_normalization/en/taggers/telephone.py | 14
fun_text_processing/text_normalization/es/taggers/date.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/telephone.py | 13
egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md | 30
fun_text_processing/text_normalization/zh/taggers/preprocessor.py | 14
fun_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py | 13
fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py | 17
fun_text_processing/inverse_text_normalization/id/utils.py | 14
egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py | 14
fun_text_processing/text_normalization/es/taggers/word.py | 14
funasr/bin/asr_inference_launch.py | 51
fun_text_processing/text_normalization/es/verbalizers/decimals.py | 14
fun_text_processing/text_normalization/ru/verbalizers/verbalize.py | 14
funasr/bin/punctuation_infer.py | 374
fun_text_processing/inverse_text_normalization/fr/graph_utils.py | 13
egs/aishell/tranformer/utils/compute_fbank.py | 24
egs/aishell2/conformer/local/prepare_data.sh | 53
fun_text_processing/num2words/requirements-test.txt | 6
fun_text_processing/text_normalization/en/verbalizers/decimal.py | 14
fun_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py | 13
fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py | 11
fun_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/README.md | 30
fun_text_processing/num2words/num2words/lang_FR_CH.py | 33
egs/aishell2/conformer/path.sh | 6
fun_text_processing/inverse_text_normalization/pt/taggers/punctuation.py | 14
fun_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py | 13
fun_text_processing/num2words/num2words/lang_CZ.py | 142
fun_text_processing/inverse_text_normalization/ru/taggers/money.py | 14
fun_text_processing/num2words/num2words/lang_ES_CO.py | 14
funasr/models/pooling/statistic_pooling.py | 36
egs/cnceleb/resnet34/path.sh | 0
fun_text_processing/inverse_text_normalization/en/taggers/electronic.py | 13
fun_text_processing/inverse_text_normalization/en/verbalizers/decimal.py | 14
fun_text_processing/text_normalization/en/verbalizers/telephone.py | 14
fun_text_processing/inverse_text_normalization/vi/verbalizers/measure.py | 13
fun_text_processing/inverse_text_normalization/zh/graph_utils.py | 13
fun_text_processing/num2words/setup.py | 64
fun_text_processing/text_normalization/de/verbalizers/electronic.py | 13
fun_text_processing/text_normalization/normalize_with_audio.py | 13
fun_text_processing/text_normalization/ru/data/utils/universal_thousands_punct.far | 0
egs/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml | 6
egs/aishell2/transformer/conf/decode_asr_transformer.yaml | 6
fun_text_processing/text_normalization/en/taggers/ordinal.py | 14
fun_text_processing/text_normalization/zh/taggers/whitelist.py | 14
fun_text_processing/inverse_text_normalization/tl/clean_eval_data.py | 13
fun_text_processing/inverse_text_normalization/tl/verbalizers/date.py | 11
egs/aishell2/paraformerbert/run.sh | 263
fun_text_processing/inverse_text_normalization/ru/verbalizers/date.py | 13
fun_text_processing/text_normalization/es/taggers/cardinal.py | 14
fun_text_processing/num2words/num2words/lang_IW.py | 91
fun_text_processing/inverse_text_normalization/zh/taggers/telephone.py | 11
fun_text_processing/inverse_text_normalization/en/verbalizers/measure.py | 13
egs/aishell2/transformer/utils/text2token.py | 0
fun_text_processing/num2words/num2words/lang_RO.py | 158
fun_text_processing/inverse_text_normalization/vi/taggers/time.py | 14
funasr/modules/data2vec/grad_multiply.py | 18
fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py | 13
fun_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py | 14
fun_text_processing/num2words/num2words/__init__.py | 147
fun_text_processing/inverse_text_normalization/zh/taggers/date.py | 11
fun_text_processing/inverse_text_normalization/id/clean_eval_data.py | 14
fun_text_processing/inverse_text_normalization/tl/verbalizers/telephone.py | 11
fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py | 11
fun_text_processing/inverse_text_normalization/ja/taggers/money.py | 12
fun_text_processing/num2words/num2words/lang_EN_IN.py | 11
fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py | 11
fun_text_processing/text_normalization/en/taggers/money.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py | 13
egs/cnceleb/resnet34/run.sh | 138
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/fr/taggers/decimal.py | 13
fun_text_processing/text_normalization/zh/verbalizers/time.py | 14
fun_text_processing/num2words/num2words/lang_HU.py | 150
funasr/models/pooling/__init__.py | 0
egs/cnceleb/resnet34/README.md | 18
fun_text_processing/inverse_text_normalization/fr/taggers/whitelist.py | 14
fun_text_processing/num2words/docker-compose.yml | 7
fun_text_processing/text_normalization/ru/taggers/telephone.py | 13
funasr/bin/asr_inference_paraformer_vad_punc.py | 1134 +
fun_text_processing/inverse_text_normalization/ko/taggers/word.py | 11
fun_text_processing/text_normalization/en/taggers/fraction.py | 14
fun_text_processing/inverse_text_normalization/es/verbalizers/date.py | 13
fun_text_processing/inverse_text_normalization/fr/taggers/measure.py | 13
fun_text_processing/inverse_text_normalization/zh/taggers/measure.py | 11
egs/cnceleb/resnet34/local/convert_tf_ckpt_to_torch.py | 93
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/finetune.py | 35
fun_text_processing/inverse_text_normalization/vi/taggers/telephone.py | 13
fun_text_processing/num2words/num2words/lang_ES_NI.py | 18
funasr/punctuation/__init__.py | 0
fun_text_processing/inverse_text_normalization/ja/taggers/word.py | 12
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/infer.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/infer.py | 14
funasr/datasets/large_datasets/utils/padding.py | 2
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py | 14
fun_text_processing/inverse_text_normalization/zh/verbalizers/measure.py | 11
fun_text_processing/inverse_text_normalization/de/verbalizers/time.py | 13
fun_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py | 13
fun_text_processing/text_normalization/export_models.py | 14
fun_text_processing/num2words/num2words/lang_KN.py | 167
fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py | 11
fun_text_processing/text_normalization/en/verbalizers/time.py | 14
fun_text_processing/num2words/num2words/lang_TH.py | 168
fun_text_processing/text_normalization/de/taggers/whitelist.py | 13
egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/README.md | 19
funasr/bin/sv_inference.py | 434
fun_text_processing/inverse_text_normalization/es/verbalizers/money.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/infer.py | 13
fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py | 12
fun_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py | 14
fun_text_processing/inverse_text_normalization/run_evaluate.py | 14
fun_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/electronic.py | 13
fun_text_processing/inverse_text_normalization/pt/taggers/word.py | 14
egs/aishell2/paraformerbert/local/extract_embeds.sh | 75
fun_text_processing/text_normalization/zh/taggers/fraction.py | 14
fun_text_processing/text_normalization/en/taggers/word.py | 14
fun_text_processing/num2words/num2words/lang_LT.py | 167
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/README.md | 30
egs/aishell/tranformer/utils/fix_data_feat.sh | 8
fun_text_processing/inverse_text_normalization/id/verbalizers/whitelist.py | 12
fun_text_processing/inverse_text_normalization/id/taggers/tokenize_and_classify.py | 12
fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize_final.py | 11
fun_text_processing/text_normalization/ru/alphabet.py | 17
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer_after_finetune.py | 53
funasr/models/encoder/resnet34_encoder.py | 144
egs/aishell/paraformerbert/local/extract_embeds.sh | 13
fun_text_processing/text_normalization/zh/verbalizers/date.py | 14
funasr/models/encoder/sanm_encoder.py | 329
fun_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/fraction.py | 12
egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/infer.py | 15
fun_text_processing/inverse_text_normalization/ru/taggers/date.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/finetune.py | 35
fun_text_processing/text_normalization/ru/verbalizers/cardinal.py | 13
fun_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py | 14
fun_text_processing/inverse_text_normalization/pt/taggers/date.py | 14
fun_text_processing/inverse_text_normalization/en/verbalizers/electronic.py | 14
fun_text_processing/inverse_text_normalization/tl/verbalizers/measure.py | 11
funasr/datasets/iterable_dataset.py | 271
fun_text_processing/text_normalization/es/verbalizers/measure.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer.py | 14
egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer.py | 33
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/finetune.py | 35
fun_text_processing/text_normalization/en/taggers/date.py | 14
fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py | 13
fun_text_processing/num2words/num2words/lang_EU.py | 76
egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py | 14
fun_text_processing/inverse_text_normalization/pt/taggers/whitelist.py | 14
fun_text_processing/inverse_text_normalization/de/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/fr/taggers/time.py | 13
fun_text_processing/inverse_text_normalization/es/taggers/telephone.py | 13
fun_text_processing/inverse_text_normalization/id/verbalizers/measure.py | 12
fun_text_processing/text_normalization/ru/verbalizers/telephone.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/infer.py | 14
fun_text_processing/inverse_text_normalization/es/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/id/taggers/cardinal.py | 12
fun_text_processing/inverse_text_normalization/vi/utils.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/finetune.py | 35
fun_text_processing/text_normalization/es/graph_utils.py | 14
fun_text_processing/inverse_text_normalization/ru/verbalizers/time.py | 13
fun_text_processing/text_normalization/zh/verbalizers/cardinal.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/electronic.py | 12
fun_text_processing/inverse_text_normalization/pt/taggers/decimal.py | 14
fun_text_processing/text_normalization/es/taggers/whitelist.py | 14
fun_text_processing/num2words/.gitignore | 7
egs/aishell2/transformer/utils/subset_data_dir_tr_cv.sh | 0
egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py | 15
fun_text_processing/inverse_text_normalization/pt/verbalizers/time.py | 14
fun_text_processing/text_normalization/es/taggers/tokenize_and_classify.py | 15
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/finetune.py | 35
fun_text_processing/text_normalization/run_evaluate.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py | 11
fun_text_processing/text_normalization/ru/taggers/number_names.py | 18
fun_text_processing/num2words/num2words/lang_ES.py | 351
fun_text_processing/text_normalization/ru/verbalizers/decimal.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py | 11
fun_text_processing/inverse_text_normalization/tl/verbalizers/time.py | 11
egs/aishell2/transformer/utils/extract_embeds.py | 0
fun_text_processing/inverse_text_normalization/pt/taggers/time.py | 14
funasr/modules/streaming_utils/load_fr_tf.py | 62
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/finetune.py | 35
fun_text_processing/inverse_text_normalization/vi/taggers/ordinal.py | 13
fun_text_processing/text_normalization/en/taggers/range.py | 13
fun_text_processing/text_normalization/ru/taggers/decimals.py | 13
fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py | 11
fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize.py | 11
fun_text_processing/inverse_text_normalization/id/taggers/telephone.py | 12
fun_text_processing/num2words/num2words/lang_ES_VE.py | 17
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/finetune.py | 35
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/README.md | 30
fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py | 13
egs/aishell2/transformer/local/prepare_data.sh | 53
fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py | 11
fun_text_processing/inverse_text_normalization/ru/taggers/measure.py | 14
fun_text_processing/inverse_text_normalization/vi/__init__.py | 14
fun_text_processing/inverse_text_normalization/vi/verbalizers/money.py | 13
fun_text_processing/inverse_text_normalization/zh/taggers/electronic.py | 11
egs/aishell/tranformer/utils/compute_fbank.sh | 5
fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py | 13
fun_text_processing/inverse_text_normalization/de/taggers/telephone.py | 13
fun_text_processing/text_normalization/zh/verbalizers/whitelist.py | 14
egs/aishell2/conformer/conf/train_asr_conformer.yaml | 86
fun_text_processing/inverse_text_normalization/es/taggers/money.py | 13
fun_text_processing/inverse_text_normalization/tl/taggers/fraction.py | 11
fun_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py | 14
fun_text_processing/inverse_text_normalization/pt/verbalizers/date.py | 14
egs/aishell2/paraformerbert/conf/train_asr_paraformerbert_conformer_20e_6d_1280_320.yaml | 101
fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py | 11
fun_text_processing/inverse_text_normalization/en/clean_eval_data.py | 14
fun_text_processing/inverse_text_normalization/pt/verbalizers/money.py | 14
egs/aishell2/conformer/run.sh | 254
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/README.md | 30
funasr/modules/subsampling.py | 129
fun_text_processing/inverse_text_normalization/en/taggers/cardinal.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/finetune.py | 35
fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py | 11
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/finetune.py | 36
fun_text_processing/inverse_text_normalization/vi/taggers/fraction.py | 13
fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize.py | 13
fun_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py | 13
fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py | 11
fun_text_processing/inverse_text_normalization/tl/__init__.py | 13
fun_text_processing/num2words/num2words/compat.py | 14
fun_text_processing/num2words/num2words/lang_AZ.py | 89
fun_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py | 14
fun_text_processing/text_normalization/en/taggers/punctuation.py | 14
fun_text_processing/num2words/num2words/lang_RU.py | 231
fun_text_processing/num2words/num2words/lang_NO.py | 89
fun_text_processing/text_normalization/de/taggers/measure.py | 13
funasr/punctuation/abs_model.py | 29
fun_text_processing/inverse_text_normalization/en/taggers/fraction.py | 13
fun_text_processing/text_normalization/zh/graph_utils.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/README.md | 30
fun_text_processing/text_normalization/zh/taggers/time.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/money.py | 12
fun_text_processing/inverse_text_normalization/id/text_cases.tsv | 40
fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py | 13
fun_text_processing/num2words/num2words/lang_SL.py | 169
fun_text_processing/inverse_text_normalization/vi/verbalizers/time.py | 13
fun_text_processing/text_normalization/es/verbalizers/verbalize_final.py | 14
egs/aishell2/paraformer/utils | 1
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/README.md | 30
funasr/utils/timestamp_tools.py | 99
fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py | 12
fun_text_processing/inverse_text_normalization/zh/taggers/punctuation.py | 11
fun_text_processing/inverse_text_normalization/pt/taggers/electronic.py | 14
fun_text_processing/text_normalization/ru/taggers/date.py | 13
egs/aishell2/transformer/utils/gen_ark_list.sh | 0
fun_text_processing/inverse_text_normalization/id/verbalizers/word.py | 12
funasr/models/decoder/sanm_decoder.py | 774 +
fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize_final.py | 12
fun_text_processing/inverse_text_normalization/ko/graph_utils.py | 13
fun_text_processing/inverse_text_normalization/es/verbalizers/decimal.py | 13
fun_text_processing/inverse_text_normalization/es/taggers/date.py | 13
fun_text_processing/num2words/num2words/lang_LV.py | 174
fun_text_processing/num2words/MANIFEST.in | 1
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py | 35
fun_text_processing/text_normalization/zh/taggers/measure.py | 14
fun_text_processing/inverse_text_normalization/ru/taggers/decimals.py | 14
fun_text_processing/inverse_text_normalization/en/taggers/punctuation.py | 13
fun_text_processing/text_normalization/de/verbalizers/cardinal.py | 13
fun_text_processing/text_normalization/ru/data/utils/util_byte.far | 0
fun_text_processing/text_normalization/en/verbalizers/verbalize.py | 14
fun_text_processing/inverse_text_normalization/es/utils.py | 13
fun_text_processing/text_normalization/es/verbalizers/telephone.py | 14
funasr/samplers/length_batch_sampler.py | 12
fun_text_processing/inverse_text_normalization/vi/taggers/word.py | 13
fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py | 14
fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py | 11
fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py | 11
fun_text_processing/text_normalization/zh/taggers/char.py | 14
fun_text_processing/inverse_text_normalization/tl/taggers/word.py | 11
fun_text_processing/text_normalization/ru/taggers/measure.py | 13
setup.py | 6
fun_text_processing/inverse_text_normalization/fr/taggers/date.py | 13
fun_text_processing/text_normalization/en/verbalizers/date.py | 14
fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py | 13
fun_text_processing/text_normalization/es/taggers/ordinal.py | 14
fun_text_processing/inverse_text_normalization/pt/taggers/telephone.py | 14
fun_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py | 13
funasr/layers/stft.py | 15
fun_text_processing/inverse_text_normalization/en/verbalizers/word.py | 13
fun_text_processing/inverse_text_normalization/zh/verbalizers/money.py | 11
funasr/datasets/large_datasets/dataset.py | 18
fun_text_processing/text_normalization/ru/verbalizers/verbalize_final.py | 13
funasr/bin/asr_inference.py | 421
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/infer.py | 13
fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py | 12
egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/README.md | 53
fun_text_processing/inverse_text_normalization/de/taggers/decimal.py | 13
fun_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py | 13
fun_text_processing/text_normalization/en/verbalizers/electronic.py | 14
fun_text_processing/inverse_text_normalization/ko/utils.py | 13
fun_text_processing/num2words/.coveragerc | 5
egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/infer.py | 23
egs/cnceleb/resnet34/local/make_cnceleb1.sh | 62
fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py | 11
egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer_sv.py | 21
fun_text_processing/text_normalization/de/taggers/ordinal.py | 13
fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize.py | 13
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/infer.py | 14
fun_text_processing/text_normalization/zh/taggers/date.py | 14
fun_text_processing/inverse_text_normalization/id/taggers/ordinal.py | 12
funasr/models/e2e_uni_asr.py | 11
fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize.py | 14
fun_text_processing/inverse_text_normalization/ko/taggers/date.py | 11
fun_text_processing/num2words/num2words/lang_HE.py | 144
egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py | 36
fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py | 12
fun_text_processing/inverse_text_normalization/ja/taggers/measure.py | 12
fun_text_processing/inverse_text_normalization/ru/verbalizers/measure.py | 13
fun_text_processing/num2words/num2words/lang_SW.py | 108
fun_text_processing/text_normalization/zh/verbalizers/money.py | 14
fun_text_processing/num2words/num2words/lang_EL.py | 115
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py | 88
fun_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/finetune.py | 35
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/README.md | 30
fun_text_processing/num2words/num2words/currency.py | 35
egs/aishell2/transformer/utils/fix_data.sh | 0
fun_text_processing/inverse_text_normalization/ru/taggers/cardinal.py | 14
fun_text_processing/inverse_text_normalization/id/taggers/measure.py | 12
egs/aishell2/transformer/utils/compute_cmvn.py | 0
fun_text_processing/text_normalization/de/taggers/date.py | 13
egs/aishell/conformer/run.sh | 8
fun_text_processing/inverse_text_normalization/en/verbalizers/time.py | 13
fun_text_processing/text_normalization/de/verbalizers/telephone.py | 13
funasr/modules/data2vec/wav2vec2.py | 407
fun_text_processing/inverse_text_normalization/en/taggers/whitelist.py | 13
funasr/bin/sv_inference_launch.py | 173
fun_text_processing/num2words/num2words/lang_CA.py | 89
fun_text_processing/inverse_text_normalization/pt/taggers/measure.py | 14
egs/aishell2/transformer/utils/__init__.py | 0
fun_text_processing/inverse_text_normalization/ko/taggers/measure.py | 11
funasr/utils/postprocess_utils.py | 84
fun_text_processing/inverse_text_normalization/fr/utils.py | 13
funasr/punctuation/espnet_model.py | 152
fun_text_processing/inverse_text_normalization/tl/taggers/time.py | 11
fun_text_processing/num2words/.travis.yml | 20
egs/cnceleb/resnet34/utils | 1
fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py | 13
fun_text_processing/text_normalization/es/verbalizers/cardinal.py | 14
fun_text_processing/text_normalization/de/verbalizers/fraction.py | 13
egs/aishell2/transformer/utils/compute_cmvn.sh | 0
fun_text_processing/text_normalization/es/taggers/measure.py | 14
fun_text_processing/text_normalization/zh/verbalizers/verbalize_final.py | 14
fun_text_processing/text_normalization/es/verbalizers/verbalize.py | 14
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/README.md | 30
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/finetune.py | 35
egs/aishell2/paraformer/local/prepare_data.sh | 53
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/finetune.py | 35
fun_text_processing/inverse_text_normalization/id/taggers/time.py | 12
fun_text_processing/num2words/num2words/lang_PT_BR.py | 98
fun_text_processing/text_normalization/es/taggers/electronic.py | 14
fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py | 11
fun_text_processing/inverse_text_normalization/id/taggers/fraction.py | 12
fun_text_processing/text_normalization/de/verbalizers/decimal.py | 13
fun_text_processing/text_normalization/zh/taggers/math_symbol.py | 14
egs/cnceleb/resnet34/sid/compute_eer.py | 54
fun_text_processing/inverse_text_normalization/pt/taggers/ordinal.py | 14
egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/README.md | 24
egs/aishell2/paraformerbert/utils | 1
funasr/modules/data2vec/utils.py | 156
fun_text_processing/text_normalization/en/verbalizers/fraction.py | 14
egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer_after_finetune.py | 53
fun_text_processing/text_normalization/ru/taggers/time.py | 14
funasr/modules/data2vec/quant_noise.py | 107
fun_text_processing/inverse_text_normalization/vi/taggers/date.py | 13
fun_text_processing/inverse_text_normalization/de/verbalizers/money.py | 13
fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py | 12
fun_text_processing/inverse_text_normalization/zh/taggers/decimal.py | 11
egs/aishell/paraformer/run.sh | 7
funasr/utils/modelscope_param.py | 25
funasr/utils/asr_utils.py | 10
funasr/modules/data2vec/__init__.py | 0
fun_text_processing/inverse_text_normalization/de/taggers/electronic.py | 13
fun_text_processing/inverse_text_normalization/en/taggers/time.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/finetune.py | 35
fun_text_processing/text_normalization/en/taggers/tokenize_and_classify.py | 14
funasr/models/e2e_sv.py | 270
funasr/utils/compute_min_dcf.py | 159
fun_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py | 14
fun_text_processing/text_normalization/ru/taggers/word.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/decimal.py | 12
egs/aishell/tranformer/utils/download_model.py | 20
fun_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py | 14
fun_text_processing/inverse_text_normalization/id/taggers/date.py | 12
fun_text_processing/text_normalization/en/verbalizers/ordinal.py | 14
egs/aishell2/transformer/utils/parse_options.sh | 0
fun_text_processing/num2words/num2words/lang_FR_BE.py | 34
fun_text_processing/text_normalization/de/verbalizers/money.py | 13
fun_text_processing/inverse_text_normalization/ru/taggers/electronic.py | 14
fun_text_processing/inverse_text_normalization/id/verbalizers/date.py | 12
fun_text_processing/text_normalization/zh/verbalizers/measure.py | 14
egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py | 15
fun_text_processing/text_normalization/de/verbalizers/verbalize.py | 13
fun_text_processing/num2words/num2words/lang_IT.py | 208
funasr/datasets/large_datasets/utils/tokenize.py | 37
fun_text_processing/num2words/num2words/lang_EN.py | 95
fun_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py | 13
fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize.py | 12
fun_text_processing/text_normalization/ru/data/utils/util_arithmetic.far | 0
funasr/version.txt | 2
docs/images/.DS_Store | 0
fun_text_processing/inverse_text_normalization/ja/__init__.py | 14
funasr/datasets/large_datasets/utils/filter.py | 15
fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py | 12
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/finetune.py | 35
fun_text_processing/text_normalization/en/verbalizers/abbreviation.py | 14
fun_text_processing/inverse_text_normalization/tl/verbalizers/electronic.py | 11
fun_text_processing/inverse_text_normalization/id/taggers/word.py | 12
fun_text_processing/inverse_text_normalization/pt/utils.py | 14
funasr/datasets/preprocessor.py | 202
egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md | 30
funasr/punctuation/target_delay_transformer.py | 137
fun_text_processing/inverse_text_normalization/tl/taggers/measure.py | 11
fun_text_processing/text_normalization/en/taggers/cardinal.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/infer.py | 13
fun_text_processing/num2words/num2words/utils.py | 20
fun_text_processing/inverse_text_normalization/zh/verbalizers/telephone.py | 11
fun_text_processing/text_normalization/en/taggers/serial.py | 13
funasr/tasks/asr.py | 1675 +-
fun_text_processing/inverse_text_normalization/en/taggers/word.py | 13
fun_text_processing/text_normalization/en/verbalizers/measure.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer.py | 14
fun_text_processing/inverse_text_normalization/en/taggers/date.py | 13
/dev/null | 1
funasr/punctuation/text_preprocessor.py | 33
fun_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py | 13
fun_text_processing/text_normalization/de/taggers/word.py | 14
fun_text_processing/text_normalization/es/__init__.py | 14
egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/finetune.py | 35
fun_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py | 12
fun_text_processing/inverse_text_normalization/tl/taggers/ordinal.py | 11
836 files changed, 34,273 insertions(+), 8,728 deletions(-)
diff --git a/README.md b/README.md
index 6dd38b2..6bf1278 100644
--- a/README.md
+++ b/README.md
@@ -2,38 +2,50 @@
# FunASR: A Fundamental End-to-End Speech Recognition Toolkit
-<strong>FunASR</strong> hopes to build a bridge between academic research and industrial applications on speech recognition. By supporting the training & finetuning of the industrial-grade speech recognition model released on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition), researchers and developers can conduct research and production of speech recognition models more conveniently, and promote the development of speech recognition ecology. ASR for Fun锛�
+<strong>FunASR</strong> hopes to build a bridge between academic research and industrial applications on speech recognition. By supporting the training & finetuning of the industrial-grade speech recognition model released on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition), researchers and developers can conduct research and production of speech recognition models more conveniently, and promote the development of speech recognition ecology. ASR for Fun锛乕Model Zoo](docs/modelscope_models.md)
-## Highlights
-- FunASR supports many types of models, such as, Tranformer, Conformer, [Paraformer](https://arxiv.org/abs/2206.08317).
-- A large number of ASR models trained on academic datasets or industrial datasets are open sourced on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition),
-- The pretrained model [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) obtains the first place on many task in [SpeechIO leaderboard](https://github.com/SpeechColab/Leaderboard)
-- FunASR supports large-scale dataset dataloader and multi-GPU training.
+## Release Notes:
+### 2023.1.16, funasr-0.1.6
+- We release a new version model [Paraformer-large-long](https://modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary), which integrate the [VAD](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) model, [ASR](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary),
+ [Punctuation](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) model and timestamp together. The model could take in several hours long inputs.
+- We release a new type model, [VAD](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary), which could predict the duration of none-silence speech. It could be freely integrated with any ASR models in [Model Zoo](docs/modelscope_models.md).
+- We release a new type model, [Punctuation](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary), which could predict the punctuation of ASR models's results. It could be freely integrated with any ASR models in [Model Zoo](docs/modelscope_models.md).
+- We release a new model, [Data2vec](https://www.modelscope.cn/models/damo/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/summary), an unsupervised pretraining model which could be finetuned on ASR and other downstream tasks.
+- We release a new model, [Paraformer-Tiny](https://www.modelscope.cn/models/damo/speech_paraformer-tiny-commandword_asr_nat-zh-cn-16k-vocab544-pytorch/summary), a lightweight Paraformer model which supports Mandarin command words recognition.
+- We release a new type model, [SV](https://www.modelscope.cn/models/damo/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/summary), which could extract speaker embeddings and further perform speaker verification on paired utterances. It will be supported for speaker diarization in the future version.
+- We improve the pipeline of modelscope to speedup the inference, by integrating the process of build model into build pipeline.
+- Various new types of audio input types are now supported by modelscope inference pipeline, including wav.scp, wav format, audio bytes, wave samples...
+
+## Key Features
+- Many types of typical models are supported, e.g., [Tranformer](https://arxiv.org/abs/1706.03762), [Conformer](https://arxiv.org/abs/2005.08100), [Paraformer](https://arxiv.org/abs/2206.08317).
+- We have released large number of academic and industrial pretrained models on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition)
+- The pretrained model [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) obtains the best performance on many tasks in [SpeechIO leaderboard](https://github.com/SpeechColab/Leaderboard)
+- FunASR supplies a easy-to-use pipeline to finetune pretrained models from [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition)
+- Compared to [Espnet](https://github.com/espnet/espnet) framework, the training speed of large-scale datasets in FunASR is much faster owning to the optimized dataloader.
## Installation(Training and Developing)
-
-- Clone the repo:
-``` sh
-git clone https://github.com/alibaba/FunASR.git
-```
- Install Conda:
``` sh
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
sh Miniconda3-latest-Linux-x86_64.sh
+source ~/.bashrc
conda create -n funasr python=3.7
conda activate funasr
```
- Install Pytorch (version >= 1.7.0):
-
-| cuda | |
-|:-----:| --- |
-| 9.2 | conda install pytorch==1.7.0 torchvision==0.8.0 torchaudio==0.7.0 cudatoolkit=9.2 -c pytorch |
-| 10.2 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=10.2 -c pytorch |
-| 11.1 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=11.1 -c pytorch |
-
+``` sh
+pip3 install torch torchvision torchaudio
+```
For more versions, please see [https://pytorch.org/get-started/locally](https://pytorch.org/get-started/locally)
+
+
+If you are in the area of China, you could set the source to speed the downloading.
+
+``` sh
+pip config set global.index-url https://mirror.sjtu.edu.cn/pypi/web/simple
+```
- Install ModelScope:
``` sh
@@ -45,10 +57,11 @@
- Install FunASR and other packages:
``` sh
+git clone https://github.com/alibaba/FunASR.git && cd FunASR
pip install --editable ./
```
-## Pretrained model hub
+## Pretrained Model Zoo
We have trained many academic and industrial models, [model hub](docs/modelscope_models.md)
@@ -59,7 +72,7 @@
- email: [funasr@list.alibaba-inc.com](funasr@list.alibaba-inc.com)
- Dingding group:
-<div align="left"><img src="docs/images/dingding.jpg" width="400"/></div>
+<div align="left"><img src="docs/images/dingding.jpg" width="250"/>!<img src="docs/images/wechat.png" width="222"/></div>
## Acknowledge
diff --git a/docs/Makefile b/docs/Makefile
deleted file mode 100644
index d58379b..0000000
--- a/docs/Makefile
+++ /dev/null
@@ -1,21 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS =
-SPHINXBUILD = sphinx-build
-SPHINXPROJ = FunASR
-SOURCEDIR = .
-BUILDDIR = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
- @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
- @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
deleted file mode 100644
index 0189991..0000000
--- a/docs/conf.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-
-# -- Project information -----------------------------------------------------
-
-project = 'FunASR'
-copyright = '2022, Speech Lab, Alibaba Group'
-author = 'Speech Lab, Alibaba Grou'
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
- "nbsphinx",
- "sphinx.ext.autodoc",
- 'sphinx.ext.napoleon',
- 'sphinx.ext.viewcode',
- "sphinx.ext.mathjax",
- "sphinx.ext.todo",
- # "sphinxarg.ext",
- "sphinx_markdown_tables",
- 'recommonmark',
- 'sphinx_rtd_theme',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-source_suffix = [".rst", ".md"]
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-#
-
-html_theme = "sphinx_rtd_theme"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
\ No newline at end of file
diff --git a/docs/get_started.md b/docs/get_started.md
deleted file mode 100644
index ca009ef..0000000
--- a/docs/get_started.md
+++ /dev/null
@@ -1,130 +0,0 @@
-# Get Started
-This is an easy example which introduces how to train a paraformer model on AISHELL-1 data from scratch. According to this example, you can train other models (conformer, paraformer, etc.) on other datasets (AISHELL-1, AISHELL-2, etc.) similarly.
-
-## Overall Introduction
-We provide a recipe `egs/aishell/paraformer/run.sh` for training a paraformer model on AISHELL-1 data . This recipe consists of five stages and support training on multiple GPUs and decoding by CPU or GPU. Before introduce each stage in detail, we first explain several variables which should be set by users.
-- `CUDA_VISIBLE_DEVICES`: visible gpu list
-- `gpu_num`: the number of GPUs used for training
-- `gpu_inference`: whether to use GPUs for decoding
-- `njob`: for CPU decoding, indicating the total number of CPU jobs; for GPU decoding, indicating the number of jobs on each GPU.
-- `feats_dir`: the path to save processed data
-- `exp_dir`: the path to save experimental results
-- `data_aishell`: the path of raw AISHELL-1 data
-- `tag`: the suffix of experimental result directory
-- `nj`: the number of jobs for data preparation
-- `speed_perturb`: the range of speech perturbed
-
-## Stage 0: Data preparation
-This stage processes raw AISHELL-1 data `$data_aishell` and generates the corresponding `wav.scp` and `text` in `$feats_dir/data/xxx` and `xxx` means `train/dev/test`. Here we assume you have already downloaded AISHELL-1 data. If not, you can download data [here](https://www.openslr.org/33/) and set the path for `$data_aishell`. Here we show examples for `wav.scp` and `text`, separately.
-* `wav.scp`
-```
-BAC009S0002W0122 /nfs/ASR_DATA/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0122.wav
-BAC009S0002W0123 /nfs/ASR_DATA/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0123.wav
-BAC009S0002W0124 /nfs/ASR_DATA/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0124.wav
-...
-```
-* `text`
-```
-BAC009S0002W0122 鑰� 瀵� 妤� 甯� 鎴� 浜� 鎶� 鍒� 浣� 鐢� 鏈� 澶� 鐨� 闄� 璐�
-BAC009S0002W0123 涔� 鎴� 涓� 鍦� 鏂� 鏀� 搴� 鐨� 鐪� 涓� 閽�
-BAC009S0002W0124 鑷� 鍏� 鏈� 搴� 鍛� 鍜� 娴� 鐗� 甯� 鐜� 鍏� 瀹� 甯� 鍙� 娑� 闄� 璐� 鍚�
-...
-```
-We can see that these two files both have two columns while the first column is the wav-id and the second column is the corresponding wav-path/label tokens.
-
-## Stage 1: Feature Generation
-This stage extracts FBank feature from raw wav `wav.scp` and apply speed perturbation as data augmentation according to `speed_perturb`. You can set `nj` to control the number of jobs for feature generation. The output features are saved in `$feats_dir/dump/xxx/ark` and the corresponding `feats.scp` files are saved as `$feats_dir/dump/xxx/feats.scp`. An example of `feats.scp` can be seen as follows:
-* `feats.scp`
-```
-...
-BAC009S0002W0122_sp0.9 /nfs/haoneng.lhn/funasr_data/aishell-1/dump/fbank/train/ark/feats.16.ark:592751055
-...
-```
-Note that samples in this file have already been shuffled. This file contains two columns. The first column is the wav-id while the second column is the kaldi-ark feature path. Besides, `speech_shape` and `text_shape` are also generated in this stage, denoting the speech feature shape and text length of each sample. The examples are shown as follows:
-* `speech_shape`
-```
-...
-BAC009S0002W0122_sp0.9 665,80
-...
-```
-* `text_shape`
-```
-...
-BAC009S0002W0122_sp0.9 15
-...
-```
-These two files have two columns. The first column is the wav-id and the second column is the corresponding speech feature shape and text length.
-
-## Stage 2: Dictionary Preparation
-This stage prepares a dictionary, which is used as a mapping between label characters and integer indices during ASR training. The output dictionary file is saved as `$feats_dir/data/$lang_toekn_list/$token_type/tokens.txt`. Here we show an example of `tokens.txt` as follows:
-* `tokens.txt`
-```
-<blank>
-<s>
-</s>
-涓�
-涓�
-...
-榫�
-榫�
-<unk>
-```
-* `<blank>`: indicates the blank token for CTC
-* `<s>`: indicates the start-of-sentence token
-* `</s>`: indicates the end-of-sentence token
-* `<unk>`: indicates the out-of-vocabulary token
-
-## Stage 3: Training
-This stage achieves the training of the specified model. To start training, you should manually set `exp_dir`, `CUDA_VISIBLE_DEVICES` and `gpu_num`, which have already been explained above. By default, the best `$keep_nbest_models` checkpoints on validation dataset will be averaged to generate a better model and adopted for decoding.
-
-* DDP Training
-
-We support the DistributedDataParallel (DDP) training and the detail can be found [here](https://pytorch.org/tutorials/intermediate/ddp_tutorial.html). To enable DDP training, please set `gpu_num` greater than 1. For example, if you set `CUDA_VISIBLE_DEVICES=0,1,5,6,7` and `gpu_num=3`, then the gpus with ids 0, 1 and 5 will be used for training.
-
-* DataLoader
-
-[comment]: <> (We support two types of DataLoaders for small and large datasets, respectively. By default, the small DataLoader is used and you can set `dataset_type=large` to enable large DataLoader. For small DataLoader, )
-We support an optional iterable-style DataLoader based on [Pytorch Iterable-style DataPipes](https://pytorch.org/data/beta/torchdata.datapipes.iter.html) for large dataset and you can set `dataset_type=large` to enable it.
-
-* Configuration
-
-The parameters of the training, including model, optimization, dataset, etc., are specified by a YAML file in `conf` directory. Also, you can directly specify the parameters in `run.sh` recipe. Please avoid to specify the same parameters in both the YAML file and the recipe.
-
-* Training Steps
-
-We support two parameters to specify the training steps, namely `max_epoch` and `max_update`. `max_epoch` indicates the total training epochs while `max_update` indicates the total training steps. If these two parameters are specified at the same time, once the training reaches any one of the two parameters, the training will be stopped.
-
-* Tensorboard
-
-You can use tensorboard to observe the loss, learning rate, etc. Please run the following command:
-```
-tensorboard --logdir ${exp_dir}/exp/${model_dir}/tensorboard/train
-```
-
-## Stage 4: Decoding
-This stage generates the recognition results with acoustic features as input and calculate the `CER` to verify the performance of the trained model.
-
-* Mode Selection
-
-As we support conformer, paraformer and uniasr in FunASR and they have different inference interfaces, a `mode` param is specified as `asr/paraformer/uniase` according to the trained model.
-
-* Configuration
-
-We support CTC decoding, attention decoding and hybrid CTC-attention decoding in FunASR, which can be specified by `ctc_weight` in a YAML file in `conf` directory. Specifically, `ctc_weight=1.0` indicates attention decoding, `ctc_weight=0.0` indicates CTC decoding, `0.0<ctc_weight<1.0` indicates hybrid CTC-attention decoding.
-
-* CPU/GPU Decoding
-
-We support CPU and GPU decoding in FunASR. For CPU decoding, you should set `gpu_inference=False` and set `njob` to specify the total number of CPU decoding jobs. For GPU decoding, you should set `gpu_inference=True`. You should also set `gpuid_list` to indicate which GPUs are used for decoding and `njobs` to indicate the number of decoding jobs on each GPU.
-
-* Performance
-
-We adopt `CER` to verify the performance. The results are in `$exp_dir/exp/$model_dir/$decoding_yaml_name/$average_model_name/$dset`, namely `text.cer` and `text.cer.txt`. `text.cer` saves the comparison between the recognized text and the reference text while `text.cer.txt` saves the final `CER` result. The following is an example of `text.cer`:
-* `text.cer`
-```
-...
-BAC009S0764W0213(nwords=11,cor=11,ins=0,del=0,sub=0) corr=100.00%,cer=0.00%
-ref: 鏋� 寤� 鑹� 濂� 鐨� 鏃� 娓� 甯� 鍦� 鐜� 澧�
-res: 鏋� 寤� 鑹� 濂� 鐨� 鏃� 娓� 甯� 鍦� 鐜� 澧�
-...
-```
-
diff --git a/docs/images/.DS_Store b/docs/images/.DS_Store
index 606ad25..5ef0f4c 100644
--- a/docs/images/.DS_Store
+++ b/docs/images/.DS_Store
Binary files differ
diff --git a/docs/images/dingding.jpg b/docs/images/dingding.jpg
index fb3ee99..4cdad28 100644
--- a/docs/images/dingding.jpg
+++ b/docs/images/dingding.jpg
Binary files differ
diff --git a/docs/images/wechat.png b/docs/images/wechat.png
index 962404c..c71b1fb 100644
--- a/docs/images/wechat.png
+++ b/docs/images/wechat.png
Binary files differ
diff --git a/docs/index.rst b/docs/index.rst
deleted file mode 100644
index ce44488..0000000
--- a/docs/index.rst
+++ /dev/null
@@ -1,31 +0,0 @@
-.. Funasr documentation master file, created by
- sphinx-quickstart on Tues Dec 6 19:05:00 2022.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-FunASR: A Fundamental End-to-End Speech Recognition Toolkit
-============================================================
-.. image:: ./images/funasr_logo.jpg
-
-FunASR hopes to build a bridge between academic research and industrial applications on speech recognition. By supporting the training & finetuning of the industrial-grade speech recognition model released on `ModelScope <https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition>`_, researchers and developers can conduct research and production of speech recognition models more conveniently, and promote the development of speech recognition ecology. ASR for Fun锛�
-
-.. toctree::
- :maxdepth: 1
- :caption: Tutorial:
-
- ./installation.md
- ./papers.md
- ./get_started.md
-
-.. toctree::
- :maxdepth: 1
- :caption: ModelScope:
-
- ./modelscope_models.md
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/docs/installation.md b/docs/installation.md
deleted file mode 100755
index 1eb813b..0000000
--- a/docs/installation.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# Installation
-FunASR is easy to install, which is mainly based on python packages.
-
-- Clone the repo
-``` sh
-git clone https://github.com/alibaba/FunASR.git
-```
-
-- Install Conda
-``` sh
-wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-sh Miniconda3-latest-Linux-x86_64.sh
-conda create -n funasr python=3.7
-conda activate funasr
-```
-
-- Install Pytorch (version >= 1.7.0):
-
-| cuda | |
-|:-----:| --- |
-| 9.2 | conda install pytorch==1.7.0 torchvision==0.8.0 torchaudio==0.7.0 cudatoolkit=9.2 -c pytorch |
-| 10.2 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=10.2 -c pytorch |
-| 11.1 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=11.1 -c pytorch |
-
-For more versions, please see [https://pytorch.org/get-started/locally](https://pytorch.org/get-started/locally)
-
-- Install ModelScope
-``` sh
-pip install "modelscope[audio]" -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
-```
-
-- Install other packages
-``` sh
-pip install --editable ./
-```
\ No newline at end of file
diff --git a/docs/make.bat b/docs/make.bat
deleted file mode 100644
index 747ffb7..0000000
--- a/docs/make.bat
+++ /dev/null
@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
- set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
- echo.
- echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
- echo.installed, then set the SPHINXBUILD environment variable to point
- echo.to the full path of the 'sphinx-build' executable. Alternatively you
- echo.may add the Sphinx directory to PATH.
- echo.
- echo.If you don't have Sphinx installed, grab it from
- echo.https://www.sphinx-doc.org/
- exit /b 1
-)
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
diff --git a/docs/papers.md b/docs/papers.md
deleted file mode 100644
index e9a83e4..0000000
--- a/docs/papers.md
+++ /dev/null
@@ -1,4 +0,0 @@
-# Papers
-
-- [Universal ASR: Unifying Streaming and Non-Streaming ASR Using a Single Encoder-Decoder Model](https://arxiv.org/abs/2010.14099), arXiv preprint arXiv:2010.14099, 2020.
-- [Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition](https://arxiv.org/abs/2206.08317), INTERSPEECH 2022.
\ No newline at end of file
diff --git a/egs/aishell/conformer/run.sh b/egs/aishell/conformer/run.sh
index d865982..41db45d 100755
--- a/egs/aishell/conformer/run.sh
+++ b/egs/aishell/conformer/run.sh
@@ -8,12 +8,12 @@
count=1
gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
-njob=8
+njob=5
train_cmd=utils/run.pl
infer_cmd=utils/run.pl
# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
+feats_dir="../DATA" #feature output dictionary
exp_dir="."
lang=zh
dumpdir=dump/fbank
@@ -34,7 +34,7 @@
data_aishell=
# exp tag
-tag=""
+tag="exp1"
. utils/parse_options.sh || exit 1;
@@ -217,7 +217,7 @@
if [ -n "${inference_config}" ]; then
_opts+="--config ${inference_config} "
fi
- ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1: "${_nj}" "${_logdir}"/asr_inference.JOB.log \
python -m funasr.bin.asr_inference_launch \
--batch_size 1 \
--ngpu "${_ngpu}" \
diff --git a/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml
index b5ab916..24b2620 100644
--- a/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml
+++ b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml
@@ -78,13 +78,13 @@
- 40
num_time_mask: 2
-predictor: cif_predictor
+predictor: cif_predictor_v2
predictor_conf:
- idim: 256
- threshold: 1.0
- l_order: 1
- r_order: 1
- tail_threshold: 0.45
+ idim: 256
+ threshold: 1.0
+ l_order: 1
+ r_order: 1
+ tail_threshold: 0.45
log_interval: 50
diff --git a/egs/aishell/paraformer/run.sh b/egs/aishell/paraformer/run.sh
index c5ee1f2..2b0f144 100755
--- a/egs/aishell/paraformer/run.sh
+++ b/egs/aishell/paraformer/run.sh
@@ -13,7 +13,7 @@
infer_cmd=utils/run.pl
# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
+feats_dir="../DATA" #feature output dictionary
exp_dir="."
lang=zh
dumpdir=dump/fbank
@@ -219,7 +219,7 @@
fi
${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
python -m funasr.bin.asr_inference_launch \
- --batch_size 100 \
+ --batch_size 1 \
--ngpu "${_ngpu}" \
--njob ${njob} \
--gpuid_list ${gpuid_list} \
@@ -244,5 +244,4 @@
tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
cat ${_dir}/text.cer.txt
done
-fi
-
+fi
\ No newline at end of file
diff --git a/egs/aishell/paraformerbert/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformerbert/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml
index 7562a49..f51a2ea 100644
--- a/egs/aishell/paraformerbert/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml
+++ b/egs/aishell/paraformerbert/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml
@@ -88,11 +88,10 @@
predictor: cif_predictor
predictor_conf:
- idim: 256
- threshold: 1.0
- l_order: 1
- r_order: 1
- tail_threshold: 0.45
+ idim: 256
+ threshold: 1.0
+ l_order: 1
+ r_order: 1
log_interval: 50
diff --git a/egs/aishell/paraformerbert/local/extract_embeds.sh b/egs/aishell/paraformerbert/local/extract_embeds.sh
index 6d99390..9cf5940 100755
--- a/egs/aishell/paraformerbert/local/extract_embeds.sh
+++ b/egs/aishell/paraformerbert/local/extract_embeds.sh
@@ -7,7 +7,7 @@
bert_model_name="bert-base-chinese"
#bert_model_name="chinese-roberta-wwm-ext"
#bert_model_name="mengzi-bert-base"
-raw_dataset_path=~/Funasr_data/aishell-1
+raw_dataset_path="../DATA"
model_path=${bert_model_root}/${bert_model_name}
. utils/parse_options.sh || exit 1;
@@ -39,11 +39,10 @@
JOB=`expr $tmp + $idx`
echo "proces jobid=$JOB"
{
-
- beg=0
- gpu=`expr $beg + $idx`
- echo ${local_scp_dir}/log.${JOB}
- python utils/extract_embeds.py $local_scp_dir/data.$JOB.text ${local_records_dir}/embeds.${JOB}.ark ${local_records_dir}/embeds.${JOB}.scp ${local_records_dir}/embeds.${JOB}.shape ${gpu} ${model_path} &> ${local_scp_dir}/log.${JOB}
+ beg=0
+ gpu=`expr $beg + $idx`
+ echo ${local_scp_dir}/log.${JOB}
+ python utils/extract_embeds.py $local_scp_dir/data.$JOB.text ${local_records_dir}/embeds.${JOB}.ark ${local_records_dir}/embeds.${JOB}.scp ${local_records_dir}/embeds.${JOB}.shape ${gpu} ${model_path} &> ${local_scp_dir}/log.${JOB}
} &
done
wait
@@ -54,8 +53,6 @@
for JOB in $(seq ${nj}); do
cat ${local_records_dir}/embeds.${JOB}.scp || exit 1;
done > ${local_scp_dir_raw}/embeds.scp
-
- sed 's#nfs#data\/volume1#g' ${local_scp_dir_raw}/embeds.scp > ${local_scp_dir_raw}/embeds.scp.pai
for JOB in $(seq ${nj}); do
cat ${local_records_dir}/embeds.${JOB}.shape || exit 1;
diff --git a/egs/aishell/paraformerbert/run.sh b/egs/aishell/paraformerbert/run.sh
index a39ac77..96310ab 100755
--- a/egs/aishell/paraformerbert/run.sh
+++ b/egs/aishell/paraformerbert/run.sh
@@ -8,7 +8,7 @@
count=1
gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
-njob=1
+njob=5
train_cmd=utils/run.pl
infer_cmd=utils/run.pl
@@ -148,17 +148,17 @@
cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev
fi
-if ! "${skip_extract_embed}"; then
- local/extract_embeds.sh \
- --bert_model_root ${bert_model_root} \
- --bert_model_name ${bert_model_name} \
- --raw_dataset_path ${feats_dir}
-fi
-
# Training Stage
world_size=$gpu_num # run on one machine
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
echo "stage 3: Training"
+ if ! "${skip_extract_embed}"; then
+ echo "extract embeddings..."
+ local/extract_embeds.sh \
+ --bert_model_root ${bert_model_root} \
+ --bert_model_name ${bert_model_name} \
+ --raw_dataset_path ${feats_dir}
+ fi
mkdir -p ${exp_dir}/exp/${model_dir}
mkdir -p ${exp_dir}/exp/${model_dir}/log
INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
@@ -192,6 +192,7 @@
--resume true \
--output_dir ${exp_dir}/exp/${model_dir} \
--config $asr_config \
+ --allow_variable_data_keys true \
--input_size $feats_dim \
--ngpu $gpu_num \
--num_worker_count $count \
@@ -199,8 +200,7 @@
--dist_init_method $init_method \
--dist_world_size $world_size \
--dist_rank $rank \
- --allow_variable_data_keys true \
- --local_rank $local_rank 1> $exp_dir/log/train.log.$i 2>&1
+ --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
} &
done
wait
@@ -235,7 +235,7 @@
fi
${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
python -m funasr.bin.asr_inference_launch \
- --batch_size 100 \
+ --batch_size 1 \
--ngpu "${_ngpu}" \
--njob ${njob} \
--gpuid_list ${gpuid_list} \
diff --git a/egs/aishell/tranformer/utils/compute_fbank.py b/egs/aishell/tranformer/utils/compute_fbank.py
index d03b5a8..9c3904f 100755
--- a/egs/aishell/tranformer/utils/compute_fbank.py
+++ b/egs/aishell/tranformer/utils/compute_fbank.py
@@ -14,7 +14,8 @@
frame_shift=10,
dither=0.0,
resample_rate=16000,
- speed=1.0):
+ speed=1.0,
+ window_type="hamming"):
waveform, sample_rate = torchaudio.load(wav_file)
if resample_rate != sample_rate:
@@ -33,7 +34,7 @@
frame_shift=frame_shift,
dither=dither,
energy_floor=0.0,
- window_type='hamming',
+ window_type=window_type,
sample_frequency=resample_rate)
return mat.numpy()
@@ -68,6 +69,13 @@
help="feature dims",
)
parser.add_argument(
+ "--max-lengths",
+ "-m",
+ default=1500,
+ type=int,
+ help="max frame numbers",
+ )
+ parser.add_argument(
"--sample-frequency",
"-s",
default=16000,
@@ -96,6 +104,13 @@
required=True,
type=str,
help="output dir",
+ )
+ parser.add_argument(
+ "--window-type",
+ default="hamming",
+ required=False,
+ type=str,
+ help="window type"
)
return parser
@@ -131,10 +146,13 @@
fbank = compute_fbank(wav_file,
num_mel_bins=args.dims,
resample_rate=args.sample_frequency,
- speed=float(speed)
+ speed=float(speed),
+ window_type=args.window_type
)
feats_dims = fbank.shape[1]
feats_lens = fbank.shape[0]
+ if feats_lens >= args.max_lengths:
+ continue
txt_lens = len(txt)
if speed == "1.0":
wav_id_sp = wav_id
diff --git a/egs/aishell/tranformer/utils/compute_fbank.sh b/egs/aishell/tranformer/utils/compute_fbank.sh
index 92a4fe6..8704b31 100755
--- a/egs/aishell/tranformer/utils/compute_fbank.sh
+++ b/egs/aishell/tranformer/utils/compute_fbank.sh
@@ -9,6 +9,8 @@
feats_dim=80
sample_frequency=16000
speed_perturb="1.0"
+window_type="hamming"
+max_lengths=1500
echo "$0 $@"
@@ -29,7 +31,8 @@
$cmd JOB=1:$nj $logdir/make_fbank.JOB.log \
python utils/compute_fbank.py -w $data/split${nj}/JOB/wav.scp -t $data/split${nj}/JOB/text \
- -d $feats_dim -s $sample_frequency -p ${speed_perturb} -a JOB -o ${fbankdir} \
+ -d $feats_dim -s $sample_frequency -m ${max_lengths} -p ${speed_perturb} -a JOB -o ${fbankdir} \
+ --window-type ${window_type} \
|| exit 1;
for n in $(seq $nj); do
diff --git a/egs/aishell/tranformer/utils/download_model.py b/egs/aishell/tranformer/utils/download_model.py
new file mode 100755
index 0000000..70ea179
--- /dev/null
+++ b/egs/aishell/tranformer/utils/download_model.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import argparse
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ description="download model configs",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+ parser.add_argument("--model_name",
+ type=str,
+ default="damo/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch",
+ help="model name in ModelScope")
+ args = parser.parse_args()
+
+ inference_pipeline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model=args.model_name)
diff --git a/egs/aishell/tranformer/utils/fix_data.sh b/egs/aishell/tranformer/utils/fix_data.sh
index 32cdde5..b1a2bb8 100755
--- a/egs/aishell/tranformer/utils/fix_data.sh
+++ b/egs/aishell/tranformer/utils/fix_data.sh
@@ -28,8 +28,8 @@
mv ${data_dir}/wav.scp ${data_dir}/wav.scp.bak
mv ${data_dir}/text ${data_dir}/text.bak
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/wav.scp.bak > ${data_dir}/wav.scp
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak > ${data_dir}/text
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/wav.scp.bak | sort -k1,1 -u > ${data_dir}/wav.scp
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak | sort -k1,1 -u > ${data_dir}/text
rm ${data_dir}/wav.scp.bak
rm ${data_dir}/text.bak
diff --git a/egs/aishell/tranformer/utils/fix_data_feat.sh b/egs/aishell/tranformer/utils/fix_data_feat.sh
index 2c92d7f..84eea36 100755
--- a/egs/aishell/tranformer/utils/fix_data_feat.sh
+++ b/egs/aishell/tranformer/utils/fix_data_feat.sh
@@ -40,10 +40,10 @@
mv ${data_dir}/speech_shape ${data_dir}/speech_shape.bak
mv ${data_dir}/text_shape ${data_dir}/text_shape.bak
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/feats.scp.bak > ${data_dir}/feats.scp
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak > ${data_dir}/text
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/speech_shape.bak > ${data_dir}/speech_shape
-utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text_shape.bak > ${data_dir}/text_shape
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/feats.scp.bak | sort -k1,1 -u > ${data_dir}/feats.scp
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak | sort -k1,1 -u > ${data_dir}/text
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/speech_shape.bak | sort -k1,1 -u > ${data_dir}/speech_shape
+utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text_shape.bak | sort -k1,1 -u > ${data_dir}/text_shape
rm ${data_dir}/feats.scp.bak
rm ${data_dir}/text.bak
diff --git a/egs/aishell2/conformer/conf/decode_asr_transformer.yaml b/egs/aishell2/conformer/conf/decode_asr_transformer.yaml
new file mode 100644
index 0000000..a147fa7
--- /dev/null
+++ b/egs/aishell2/conformer/conf/decode_asr_transformer.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.5
+lm_weight: 0.7
diff --git a/egs/aishell2/conformer/conf/train_asr_conformer.yaml b/egs/aishell2/conformer/conf/train_asr_conformer.yaml
new file mode 100644
index 0000000..02fc5a8
--- /dev/null
+++ b/egs/aishell2/conformer/conf/train_asr_conformer.yaml
@@ -0,0 +1,86 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+ output_size: 256 # dimension of attention
+ attention_heads: 4
+ linear_units: 2048 # the number of units of position-wise feed forward
+ num_blocks: 12 # the number of encoder blocks
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ attention_dropout_rate: 0.0
+ input_layer: conv2d # encoder architecture type
+ normalize_before: true
+ pos_enc_layer_type: rel_pos
+ selfattention_layer_type: rel_selfattn
+ activation_type: swish
+ macaron_style: true
+ use_cnn_module: true
+ cnn_module_kernel: 15
+
+# decoder related
+decoder: transformer
+decoder_conf:
+ attention_heads: 4
+ linear_units: 2048
+ num_blocks: 6
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ self_attention_dropout_rate: 0.0
+ src_attention_dropout_rate: 0.0
+
+# hybrid CTC/attention
+model_conf:
+ ctc_weight: 0.3
+ lsm_weight: 0.1 # label smoothing option
+ length_normalized_loss: false
+
+# optimization related
+accum_grad: 2
+grad_clip: 5
+patience: none
+max_epoch: 50
+val_scheduler_criterion:
+ - valid
+ - acc
+best_model_criterion:
+- - valid
+ - acc
+ - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+ lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+ warmup_steps: 30000
+
+specaug: specaug
+specaug_conf:
+ apply_time_warp: true
+ time_warp_window: 5
+ time_warp_mode: bicubic
+ apply_freq_mask: true
+ freq_mask_width_range:
+ - 0
+ - 30
+ num_freq_mask: 2
+ apply_time_mask: true
+ time_mask_width_range:
+ - 0
+ - 40
+ num_time_mask: 2
+
+log_interval: 50
+normalize: None
+
+dataset_conf:
+ shuffle: True
+ shuffle_conf:
+ shuffle_size: 2048
+ sort_size: 500
+ batch_conf:
+ batch_type: token
+ batch_size: 25000
+ num_workers: 8
\ No newline at end of file
diff --git a/egs/aishell2/conformer/local/prepare_data.sh b/egs/aishell2/conformer/local/prepare_data.sh
new file mode 100755
index 0000000..ce6ee19
--- /dev/null
+++ b/egs/aishell2/conformer/local/prepare_data.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG)
+# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU)
+# Apache 2.0
+
+# transform raw AISHELL-2 data to kaldi format
+
+. ./path.sh || exit 1;
+
+tmp=
+dir=
+
+if [ $# != 3 ]; then
+ echo "Usage: $0 <corpus-data-dir> <tmp-dir> <output-dir>"
+ echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train"
+ exit 1;
+fi
+
+corpus=$1
+tmp=$2
+dir=$3
+
+echo "prepare_data.sh: Preparing data in $corpus"
+
+mkdir -p $tmp
+mkdir -p $dir
+
+# corpus check
+if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then
+ echo "Error: $0 requires wav.scp and trans.txt under $corpus directory."
+ exit 1;
+fi
+
+# validate utt-key list, IC0803W0380 is a bad utterance
+awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list
+awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list
+tools/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list
+
+# wav.scp
+awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp
+tools/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp
+
+# text
+tools/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text
+
+# copy prepared resources from tmp_dir to target dir
+mkdir -p $dir
+for f in wav.scp text; do
+ cp $tmp/$f $dir/$f || exit 1;
+done
+
+echo "local/prepare_data.sh succeeded"
+exit 0;
diff --git a/egs/aishell2/conformer/path.sh b/egs/aishell2/conformer/path.sh
new file mode 100755
index 0000000..ea3c0be
--- /dev/null
+++ b/egs/aishell2/conformer/path.sh
@@ -0,0 +1,6 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
+export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs/aishell2/conformer/run.sh b/egs/aishell2/conformer/run.sh
new file mode 100755
index 0000000..bd6d81e
--- /dev/null
+++ b/egs/aishell2/conformer/run.sh
@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+
+# machines configuration
+CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+gpu_num=8
+count=1
+gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
+# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
+njob=5
+train_cmd=tools/run.pl
+infer_cmd=utils/run.pl
+
+# general configuration
+feats_dir="../DATA" #feature output dictionary
+exp_dir="."
+lang=zh
+dumpdir=dump/fbank
+feats_type=fbank
+token_type=char
+dataset_type=large
+scp=feats.scp
+type=kaldi_ark
+stage=0
+stop_stage=4
+
+# feature configuration
+feats_dim=80
+sample_frequency=16000
+nj=100
+speed_perturb="0.9,1.0,1.1"
+
+# data
+tr_dir=
+dev_tst_dir=
+
+# exp tag
+tag="exp1"
+
+. utils/parse_options.sh || exit 1;
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train
+valid_set=dev_ios
+test_sets="dev_ios test_ios"
+
+asr_config=conf/train_asr_conformer.yaml
+model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
+
+inference_config=conf/decode_asr_transformer.yaml
+inference_asr_model=valid.acc.ave_10best.pth
+
+# you can set gpu num for decoding here
+gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, e.g., gpuid_list=2,3, the same as training stage by default
+ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
+
+if ${gpu_inference}; then
+ inference_nj=$[${ngpu}*${njob}]
+ _ngpu=1
+else
+ inference_nj=$njob
+ _ngpu=0
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+ echo "stage 0: Data preparation"
+ # For training set
+ local/prepare_data.sh ${tr_dir} ${feats_dir}/data/local/train ${feats_dir}/data/train || exit 1;
+ # # For dev and test set
+ for x in Android iOS Mic; do
+ local/prepare_data.sh ${dev_tst_dir}/${x}/dev ${feats_dir}/data/local/dev_${x,,} ${feats_dir}/data/dev_${x,,} || exit 1;
+ local/prepare_data.sh ${dev_tst_dir}/${x}/test ${feats_dir}/data/local/test_${x,,} ${feats_dir}/data/test_${x,,} || exit 1;
+ done
+ # Normalize text to capital letters
+ for x in train dev_ios test_ios; do
+ mv ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org
+ paste -d " " <(cut -f 1 ${feats_dir}/data/${x}/text.org) <(cut -f 2- ${feats_dir}/data/${x}/text.org \
+ | tr 'A-Z' 'a-z' | tr -d " ") \
+ > ${feats_dir}/data/${x}/text
+ tools/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org
+ mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text
+ done
+fi
+
+feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir}
+feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir}
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ echo "stage 1: Feature Generation"
+ # compute fbank features
+ fbankdir=${feats_dir}/fbank
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
+ ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
+ tools/fix_data_feat.sh ${fbankdir}/train
+ for x in android ios mic; do
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/dev_${x} ${exp_dir}/exp/make_fbank/dev_${x} ${fbankdir}/dev_${x}
+ tools/fix_data_feat.sh ${fbankdir}/dev_${x}
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/test_${x} ${exp_dir}/exp/make_fbank/test_${x} ${fbankdir}/test_${x}
+ tools/fix_data_feat.sh ${fbankdir}/test_${x}
+ done
+
+ # compute global cmvn
+ steps/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/train ${exp_dir}/exp/make_fbank/train
+
+ # apply cmvn
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${train_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${train_set} ${feat_train_dir}
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${valid_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${valid_set} ${feat_dev_dir}
+ for x in android ios mic; do
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/test_${x} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test_${x} ${feats_dir}/${dumpdir}/test_${x}
+ done
+
+ cp ${fbankdir}/${train_set}/text ${fbankdir}/${train_set}/speech_shape ${fbankdir}/${train_set}/text_shape ${feat_train_dir}
+ tools/fix_data_feat.sh ${feat_train_dir}
+ cp ${fbankdir}/${valid_set}/text ${fbankdir}/${valid_set}/speech_shape ${fbankdir}/${valid_set}/text_shape ${feat_dev_dir}
+ tools/fix_data_feat.sh ${feat_dev_dir}
+ for x in android ios mic; do
+ cp ${fbankdir}/test_${x}/text ${fbankdir}/test_${x}/speech_shape ${fbankdir}/test_${x}/text_shape ${feats_dir}/${dumpdir}/test_${x}
+ tools/fix_data_feat.sh ${feats_dir}/${dumpdir}/test_${x}
+ done
+fi
+
+token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
+echo "dictionary: ${token_list}"
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+ echo "stage 2: Dictionary Preparation"
+ mkdir -p ${feats_dir}/data/${lang}_token_list/char/
+
+ echo "make a dictionary"
+ echo "<blank>" > ${token_list}
+ echo "<s>" >> ${token_list}
+ echo "</s>" >> ${token_list}
+ tools/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/${train_set}/text | cut -f 2- -d" " | tr " " "\n" \
+ | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list}
+ num_token=$(cat ${token_list} | wc -l)
+ echo "<unk>" >> ${token_list}
+ vocab_size=$(cat ${token_list} | wc -l)
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+ cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+fi
+
+# Training Stage
+world_size=$gpu_num # run on one machine
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+ echo "stage 3: Training"
+ mkdir -p ${exp_dir}/exp/${model_dir}
+ mkdir -p ${exp_dir}/exp/${model_dir}/log
+ INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
+ if [ -f $INIT_FILE ];then
+ rm -f $INIT_FILE
+ fi
+ init_method=file://$(readlink -f $INIT_FILE)
+ echo "$0: init method is $init_method"
+ for ((i = 0; i < $gpu_num; ++i)); do
+ {
+ rank=$i
+ local_rank=$i
+ gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
+ asr_train.py \
+ --gpu_id $gpu_id \
+ --use_preprocessor true \
+ --dataset_type $dataset_type \
+ --token_type char \
+ --token_list $token_list \
+ --train_data_file $feats_dir/$dumpdir/${train_set}/data.list \
+ --valid_data_file $feats_dir/$dumpdir/${valid_set}/data.list \
+ --resume true \
+ --output_dir ${exp_dir}/exp/${model_dir} \
+ --config $asr_config \
+ --input_size $feats_dim \
+ --ngpu $gpu_num \
+ --num_worker_count $count \
+ --multiprocessing_distributed true \
+ --dist_init_method $init_method \
+ --dist_world_size $world_size \
+ --dist_rank $rank \
+ --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
+ } &
+ done
+ wait
+fi
+
+# Testing Stage
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+ echo "stage 4: Inference"
+ for dset in ${test_sets}; do
+ asr_exp=${exp_dir}/exp/${model_dir}
+ inference_tag="$(basename "${inference_config}" .yaml)"
+ _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
+ _logdir="${_dir}/logdir"
+ if [ -d ${_dir} ]; then
+ echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
+ exit 0
+ fi
+ mkdir -p "${_logdir}"
+ _data="${feats_dir}/${dumpdir}/${dset}"
+ key_file=${_data}/${scp}
+ num_scp_file="$(<${key_file} wc -l)"
+ _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
+ split_scps=
+ for n in $(seq "${_nj}"); do
+ split_scps+=" ${_logdir}/keys.${n}.scp"
+ done
+ # shellcheck disable=SC2086
+ utils/split_scp.pl "${key_file}" ${split_scps}
+ _opts=
+ if [ -n "${inference_config}" ]; then
+ _opts+="--config ${inference_config} "
+ fi
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
+ python -m funasr.bin.asr_inference_launch \
+ --batch_size 1 \
+ --ngpu "${_ngpu}" \
+ --njob ${njob} \
+ --gpuid_list ${gpuid_list} \
+ --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
+ --key_file "${_logdir}"/keys.JOB.scp \
+ --asr_train_config "${asr_exp}"/config.yaml \
+ --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
+ --output_dir "${_logdir}"/output.JOB \
+ --mode asr \
+ ${_opts}
+
+ for f in token token_int score text; do
+ if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
+ for i in $(seq "${_nj}"); do
+ cat "${_logdir}/output.${i}/1best_recog/${f}"
+ done | sort -k1 >"${_dir}/${f}"
+ fi
+ done
+ python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
+ python utils/proce_text.py ${_data}/text ${_data}/text.proc
+ python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
+ tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
+ cat ${_dir}/text.cer.txt
+ done
+fi
+
+
+
diff --git a/egs/aishell2/conformer/utils b/egs/aishell2/conformer/utils
new file mode 120000
index 0000000..f245098
--- /dev/null
+++ b/egs/aishell2/conformer/utils
@@ -0,0 +1 @@
+../../aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs/aishell2/paraformer/conf/decode_asr_transformer.yaml b/egs/aishell2/paraformer/conf/decode_asr_transformer.yaml
new file mode 100644
index 0000000..a147fa7
--- /dev/null
+++ b/egs/aishell2/paraformer/conf/decode_asr_transformer.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.5
+lm_weight: 0.7
diff --git a/egs/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
new file mode 100644
index 0000000..a147fa7
--- /dev/null
+++ b/egs/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.5
+lm_weight: 0.7
diff --git a/egs/aishell2/paraformer/conf/train_asr_paraformer_conformer_20e_1280_320_6d_1280_320.yaml b/egs/aishell2/paraformer/conf/train_asr_paraformer_conformer_20e_1280_320_6d_1280_320.yaml
new file mode 100644
index 0000000..450f71a
--- /dev/null
+++ b/egs/aishell2/paraformer/conf/train_asr_paraformer_conformer_20e_1280_320_6d_1280_320.yaml
@@ -0,0 +1,96 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+ output_size: 320 # dimension of attention
+ attention_heads: 4
+ linear_units: 1280 # the number of units of position-wise feed forward
+ num_blocks: 20 # the number of encoder blocks
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ attention_dropout_rate: 0.0
+ input_layer: conv2d # encoder architecture type
+ normalize_before: true
+ pos_enc_layer_type: rel_pos
+ selfattention_layer_type: rel_selfattn
+ activation_type: swish
+ macaron_style: true
+ use_cnn_module: true
+ cnn_module_kernel: 15
+
+# decoder related
+decoder: paraformer_decoder_san
+decoder_conf:
+ attention_heads: 4
+ linear_units: 1280
+ num_blocks: 6
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ self_attention_dropout_rate: 0.0
+ src_attention_dropout_rate: 0.0
+
+# hybrid CTC/attention
+model: paraformer
+model_conf:
+ ctc_weight: 0.3
+ lsm_weight: 0.1
+ length_normalized_loss: false
+ predictor_weight: 1.0
+ sampling_ratio: 0.4
+
+# optimization related
+accum_grad: 2
+grad_clip: 5
+patience: none
+max_epoch: 50
+val_scheduler_criterion:
+ - valid
+ - acc
+best_model_criterion:
+- - valid
+ - acc
+ - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+ lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+ warmup_steps: 30000
+
+specaug: specaug
+specaug_conf:
+ apply_time_warp: true
+ time_warp_window: 5
+ time_warp_mode: bicubic
+ apply_freq_mask: true
+ freq_mask_width_range:
+ - 0
+ - 30
+ num_freq_mask: 2
+ apply_time_mask: true
+ time_mask_width_range:
+ - 0
+ - 40
+ num_time_mask: 2
+
+predictor: cif_predictor
+predictor_conf:
+ idim: 320
+ threshold: 1.0
+ l_order: 1
+ r_order: 1
+
+log_interval: 50
+normalize: None
+
+dataset_conf:
+ shuffle: True
+ shuffle_conf:
+ shuffle_size: 2048
+ sort_size: 500
+ batch_conf:
+ batch_type: token
+ batch_size: 25000
+ num_workers: 8
\ No newline at end of file
diff --git a/egs/aishell2/paraformer/local/prepare_data.sh b/egs/aishell2/paraformer/local/prepare_data.sh
new file mode 100755
index 0000000..ce6ee19
--- /dev/null
+++ b/egs/aishell2/paraformer/local/prepare_data.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG)
+# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU)
+# Apache 2.0
+
+# transform raw AISHELL-2 data to kaldi format
+
+. ./path.sh || exit 1;
+
+tmp=
+dir=
+
+if [ $# != 3 ]; then
+ echo "Usage: $0 <corpus-data-dir> <tmp-dir> <output-dir>"
+ echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train"
+ exit 1;
+fi
+
+corpus=$1
+tmp=$2
+dir=$3
+
+echo "prepare_data.sh: Preparing data in $corpus"
+
+mkdir -p $tmp
+mkdir -p $dir
+
+# corpus check
+if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then
+ echo "Error: $0 requires wav.scp and trans.txt under $corpus directory."
+ exit 1;
+fi
+
+# validate utt-key list, IC0803W0380 is a bad utterance
+awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list
+awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list
+tools/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list
+
+# wav.scp
+awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp
+tools/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp
+
+# text
+tools/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text
+
+# copy prepared resources from tmp_dir to target dir
+mkdir -p $dir
+for f in wav.scp text; do
+ cp $tmp/$f $dir/$f || exit 1;
+done
+
+echo "local/prepare_data.sh succeeded"
+exit 0;
diff --git a/egs/aishell2/paraformer/path.sh b/egs/aishell2/paraformer/path.sh
new file mode 100755
index 0000000..ea3c0be
--- /dev/null
+++ b/egs/aishell2/paraformer/path.sh
@@ -0,0 +1,6 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
+export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs/aishell2/paraformer/run.sh b/egs/aishell2/paraformer/run.sh
new file mode 100755
index 0000000..2b7d841
--- /dev/null
+++ b/egs/aishell2/paraformer/run.sh
@@ -0,0 +1,251 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+
+# machines configuration
+CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+gpu_num=8
+count=1
+gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
+# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
+njob=1
+train_cmd=tools/run.pl
+infer_cmd=utils/run.pl
+
+# general configuration
+feats_dir="../DATA" #feature output dictionary
+exp_dir="."
+lang=zh
+dumpdir=dump/fbank
+feats_type=fbank
+token_type=char
+dataset_type=large
+scp=feats.scp
+type=kaldi_ark
+stage=0
+stop_stage=4
+
+# feature configuration
+feats_dim=80
+sample_frequency=16000
+nj=100
+speed_perturb="0.9,1.0,1.1"
+
+# data
+tr_dir=
+dev_tst_dir=
+
+# exp tag
+tag="exp1"
+
+. utils/parse_options.sh || exit 1;
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train
+valid_set=dev_ios
+test_sets="dev_ios test_ios"
+
+asr_config=conf/train_asr_paraformer_conformer_20e_1280_320_6d_1280_320.yaml
+model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
+
+inference_config=conf/decode_asr_transformer_noctc_1best.yaml
+inference_asr_model=valid.acc.ave_10best.pth
+
+# you can set gpu num for decoding here
+gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, e.g., gpuid_list=2,3, the same as training stage by default
+ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
+
+if ${gpu_inference}; then
+ inference_nj=$[${ngpu}*${njob}]
+ _ngpu=1
+else
+ inference_nj=$njob
+ _ngpu=0
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+ echo "stage 0: Data preparation"
+ # For training set
+ local/prepare_data.sh ${tr_dir} ${feats_dir}/data/local/train ${feats_dir}/data/train || exit 1;
+ # # For dev and test set
+ for x in Android iOS Mic; do
+ local/prepare_data.sh ${dev_tst_dir}/${x}/dev ${feats_dir}/data/local/dev_${x,,} ${feats_dir}/data/dev_${x,,} || exit 1;
+ local/prepare_data.sh ${dev_tst_dir}/${x}/test ${feats_dir}/data/local/test_${x,,} ${feats_dir}/data/test_${x,,} || exit 1;
+ done
+ # Normalize text to capital letters
+ for x in train dev_android dev_ios dev_mic test_android test_ios test_mic; do
+ mv ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org
+ paste -d " " <(cut -f 1 ${feats_dir}/data/${x}/text.org) <(cut -f 2- ${feats_dir}/data/${x}/text.org \
+ | tr 'A-Z' 'a-z' | tr -d " ") \
+ > ${feats_dir}/data/${x}/text
+ tools/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org
+ mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text
+ done
+fi
+
+feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir}
+feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir}
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ echo "stage 1: Feature Generation"
+ # compute fbank features
+ fbankdir=${feats_dir}/fbank
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
+ ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
+ tools/fix_data_feat.sh ${fbankdir}/train
+ for x in android ios mic; do
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/dev_${x} ${exp_dir}/exp/make_fbank/dev_${x} ${fbankdir}/dev_${x}
+ tools/fix_data_feat.sh ${fbankdir}/dev_${x}
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/test_${x} ${exp_dir}/exp/make_fbank/test_${x} ${fbankdir}/test_${x}
+ tools/fix_data_feat.sh ${fbankdir}/test_${x}
+ done
+
+ # compute global cmvn
+ steps/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/train ${exp_dir}/exp/make_fbank/train
+
+ # apply cmvn
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${train_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${train_set} ${feat_train_dir}
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${valid_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${valid_set} ${feat_dev_dir}
+ for x in android ios mic; do
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/test_${x} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test_${x} ${feats_dir}/${dumpdir}/test_${x}
+ done
+
+ cp ${fbankdir}/${train_set}/text ${fbankdir}/${train_set}/speech_shape ${fbankdir}/${train_set}/text_shape ${feat_train_dir}
+ tools/fix_data_feat.sh ${feat_train_dir}
+ cp ${fbankdir}/${valid_set}/text ${fbankdir}/${valid_set}/speech_shape ${fbankdir}/${valid_set}/text_shape ${feat_dev_dir}
+ tools/fix_data_feat.sh ${feat_dev_dir}
+ for x in android ios mic; do
+ cp ${fbankdir}/test_${x}/text ${fbankdir}/test_${x}/speech_shape ${fbankdir}/test_${x}/text_shape ${feats_dir}/${dumpdir}/test_${x}
+ tools/fix_data_feat.sh ${feats_dir}/${dumpdir}/test_${x}
+ done
+fi
+
+token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
+echo "dictionary: ${token_list}"
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+ echo "stage 2: Dictionary Preparation"
+ mkdir -p ${feats_dir}/data/${lang}_token_list/char/
+
+ echo "make a dictionary"
+ echo "<blank>" > ${token_list}
+ echo "<s>" >> ${token_list}
+ echo "</s>" >> ${token_list}
+ tools/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/${train_set}/text | cut -f 2- -d" " | tr " " "\n" \
+ | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list}
+ num_token=$(cat ${token_list} | wc -l)
+ echo "<unk>" >> ${token_list}
+ vocab_size=$(cat ${token_list} | wc -l)
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+ cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+fi
+
+# Training Stage
+world_size=$gpu_num # run on one machine
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+ echo "stage 3: Training"
+ mkdir -p ${exp_dir}/exp/${model_dir}
+ mkdir -p ${exp_dir}/exp/${model_dir}/log
+ INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
+ if [ -f $INIT_FILE ];then
+ rm -f $INIT_FILE
+ fi
+ init_method=file://$(readlink -f $INIT_FILE)
+ echo "$0: init method is $init_method"
+ for ((i = 0; i < $gpu_num; ++i)); do
+ {
+ rank=$i
+ local_rank=$i
+ gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
+ asr_train_paraformer.py \
+ --gpu_id $gpu_id \
+ --use_preprocessor true \
+ --dataset_type $dataset_type \
+ --token_type char \
+ --token_list $token_list \
+ --train_data_file $feats_dir/$dumpdir/${train_set}/data.list \
+ --valid_data_file $feats_dir/$dumpdir/${valid_set}/data.list \
+ --resume true \
+ --output_dir ${exp_dir}/exp/${model_dir} \
+ --config $asr_config \
+ --input_size $feats_dim \
+ --ngpu $gpu_num \
+ --num_worker_count $count \
+ --multiprocessing_distributed true \
+ --dist_init_method $init_method \
+ --dist_world_size $world_size \
+ --dist_rank $rank \
+ --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
+ } &
+ done
+ wait
+fi
+
+# Testing Stage
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+ echo "stage 4: Inference"
+ for dset in ${test_sets}; do
+ asr_exp=${exp_dir}/exp/${model_dir}
+ inference_tag="$(basename "${inference_config}" .yaml)"
+ _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
+ _logdir="${_dir}/logdir"
+ if [ -d ${_dir} ]; then
+ echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
+ exit 0
+ fi
+ mkdir -p "${_logdir}"
+ _data="${feats_dir}/${dumpdir}/${dset}"
+ key_file=${_data}/${scp}
+ num_scp_file="$(<${key_file} wc -l)"
+ _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
+ split_scps=
+ for n in $(seq "${_nj}"); do
+ split_scps+=" ${_logdir}/keys.${n}.scp"
+ done
+ # shellcheck disable=SC2086
+ utils/split_scp.pl "${key_file}" ${split_scps}
+ _opts=
+ if [ -n "${inference_config}" ]; then
+ _opts+="--config ${inference_config} "
+ fi
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
+ python -m funasr.bin.asr_inference_launch \
+ --batch_size 1 \
+ --ngpu "${_ngpu}" \
+ --njob ${njob} \
+ --gpuid_list ${gpuid_list} \
+ --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
+ --key_file "${_logdir}"/keys.JOB.scp \
+ --asr_train_config "${asr_exp}"/config.yaml \
+ --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
+ --output_dir "${_logdir}"/output.JOB \
+ --mode paraformer \
+ ${_opts}
+
+ for f in token token_int score text; do
+ if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
+ for i in $(seq "${_nj}"); do
+ cat "${_logdir}/output.${i}/1best_recog/${f}"
+ done | sort -k1 >"${_dir}/${f}"
+ fi
+ done
+ python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
+ python utils/proce_text.py ${_data}/text ${_data}/text.proc
+ python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
+ tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
+ cat ${_dir}/text.cer.txt
+ done
+fi
\ No newline at end of file
diff --git a/egs/aishell2/paraformer/utils b/egs/aishell2/paraformer/utils
new file mode 120000
index 0000000..f245098
--- /dev/null
+++ b/egs/aishell2/paraformer/utils
@@ -0,0 +1 @@
+../../aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs/aishell2/paraformerbert/conf/decode_asr_transformer.yaml b/egs/aishell2/paraformerbert/conf/decode_asr_transformer.yaml
new file mode 100644
index 0000000..a147fa7
--- /dev/null
+++ b/egs/aishell2/paraformerbert/conf/decode_asr_transformer.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.5
+lm_weight: 0.7
diff --git a/egs/aishell2/paraformerbert/conf/train_asr_paraformerbert_conformer_20e_6d_1280_320.yaml b/egs/aishell2/paraformerbert/conf/train_asr_paraformerbert_conformer_20e_6d_1280_320.yaml
new file mode 100644
index 0000000..19f123e
--- /dev/null
+++ b/egs/aishell2/paraformerbert/conf/train_asr_paraformerbert_conformer_20e_6d_1280_320.yaml
@@ -0,0 +1,101 @@
+# network architecture
+# encoder related
+encoder: conformer
+encoder_conf:
+ output_size: 320 # dimension of attention
+ attention_heads: 4
+ linear_units: 1280 # the number of units of position-wise feed forward
+ num_blocks: 20 # the number of encoder blocks
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ attention_dropout_rate: 0.0
+ input_layer: conv2d # encoder architecture type
+ normalize_before: true
+ pos_enc_layer_type: rel_pos
+ selfattention_layer_type: rel_selfattn
+ activation_type: swish
+ macaron_style: true
+ use_cnn_module: true
+ cnn_module_kernel: 15
+
+# decoder related
+decoder: paraformer_decoder_san
+decoder_conf:
+ attention_heads: 4
+ linear_units: 1280
+ num_blocks: 6
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ self_attention_dropout_rate: 0.0
+ src_attention_dropout_rate: 0.0
+
+# hybrid CTC/attention
+model: paraformer_bert
+model_conf:
+ ctc_weight: 0.3
+ lsm_weight: 0.1 # label smoothing option
+ length_normalized_loss: false
+ predictor_weight: 1.0
+ glat_context_p: 0.4
+ embeds_id: 3
+ embed_dims: 768
+ embeds_loss_weight: 2.0
+
+# optimization related
+accum_grad: 2
+grad_clip: 5
+patience: none
+max_epoch: 50
+val_scheduler_criterion:
+ - valid
+ - acc
+best_model_criterion:
+- - valid
+ - acc
+ - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+ lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+ warmup_steps: 30000
+
+specaug: specaug
+specaug_conf:
+ apply_time_warp: true
+ time_warp_window: 5
+ time_warp_mode: bicubic
+ apply_freq_mask: true
+ freq_mask_width_range:
+ - 0
+ - 30
+ num_freq_mask: 2
+ apply_time_mask: true
+ time_mask_width_range:
+ - 0
+ - 40
+ num_time_mask: 2
+
+predictor: cif_predictor_sanm
+predictor_conf:
+ idim: 320
+ threshold: 1.0
+ l_order: 1
+ r_order: 1
+
+log_interval: 50
+normalize: None
+
+dataset_conf:
+ data_names: speech,text,embed
+ data_types: kaldi_ark,text,kaldi_ark
+ shuffle: True
+ shuffle_conf:
+ shuffle_size: 10240
+ sort_size: 500
+ batch_conf:
+ batch_type: token
+ batch_size: 25000
+ num_workers: 8
\ No newline at end of file
diff --git a/egs/aishell2/paraformerbert/local/extract_embeds.sh b/egs/aishell2/paraformerbert/local/extract_embeds.sh
new file mode 100755
index 0000000..5f45ff3
--- /dev/null
+++ b/egs/aishell2/paraformerbert/local/extract_embeds.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+stage=1
+stop_stage=3
+
+bert_model_root="../../huggingface_models"
+bert_model_name="bert-base-chinese"
+#bert_model_name="chinese-roberta-wwm-ext"
+#bert_model_name="mengzi-bert-base"
+raw_dataset_path="../DATA"
+model_path=${bert_model_root}/${bert_model_name}
+
+. utils/parse_options.sh || exit 1;
+
+nj=100
+
+for data_set in train dev_ios test_ios;do
+ scp=$raw_dataset_path/dump/fbank/${data_set}/text
+ local_scp_dir_raw=$raw_dataset_path/embeds/$bert_model_name/${data_set}
+ local_scp_dir=$local_scp_dir_raw/split$nj
+ local_records_dir=$local_scp_dir_raw/ark
+
+ mkdir -p $local_records_dir
+ mkdir -p $local_scp_dir
+
+ split_scps=""
+ for JOB in $(seq ${nj}); do
+ split_scps="$split_scps $local_scp_dir/data.$JOB.text"
+ done
+
+ utils/split_scp.pl $scp ${split_scps}
+
+
+ for num in {0..24};do
+ tmp=`expr $num \* 4`
+
+ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ for idx in {1..4}; do
+ JOB=`expr $tmp + $idx`
+ echo "proces jobid=$JOB"
+ {
+ beg=0
+ gpu=`expr $beg + $idx`
+ echo $local_scp_dir_raw/log/log.${JOB}
+ python tools/extract_embeds.py $local_scp_dir/text.$JOB.txt ${local_records_dir}/embeds.${JOB}.ark ${local_records_dir}/embeds.${JOB}.scp ${local_records_dir}/embeds.${JOB}.shape ${gpu} ${model_path} &> $local_scp_dir_raw/log/log.${JOB}
+ } &
+ done
+ wait
+ fi
+
+ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+ for idx in {1..4}; do
+ JOB=`expr $tmp + $idx`
+ echo "upload jobid=$JOB"
+ {
+ hadoop fs -put -f ${local_records_dir}/embeds.${JOB}.ark ${odps_des_feature_dir}/embeds.${JOB}.ark
+ } &
+ done
+ wait
+ fi
+ done
+
+ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+ for JOB in $(seq ${nj}); do
+ cat ${local_records_dir}/embeds.${JOB}.scp || exit 1;
+ done > ${local_scp_dir_raw}/embeds.scp
+
+ for JOB in $(seq ${nj}); do
+ cat ${local_records_dir}/embeds.${JOB}.shape || exit 1;
+ done > ${local_scp_dir_raw}/embeds.shape
+ fi
+done
+
+echo "embeds is in: ${local_scp_dir_raw}"
+echo "success"
\ No newline at end of file
diff --git a/egs/aishell2/paraformerbert/local/prepare_data.sh b/egs/aishell2/paraformerbert/local/prepare_data.sh
new file mode 100755
index 0000000..801dbe5
--- /dev/null
+++ b/egs/aishell2/paraformerbert/local/prepare_data.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG)
+# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU)
+# Apache 2.0
+
+# transform raw AISHELL-2 data to kaldi format
+
+. ./path.sh || exit 1;
+
+tmp=
+dir=
+
+if [ $# != 3 ]; then
+ echo "Usage: $0 <corpus-data-dir> <tmp-dir> <output-dir>"
+ echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train"
+ exit 1;
+fi
+
+corpus=$1
+#dict_dir=$2
+tmp=$2
+dir=$3
+
+echo "prepare_data.sh: Preparing data in $corpus"
+
+mkdir -p $tmp
+mkdir -p $dir
+
+# corpus check
+if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then
+ echo "Error: $0 requires wav.scp and trans.txt under $corpus directory."
+ exit 1;
+fi
+
+# validate utt-key list, IC0803W0380 is a bad utterance
+awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list
+awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list
+tools/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list
+
+# wav.scp
+awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp
+tools/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp
+
+# text
+tools/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text
+
+# copy prepared resources from tmp_dir to target dir
+mkdir -p $dir
+for f in wav.scp text; do
+ cp $tmp/$f $dir/$f || exit 1;
+done
+
+echo "local/prepare_data.sh succeeded"
+exit 0;
diff --git a/egs/aishell2/paraformerbert/path.sh b/egs/aishell2/paraformerbert/path.sh
new file mode 100755
index 0000000..ea3c0be
--- /dev/null
+++ b/egs/aishell2/paraformerbert/path.sh
@@ -0,0 +1,6 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
+export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs/aishell2/paraformerbert/run.sh b/egs/aishell2/paraformerbert/run.sh
new file mode 100755
index 0000000..d0407d4
--- /dev/null
+++ b/egs/aishell2/paraformerbert/run.sh
@@ -0,0 +1,263 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+
+# machines configuration
+CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+gpu_num=8
+count=1
+gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
+# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
+njob=5
+train_cmd=tools/run.pl
+infer_cmd=utils/run.pl
+
+# general configuration
+feats_dir="../DATA" #feature output dictionary, for large data
+exp_dir="."
+lang=zh
+dumpdir=dump/fbank
+feats_type=fbank
+token_type=char
+dataset_type=large
+scp=feats.scp
+type=kaldi_ark
+stage=0
+stop_stage=5
+
+skip_extract_embed=false
+bert_model_root="../../huggingface_models"
+bert_model_name="bert-base-chinese"
+
+# feature configuration
+feats_dim=80
+sample_frequency=16000
+nj=100
+speed_perturb="0.9,1.0,1.1"
+
+# data
+tr_dir=
+dev_tst_dir=
+
+# exp tag
+tag="exp1"
+
+. utils/parse_options.sh || exit 1;
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train
+valid_set=dev_ios
+test_sets="dev_ios test_ios"
+
+asr_config=conf/train_asr_paraformerbert_conformer_20e_6d_1280_320.yaml
+model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
+
+inference_config=conf/decode_asr_transformer_noctc_1best.yaml
+inference_asr_model=valid.acc.ave_10best.pth
+
+# you can set gpu num for decoding here
+gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, e.g., gpuid_list=2,3, the same as training stage by default
+ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
+
+if ${gpu_inference}; then
+ inference_nj=$[${ngpu}*${njob}]
+ _ngpu=1
+else
+ inference_nj=$njob
+ _ngpu=0
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+ echo "stage 0: Data preparation"
+ # For training set
+ local/prepare_data.sh ${tr_dir} data/local/train data/train || exit 1;
+ # # For dev and test set
+ for x in Android iOS Mic; do
+ local/prepare_data.sh ${dev_tst_dir}/${x}/dev data/local/dev_${x,,} data/dev_${x,,} || exit 1;
+ local/prepare_data.sh ${dev_tst_dir}/${x}/test data/local/test_${x,,} data/test_${x,,} || exit 1;
+ done
+ # Normalize text to capital letters
+ for x in train dev_android dev_ios dev_mic test_android test_ios test_mic; do
+ mv data/${x}/text data/${x}/text.org
+ paste <(cut -f 1 data/${x}/text.org) <(cut -f 2 data/${x}/text.org | tr '[:lower:]' '[:upper:]') \
+ > data/${x}/text
+ tools/text2token.py -n 1 -s 1 data/${x}/text > data/${x}/text.org
+ mv data/${x}/text.org data/${x}/text
+ done
+fi
+
+feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir}
+feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir}
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ echo "stage 1: Feature Generation"
+ # compute fbank features
+ fbankdir=${feats_dir}/fbank
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
+ data/train exp/make_fbank/train ${fbankdir}/train
+ tools/fix_data_feat.sh ${fbankdir}/train
+ for x in android ios mic; do
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ data/dev_${x} exp/make_fbank/dev_${x} ${fbankdir}/dev_${x}
+ tools/fix_data_feat.sh ${fbankdir}/dev_${x}
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ data/test_${x} exp/make_fbank/test_${x} ${fbankdir}/test_${x}
+ tools/fix_data_feat.sh ${fbankdir}/test_${x}
+ done
+
+ # compute global cmvn
+ steps/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/train exp/make_fbank/train
+
+ # apply cmvn
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${train_set} ${fbankdir}/train/cmvn.json exp/make_fbank/${train_set} ${feat_train_dir}
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${valid_set} ${fbankdir}/train/cmvn.json exp/make_fbank/${valid_set} ${feat_dev_dir}
+ for x in android ios mic; do
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/test_${x} ${fbankdir}/train/cmvn.json exp/make_fbank/test_${x} ${feats_dir}/${dumpdir}/test_${x}
+ done
+
+ cp ${fbankdir}/${train_set}/text ${fbankdir}/${train_set}/speech_shape ${fbankdir}/${train_set}/text_shape ${feat_train_dir}
+ tools/fix_data_feat.sh ${feat_train_dir}
+ cp ${fbankdir}/${valid_set}/text ${fbankdir}/${valid_set}/speech_shape ${fbankdir}/${valid_set}/text_shape ${feat_dev_dir}
+ tools/fix_data_feat.sh ${feat_dev_dir}
+ for x in android ios mic; do
+ cp ${fbankdir}/test_${x}/text ${fbankdir}/test_${x}/speech_shape ${fbankdir}/test_${x}/text_shape ${feats_dir}/${dumpdir}/test_${x}
+ tools/fix_data_feat.sh ${feats_dir}/${dumpdir}/test_${x}
+ done
+fi
+
+token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
+echo "dictionary: ${token_list}"
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+ echo "stage 2: Dictionary Preparation"
+ mkdir -p data/${lang}_token_list/char/
+
+ echo "make a dictionary"
+ echo "<blank>" > ${token_list}
+ echo "<s>" >> ${token_list}
+ echo "</s>" >> ${token_list}
+ tools/text2token.py -s 1 -n 1 --space "" data/${train_set}/text | cut -f 2- -d" " | tr " " "\n" \
+ | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list}
+ num_token=$(cat ${token_list} | wc -l)
+ echo "<unk>" >> ${token_list}
+ vocab_size=$(cat ${token_list} | wc -l)
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
+ mkdir -p asr_stats_fbank_zh_char/${train_set}
+ mkdir -p asr_stats_fbank_zh_char/${valid_set}
+ cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char asr_stats_fbank_zh_char/${train_set}
+ cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char asr_stats_fbank_zh_char/${valid_set}
+fi
+
+# Training Stage
+world_size=$gpu_num # run on one machine
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+ echo "stage 3: Training"
+ if ! "${skip_extract_embed}"; then
+ echo "extract embeddings..."
+ local/extract_embeds.sh \
+ --bert_model_root ${bert_model_root} \
+ --bert_model_name ${bert_model_name} \
+ --raw_dataset_path ${feats_dir}
+ fi
+ mkdir -p ${exp_dir}/exp/${model_dir}
+ mkdir -p ${exp_dir}/exp/${model_dir}/log
+ INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
+ if [ -f $INIT_FILE ];then
+ rm -f $INIT_FILE
+ fi
+ init_method=file://$(readlink -f $INIT_FILE)
+ echo "$0: init method is $init_method"
+ for ((i = 0; i < $gpu_num; ++i)); do
+ {
+ rank=$i
+ local_rank=$i
+ gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
+ asr_train_paraformer.py \
+ --gpu_id $gpu_id \
+ --use_preprocessor true \
+ --dataset_type $dataset_type \
+ --token_type $token_type \
+ --token_list $token_list \
+ --train_data_file $feats_dir/$dumpdir/${train_set}/data_bert.list \
+ --valid_data_file $feats_dir/$dumpdir/${valid_set}/data_bert.list \
+ --resume true \
+ --output_dir ${exp_dir}/exp/${model_dir} \
+ --config $asr_config \
+ --allow_variable_data_keys true \
+ --input_size $feats_dim \
+ --ngpu $gpu_num \
+ --num_worker_count $count \
+ --multiprocessing_distributed true \
+ --dist_init_method $init_method \
+ --dist_world_size $world_size \
+ --dist_rank $rank \
+ --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
+ } &
+ done
+ wait
+fi
+
+# Testing Stage
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+ echo "stage 4: Inference"
+ for dset in ${test_sets}; do
+ asr_exp=${exp_dir}/exp/${model_dir}
+ inference_tag="$(basename "${inference_config}" .yaml)"
+ _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
+ _logdir="${_dir}/logdir"
+ if [ -d ${_dir} ]; then
+ echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
+ exit 0
+ fi
+ mkdir -p "${_logdir}"
+ _data="${feats_dir}/${dumpdir}/${dset}"
+ key_file=${_data}/${scp}
+ num_scp_file="$(<${key_file} wc -l)"
+ _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
+ split_scps=
+ for n in $(seq "${_nj}"); do
+ split_scps+=" ${_logdir}/keys.${n}.scp"
+ done
+ # shellcheck disable=SC2086
+ utils/split_scp.pl "${key_file}" ${split_scps}
+ _opts=
+ if [ -n "${inference_config}" ]; then
+ _opts+="--config ${inference_config} "
+ fi
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
+ python -m funasr.bin.asr_inference_launch \
+ --batch_size 1 \
+ --ngpu "${_ngpu}" \
+ --njob ${njob} \
+ --gpuid_list ${gpuid_list} \
+ --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
+ --key_file "${_logdir}"/keys.JOB.scp \
+ --asr_train_config "${asr_exp}"/config.yaml \
+ --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
+ --output_dir "${_logdir}"/output.JOB \
+ --mode paraformer \
+ ${_opts}
+
+ for f in token token_int score text; do
+ if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
+ for i in $(seq "${_nj}"); do
+ cat "${_logdir}/output.${i}/1best_recog/${f}"
+ done | sort -k1 >"${_dir}/${f}"
+ fi
+ done
+ python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
+ python utils/proce_text.py ${_data}/text ${_data}/text.proc
+ python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
+ tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
+ cat ${_dir}/text.cer.txt
+ done
+fi
+
diff --git a/egs/aishell2/paraformerbert/utils b/egs/aishell2/paraformerbert/utils
new file mode 120000
index 0000000..f245098
--- /dev/null
+++ b/egs/aishell2/paraformerbert/utils
@@ -0,0 +1 @@
+../../aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs/aishell2/transformer/conf/decode_asr_transformer.yaml b/egs/aishell2/transformer/conf/decode_asr_transformer.yaml
new file mode 100644
index 0000000..a147fa7
--- /dev/null
+++ b/egs/aishell2/transformer/conf/decode_asr_transformer.yaml
@@ -0,0 +1,6 @@
+beam_size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc_weight: 0.5
+lm_weight: 0.7
diff --git a/egs/aishell2/transformer/conf/train_asr_transformer.yaml b/egs/aishell2/transformer/conf/train_asr_transformer.yaml
new file mode 100644
index 0000000..3e2172d
--- /dev/null
+++ b/egs/aishell2/transformer/conf/train_asr_transformer.yaml
@@ -0,0 +1,80 @@
+# network architecture
+# encoder related
+encoder: transformer
+encoder_conf:
+ output_size: 256 # dimension of attention
+ attention_heads: 4
+ linear_units: 2048 # the number of units of position-wise feed forward
+ num_blocks: 12 # the number of encoder blocks
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ attention_dropout_rate: 0.0
+ input_layer: conv2d # encoder architecture type
+ normalize_before: true
+
+# decoder related
+decoder: transformer
+decoder_conf:
+ attention_heads: 4
+ linear_units: 2048
+ num_blocks: 6
+ dropout_rate: 0.1
+ positional_dropout_rate: 0.1
+ self_attention_dropout_rate: 0.0
+ src_attention_dropout_rate: 0.0
+
+# hybrid CTC/attention
+model_conf:
+ ctc_weight: 0.3
+ lsm_weight: 0.1 # label smoothing option
+ length_normalized_loss: false
+
+# optimization related
+accum_grad: 2
+grad_clip: 5
+patience: none
+max_epoch: 50
+val_scheduler_criterion:
+ - valid
+ - acc
+best_model_criterion:
+- - valid
+ - acc
+ - max
+keep_nbest_models: 10
+
+optim: adam
+optim_conf:
+ lr: 0.002
+scheduler: warmuplr # pytorch v1.1.0+ required
+scheduler_conf:
+ warmup_steps: 25000
+
+specaug: specaug
+specaug_conf:
+ apply_time_warp: true
+ time_warp_window: 5
+ time_warp_mode: bicubic
+ apply_freq_mask: true
+ freq_mask_width_range:
+ - 0
+ - 30
+ num_freq_mask: 2
+ apply_time_mask: true
+ time_mask_width_range:
+ - 0
+ - 40
+ num_time_mask: 2
+
+log_interval: 50
+normalize: None
+
+dataset_conf:
+ shuffle: True
+ shuffle_conf:
+ shuffle_size: 2048
+ sort_size: 500
+ batch_conf:
+ batch_type: token
+ batch_size: 25000
+ num_workers: 8
\ No newline at end of file
diff --git a/egs/aishell2/transformer/local/prepare_data.sh b/egs/aishell2/transformer/local/prepare_data.sh
new file mode 100755
index 0000000..ce6ee19
--- /dev/null
+++ b/egs/aishell2/transformer/local/prepare_data.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG)
+# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU)
+# Apache 2.0
+
+# transform raw AISHELL-2 data to kaldi format
+
+. ./path.sh || exit 1;
+
+tmp=
+dir=
+
+if [ $# != 3 ]; then
+ echo "Usage: $0 <corpus-data-dir> <tmp-dir> <output-dir>"
+ echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train"
+ exit 1;
+fi
+
+corpus=$1
+tmp=$2
+dir=$3
+
+echo "prepare_data.sh: Preparing data in $corpus"
+
+mkdir -p $tmp
+mkdir -p $dir
+
+# corpus check
+if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then
+ echo "Error: $0 requires wav.scp and trans.txt under $corpus directory."
+ exit 1;
+fi
+
+# validate utt-key list, IC0803W0380 is a bad utterance
+awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list
+awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list
+tools/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list
+
+# wav.scp
+awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp
+tools/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp
+
+# text
+tools/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text
+
+# copy prepared resources from tmp_dir to target dir
+mkdir -p $dir
+for f in wav.scp text; do
+ cp $tmp/$f $dir/$f || exit 1;
+done
+
+echo "local/prepare_data.sh succeeded"
+exit 0;
diff --git a/egs/aishell2/transformer/path.sh b/egs/aishell2/transformer/path.sh
new file mode 100755
index 0000000..ea3c0be
--- /dev/null
+++ b/egs/aishell2/transformer/path.sh
@@ -0,0 +1,6 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=../../../:$PYTHONPATH
+export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs/aishell2/transformer/run.sh b/egs/aishell2/transformer/run.sh
new file mode 100755
index 0000000..a5a14ec
--- /dev/null
+++ b/egs/aishell2/transformer/run.sh
@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+
+# machines configuration
+CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+gpu_num=8
+count=1
+gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
+# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
+njob=5
+train_cmd=tools/run.pl
+infer_cmd=utils/run.pl
+
+# general configuration
+feats_dir="../DATA" #feature output dictionary
+exp_dir="."
+lang=zh
+dumpdir=dump/fbank
+feats_type=fbank
+token_type=char
+dataset_type=large
+scp=feats.scp
+type=kaldi_ark
+stage=0
+stop_stage=4
+
+# feature configuration
+feats_dim=80
+sample_frequency=16000
+nj=100
+speed_perturb="0.9,1.0,1.1"
+
+# data
+tr_dir=
+dev_tst_dir=
+
+# exp tag
+tag="exp1"
+
+. utils/parse_options.sh || exit 1;
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train
+valid_set=dev_ios
+test_sets="dev_ios test_ios"
+
+asr_config=conf/train_asr_transformer.yaml
+model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
+
+inference_config=conf/decode_asr_transformer.yaml
+inference_asr_model=valid.acc.ave_10best.pth
+
+# you can set gpu num for decoding here
+gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, e.g., gpuid_list=2,3, the same as training stage by default
+ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
+
+if ${gpu_inference}; then
+ inference_nj=$[${ngpu}*${njob}]
+ _ngpu=1
+else
+ inference_nj=$njob
+ _ngpu=0
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+ echo "stage 0: Data preparation"
+ # For training set
+ local/prepare_data.sh ${tr_dir} ${feats_dir}/data/local/train ${feats_dir}/data/train || exit 1;
+ # # For dev and test set
+ for x in Android iOS Mic; do
+ local/prepare_data.sh ${dev_tst_dir}/${x}/dev ${feats_dir}/data/local/dev_${x,,} ${feats_dir}/data/dev_${x,,} || exit 1;
+ local/prepare_data.sh ${dev_tst_dir}/${x}/test ${feats_dir}/data/local/test_${x,,} ${feats_dir}/data/test_${x,,} || exit 1;
+ done
+ # Normalize text to capital letters
+ for x in train dev_ios test_ios; do
+ mv ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org
+ paste -d " " <(cut -f 1 ${feats_dir}/data/${x}/text.org) <(cut -f 2- ${feats_dir}/data/${x}/text.org \
+ | tr 'A-Z' 'a-z' | tr -d " ") \
+ > ${feats_dir}/data/${x}/text
+ tools/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org
+ mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text
+ done
+fi
+
+feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir}
+feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir}
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ echo "stage 1: Feature Generation"
+ # compute fbank features
+ fbankdir=${feats_dir}/fbank
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
+ ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
+ tools/fix_data_feat.sh ${fbankdir}/train
+ for x in android ios mic; do
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/dev_${x} ${exp_dir}/exp/make_fbank/dev_${x} ${fbankdir}/dev_${x}
+ tools/fix_data_feat.sh ${fbankdir}/dev_${x}
+ steps/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
+ ${feats_dir}/data/test_${x} ${exp_dir}/exp/make_fbank/test_${x} ${fbankdir}/test_${x}
+ tools/fix_data_feat.sh ${fbankdir}/test_${x}
+ done
+
+ # compute global cmvn
+ steps/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/train ${exp_dir}/exp/make_fbank/train
+
+ # apply cmvn
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${train_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${train_set} ${feat_train_dir}
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/${valid_set} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/${valid_set} ${feat_dev_dir}
+ for x in android ios mic; do
+ steps/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \
+ ${fbankdir}/test_${x} ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test_${x} ${feats_dir}/${dumpdir}/test_${x}
+ done
+
+ cp ${fbankdir}/${train_set}/text ${fbankdir}/${train_set}/speech_shape ${fbankdir}/${train_set}/text_shape ${feat_train_dir}
+ tools/fix_data_feat.sh ${feat_train_dir}
+ cp ${fbankdir}/${valid_set}/text ${fbankdir}/${valid_set}/speech_shape ${fbankdir}/${valid_set}/text_shape ${feat_dev_dir}
+ tools/fix_data_feat.sh ${feat_dev_dir}
+ for x in android ios mic; do
+ cp ${fbankdir}/test_${x}/text ${fbankdir}/test_${x}/speech_shape ${fbankdir}/test_${x}/text_shape ${feats_dir}/${dumpdir}/test_${x}
+ tools/fix_data_feat.sh ${feats_dir}/${dumpdir}/test_${x}
+ done
+fi
+
+token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
+echo "dictionary: ${token_list}"
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+ echo "stage 2: Dictionary Preparation"
+ mkdir -p ${feats_dir}/data/${lang}_token_list/char/
+
+ echo "make a dictionary"
+ echo "<blank>" > ${token_list}
+ echo "<s>" >> ${token_list}
+ echo "</s>" >> ${token_list}
+ tools/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/${train_set}/text | cut -f 2- -d" " | tr " " "\n" \
+ | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list}
+ num_token=$(cat ${token_list} | wc -l)
+ echo "<unk>" >> ${token_list}
+ vocab_size=$(cat ${token_list} | wc -l)
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
+ awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+ cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
+ cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
+fi
+
+# Training Stage
+world_size=$gpu_num # run on one machine
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+ echo "stage 3: Training"
+ mkdir -p ${exp_dir}/exp/${model_dir}
+ mkdir -p ${exp_dir}/exp/${model_dir}/log
+ INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
+ if [ -f $INIT_FILE ];then
+ rm -f $INIT_FILE
+ fi
+ init_method=file://$(readlink -f $INIT_FILE)
+ echo "$0: init method is $init_method"
+ for ((i = 0; i < $gpu_num; ++i)); do
+ {
+ rank=$i
+ local_rank=$i
+ gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
+ asr_train.py \
+ --gpu_id $gpu_id \
+ --use_preprocessor true \
+ --dataset_type $dataset_type \
+ --token_type char \
+ --token_list $token_list \
+ --train_data_file $feats_dir/$dumpdir/${train_set}/data.list \
+ --valid_data_file $feats_dir/$dumpdir/${valid_set}/data.list \
+ --resume true \
+ --output_dir ${exp_dir}/exp/${model_dir} \
+ --config $asr_config \
+ --input_size $feats_dim \
+ --ngpu $gpu_num \
+ --num_worker_count $count \
+ --multiprocessing_distributed true \
+ --dist_init_method $init_method \
+ --dist_world_size $world_size \
+ --dist_rank $rank \
+ --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
+ } &
+ done
+ wait
+fi
+
+# Testing Stage
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+ echo "stage 4: Inference"
+ for dset in ${test_sets}; do
+ asr_exp=${exp_dir}/exp/${model_dir}
+ inference_tag="$(basename "${inference_config}" .yaml)"
+ _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
+ _logdir="${_dir}/logdir"
+ if [ -d ${_dir} ]; then
+ echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
+ exit 0
+ fi
+ mkdir -p "${_logdir}"
+ _data="${feats_dir}/${dumpdir}/${dset}"
+ key_file=${_data}/${scp}
+ num_scp_file="$(<${key_file} wc -l)"
+ _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
+ split_scps=
+ for n in $(seq "${_nj}"); do
+ split_scps+=" ${_logdir}/keys.${n}.scp"
+ done
+ # shellcheck disable=SC2086
+ utils/split_scp.pl "${key_file}" ${split_scps}
+ _opts=
+ if [ -n "${inference_config}" ]; then
+ _opts+="--config ${inference_config} "
+ fi
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
+ python -m funasr.bin.asr_inference_launch \
+ --batch_size 1 \
+ --ngpu "${_ngpu}" \
+ --njob ${njob} \
+ --gpuid_list ${gpuid_list} \
+ --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
+ --key_file "${_logdir}"/keys.JOB.scp \
+ --asr_train_config "${asr_exp}"/config.yaml \
+ --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
+ --output_dir "${_logdir}"/output.JOB \
+ --mode asr \
+ ${_opts}
+
+ for f in token token_int score text; do
+ if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
+ for i in $(seq "${_nj}"); do
+ cat "${_logdir}/output.${i}/1best_recog/${f}"
+ done | sort -k1 >"${_dir}/${f}"
+ fi
+ done
+ python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
+ python utils/proce_text.py ${_data}/text ${_data}/text.proc
+ python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
+ tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
+ cat ${_dir}/text.cer.txt
+ done
+fi
+
+
+
diff --git a/egs_modelscope/common/utils/__init__.py b/egs/aishell2/transformer/utils/__init__.py
similarity index 100%
rename from egs_modelscope/common/utils/__init__.py
rename to egs/aishell2/transformer/utils/__init__.py
diff --git a/egs_modelscope/common/utils/apply_cmvn.py b/egs/aishell2/transformer/utils/apply_cmvn.py
similarity index 100%
rename from egs_modelscope/common/utils/apply_cmvn.py
rename to egs/aishell2/transformer/utils/apply_cmvn.py
diff --git a/egs_modelscope/common/utils/apply_cmvn.sh b/egs/aishell2/transformer/utils/apply_cmvn.sh
similarity index 100%
rename from egs_modelscope/common/utils/apply_cmvn.sh
rename to egs/aishell2/transformer/utils/apply_cmvn.sh
diff --git a/egs_modelscope/common/utils/apply_lfr_and_cmvn.py b/egs/aishell2/transformer/utils/apply_lfr_and_cmvn.py
similarity index 100%
rename from egs_modelscope/common/utils/apply_lfr_and_cmvn.py
rename to egs/aishell2/transformer/utils/apply_lfr_and_cmvn.py
diff --git a/egs_modelscope/common/utils/apply_lfr_and_cmvn.sh b/egs/aishell2/transformer/utils/apply_lfr_and_cmvn.sh
similarity index 100%
rename from egs_modelscope/common/utils/apply_lfr_and_cmvn.sh
rename to egs/aishell2/transformer/utils/apply_lfr_and_cmvn.sh
diff --git a/egs_modelscope/common/utils/combine_cmvn_file.py b/egs/aishell2/transformer/utils/combine_cmvn_file.py
similarity index 100%
rename from egs_modelscope/common/utils/combine_cmvn_file.py
rename to egs/aishell2/transformer/utils/combine_cmvn_file.py
diff --git a/egs_modelscope/common/utils/compute_cmvn.py b/egs/aishell2/transformer/utils/compute_cmvn.py
similarity index 100%
rename from egs_modelscope/common/utils/compute_cmvn.py
rename to egs/aishell2/transformer/utils/compute_cmvn.py
diff --git a/egs_modelscope/common/utils/compute_cmvn.sh b/egs/aishell2/transformer/utils/compute_cmvn.sh
similarity index 100%
rename from egs_modelscope/common/utils/compute_cmvn.sh
rename to egs/aishell2/transformer/utils/compute_cmvn.sh
diff --git a/egs_modelscope/common/utils/compute_fbank.py b/egs/aishell2/transformer/utils/compute_fbank.py
similarity index 100%
rename from egs_modelscope/common/utils/compute_fbank.py
rename to egs/aishell2/transformer/utils/compute_fbank.py
diff --git a/egs_modelscope/common/utils/compute_fbank.sh b/egs/aishell2/transformer/utils/compute_fbank.sh
similarity index 100%
rename from egs_modelscope/common/utils/compute_fbank.sh
rename to egs/aishell2/transformer/utils/compute_fbank.sh
diff --git a/egs_modelscope/common/utils/compute_wer.py b/egs/aishell2/transformer/utils/compute_wer.py
similarity index 100%
rename from egs_modelscope/common/utils/compute_wer.py
rename to egs/aishell2/transformer/utils/compute_wer.py
diff --git a/egs_modelscope/common/utils/error_rate_zh b/egs/aishell2/transformer/utils/error_rate_zh
similarity index 100%
rename from egs_modelscope/common/utils/error_rate_zh
rename to egs/aishell2/transformer/utils/error_rate_zh
diff --git a/egs_modelscope/common/utils/extract_embeds.py b/egs/aishell2/transformer/utils/extract_embeds.py
similarity index 100%
rename from egs_modelscope/common/utils/extract_embeds.py
rename to egs/aishell2/transformer/utils/extract_embeds.py
diff --git a/egs_modelscope/common/utils/filter_scp.pl b/egs/aishell2/transformer/utils/filter_scp.pl
similarity index 100%
rename from egs_modelscope/common/utils/filter_scp.pl
rename to egs/aishell2/transformer/utils/filter_scp.pl
diff --git a/egs_modelscope/common/utils/fix_data.sh b/egs/aishell2/transformer/utils/fix_data.sh
similarity index 100%
rename from egs_modelscope/common/utils/fix_data.sh
rename to egs/aishell2/transformer/utils/fix_data.sh
diff --git a/egs_modelscope/common/utils/fix_data_feat.sh b/egs/aishell2/transformer/utils/fix_data_feat.sh
similarity index 100%
rename from egs_modelscope/common/utils/fix_data_feat.sh
rename to egs/aishell2/transformer/utils/fix_data_feat.sh
diff --git a/egs_modelscope/common/utils/gen_ark_list.sh b/egs/aishell2/transformer/utils/gen_ark_list.sh
similarity index 100%
rename from egs_modelscope/common/utils/gen_ark_list.sh
rename to egs/aishell2/transformer/utils/gen_ark_list.sh
diff --git a/egs_modelscope/common/utils/parse_options.sh b/egs/aishell2/transformer/utils/parse_options.sh
similarity index 100%
rename from egs_modelscope/common/utils/parse_options.sh
rename to egs/aishell2/transformer/utils/parse_options.sh
diff --git a/egs_modelscope/common/utils/print_args.py b/egs/aishell2/transformer/utils/print_args.py
similarity index 100%
rename from egs_modelscope/common/utils/print_args.py
rename to egs/aishell2/transformer/utils/print_args.py
diff --git a/egs_modelscope/common/utils/proc_conf_oss.py b/egs/aishell2/transformer/utils/proc_conf_oss.py
similarity index 100%
rename from egs_modelscope/common/utils/proc_conf_oss.py
rename to egs/aishell2/transformer/utils/proc_conf_oss.py
diff --git a/egs_modelscope/common/utils/proce_text.py b/egs/aishell2/transformer/utils/proce_text.py
similarity index 100%
rename from egs_modelscope/common/utils/proce_text.py
rename to egs/aishell2/transformer/utils/proce_text.py
diff --git a/egs_modelscope/common/utils/run.pl b/egs/aishell2/transformer/utils/run.pl
similarity index 100%
rename from egs_modelscope/common/utils/run.pl
rename to egs/aishell2/transformer/utils/run.pl
diff --git a/egs_modelscope/common/utils/shuffle_list.pl b/egs/aishell2/transformer/utils/shuffle_list.pl
similarity index 100%
rename from egs_modelscope/common/utils/shuffle_list.pl
rename to egs/aishell2/transformer/utils/shuffle_list.pl
diff --git a/egs_modelscope/common/utils/split_data.py b/egs/aishell2/transformer/utils/split_data.py
similarity index 100%
rename from egs_modelscope/common/utils/split_data.py
rename to egs/aishell2/transformer/utils/split_data.py
diff --git a/egs_modelscope/common/utils/split_scp.pl b/egs/aishell2/transformer/utils/split_scp.pl
similarity index 100%
rename from egs_modelscope/common/utils/split_scp.pl
rename to egs/aishell2/transformer/utils/split_scp.pl
diff --git a/egs_modelscope/common/utils/subset_data_dir_tr_cv.sh b/egs/aishell2/transformer/utils/subset_data_dir_tr_cv.sh
similarity index 100%
rename from egs_modelscope/common/utils/subset_data_dir_tr_cv.sh
rename to egs/aishell2/transformer/utils/subset_data_dir_tr_cv.sh
diff --git a/egs_modelscope/common/utils/text2token.py b/egs/aishell2/transformer/utils/text2token.py
similarity index 100%
rename from egs_modelscope/common/utils/text2token.py
rename to egs/aishell2/transformer/utils/text2token.py
diff --git a/egs_modelscope/common/utils/text_tokenize.py b/egs/aishell2/transformer/utils/text_tokenize.py
similarity index 100%
rename from egs_modelscope/common/utils/text_tokenize.py
rename to egs/aishell2/transformer/utils/text_tokenize.py
diff --git a/egs_modelscope/common/utils/text_tokenize.sh b/egs/aishell2/transformer/utils/text_tokenize.sh
similarity index 100%
rename from egs_modelscope/common/utils/text_tokenize.sh
rename to egs/aishell2/transformer/utils/text_tokenize.sh
diff --git a/egs_modelscope/common/utils/textnorm_zh.py b/egs/aishell2/transformer/utils/textnorm_zh.py
similarity index 100%
rename from egs_modelscope/common/utils/textnorm_zh.py
rename to egs/aishell2/transformer/utils/textnorm_zh.py
diff --git a/egs/cnceleb/resnet34/README.md b/egs/cnceleb/resnet34/README.md
new file mode 100644
index 0000000..994e315
--- /dev/null
+++ b/egs/cnceleb/resnet34/README.md
@@ -0,0 +1,18 @@
+
+# ResNet34 Result
+
+## Training Config
+- Feature info: using 80 dims fbank, no cmvn, speed perturb(0.9, 1.0, 1.1)
+- Train info: lr 1e-4, batch_size 64, 1 gpu(Tesla V100), acc_grad 1, 300000 steps, clip_gradient_norm 3.0, weight_l2_regularizer 0.01
+- Loss info: additive angular margin softmax, feature_scaling_factor=8, margin 0.25
+- Model info: ResNet34, global statistics pooling, Dense
+- Train config: conf/train_sv_resnet34.yaml
+- Model size: 5.60 M parameters
+
+## Results (EER & minDCF)
+- Test set: Alimeeting-test, CN-Celeb-eval-speech
+
+| testset | EER(%) | minDCF | Threshold |
+|:---------------------:|:-------:|:-------:| :--------:|
+| Alimeeting-test | 1.45 | 0.0849 | 0.9666 |
+| CN-Celeb-eval-speech | 9.00 | 0.2936 | 0.9465 |
\ No newline at end of file
diff --git a/egs/cnceleb/resnet34/configuration.json b/egs/cnceleb/resnet34/configuration.json
new file mode 100644
index 0000000..0070da3
--- /dev/null
+++ b/egs/cnceleb/resnet34/configuration.json
@@ -0,0 +1,20 @@
+{
+ "framework": "pytorch",
+ "task": "speaker-verification",
+ "model": {
+ "type": "generic-sv",
+ "sv_model_name": "sv.pb",
+ "model_config": {
+ "type": "pytorch",
+ "code_base": "funasr",
+ "mode": "sv",
+ "lang": "zh-cn",
+ "batch_size": 1,
+ "sv_model_name": "sv.pb",
+ "sv_model_config": "sv.yaml"
+ }
+ },
+ "pipeline": {
+ "type": "sv-inference"
+ }
+}
\ No newline at end of file
diff --git a/egs/cnceleb/resnet34/local/convert_tf_ckpt_to_torch.py b/egs/cnceleb/resnet34/local/convert_tf_ckpt_to_torch.py
new file mode 100644
index 0000000..0dcf2e4
--- /dev/null
+++ b/egs/cnceleb/resnet34/local/convert_tf_ckpt_to_torch.py
@@ -0,0 +1,93 @@
+import numpy as np
+import logging
+from typing import Dict
+import sys
+import torch
+
+
+def load_ckpt(checkpoint_path: str) -> Dict[str, np.ndarray]:
+ from tensorflow.python import pywrap_tensorflow
+
+ reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path)
+ var_to_shape_map = reader.get_variable_to_shape_map()
+
+ var_dict = dict()
+ for var_name in sorted(var_to_shape_map):
+ if "optimizer" in var_name:
+ continue
+ tensor = reader.get_tensor(var_name)
+ print("in ckpt: {}, {}".format(var_name, tensor.shape)) # print(tensor)
+ var_dict[var_name] = tensor
+
+ return var_dict
+
+
+def convert_parameter_name_for_asv_resnet34(
+ var_dict: Dict[str, np.ndarray],
+ old_prefix: str = "EAND/speech_encoder",
+ new_prefix: str = "encoder",
+ train_steps: int = 0
+) -> Dict[str, np.ndarray]:
+ new_dict = dict()
+ model_size = 0
+ for name, tensor in var_dict.items():
+ if not name.startswith(old_prefix):
+ if name == "softmax/output/kernel":
+ new_name = "decoder.output_dense.weight"
+ tensor = np.transpose(tensor, [1, 0])
+ new_dict[new_name] = torch.Tensor(tensor)
+ continue
+ new_name = name.replace(old_prefix, new_prefix)
+ new_name = new_name.replace("/", ".")
+ if "resnet1" in new_name or "resnet2" in new_name:
+ new_name = new_name.replace("encoder", "decoder")
+ module_name, para_name = new_name.rsplit(".", 1)
+ # process for batch normalization
+ if "bn" in module_name:
+ new_name = new_name.replace("gamma", "weight")
+ new_name = new_name.replace("beta", "bias")
+ new_name = new_name.replace("moving_mean", "running_mean")
+ new_name = new_name.replace("moving_variance", "running_var")
+
+ new_dict[new_name] = torch.Tensor(tensor)
+ new_dict[module_name + ".num_batches_tracked"] = torch.Tensor(train_steps)
+
+ # process for dense layers
+ elif "dense" in module_name:
+ new_name = new_name.replace("kernel", "weight")
+ if para_name == "kernel":
+ if len(tensor.shape) == 2:
+ tensor = np.transpose(tensor, [1, 0])
+ elif len(tensor.shape) == 3:
+ tensor = np.transpose(tensor, [2, 1, 0])
+ # for dense0
+ elif len(tensor.shape) == 4:
+ tensor = np.transpose(tensor, [3, 2, 0, 1])
+
+ new_dict[new_name] = torch.Tensor(tensor)
+
+ # process for conv layers
+ elif "conv" in module_name:
+ new_name = new_name.replace("kernel", "weight")
+ if para_name == "kernel":
+ tensor = np.transpose(tensor, [3, 2, 0, 1])
+
+ new_dict[new_name] = torch.Tensor(tensor)
+
+ print("{} -> {}".format(name, new_name))
+ model_size += new_dict[new_name].numel()
+ print("Model size: {}".format(model_size))
+ return new_dict
+
+
+if __name__ == '__main__':
+ checkpoint_path = sys.argv[1]
+ pkl_path = sys.argv[2]
+ tf_dict = load_ckpt(checkpoint_path)
+ torch_dict = convert_parameter_name_for_asv_resnet34(
+ tf_dict,
+ train_steps=300000,
+ )
+ torch.save(
+ torch_dict, pkl_path
+ )
diff --git a/egs/cnceleb/resnet34/local/make_cnceleb1.sh b/egs/cnceleb/resnet34/local/make_cnceleb1.sh
new file mode 100644
index 0000000..59bce15
--- /dev/null
+++ b/egs/cnceleb/resnet34/local/make_cnceleb1.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Copyright 2019 Jiawen Kang
+# Apache 2.0.
+#
+# This script prepares the CN-Celeb1 dataset.
+# It creates separate directories for train, eval enroll and eval test.
+# It also prepares a trials files, in the eval test directory.
+
+if [ $# != 2 ]; then
+ echo "Usage: make_cnceleb1.sh <CN-Celeb1_PATH> <out_dir>"
+ echo "E.g.: make_cnceleb1.sh /export/corpora/CN-Celeb1 data"
+ exit 1
+fi
+
+in_dir=$1
+out_dir=$2
+
+# Prepare the cnceleb1 training data
+this_out_dir=${out_dir}/cnceleb1_train
+mkdir -p $this_out_dir 2>/dev/null
+WAVFILE=$this_out_dir/wav.scp
+SPKFILE=$this_out_dir/utt2spk
+rm $WAVFILE $SPKFILE 2>/dev/null
+this_in_dir=${in_dir}/dev
+
+#for spkr_id in `cat $this_in_dir/dev.lst`; do
+# for f in $in_dir/data/$spkr_id/*.wav; do
+# wav_id=$(basename $f | sed s:.wav$::)
+# echo "${spkr_id}-${wav_id} $f" >> $WAVFILE
+# echo "${spkr_id}-${wav_id} ${spkr_id}" >> $SPKFILE
+# done
+#done
+# utils/fix_data_dir.sh $this_out_dir
+
+# Prepare the evaluation data
+for mode in enroll test; do
+ this_out_dir=${out_dir}/eval_${mode}
+ mkdir -p $this_out_dir 2>/dev/null
+ WAVFILE=$this_out_dir/wav.scp
+ SPKFILE=$this_out_dir/utt2spk
+ rm $WAVFILE $SPKFILE 2>/dev/null
+ this_in_dir=${in_dir}/eval/${mode}
+
+ for f in $this_in_dir/*.wav; do
+ wav_id=$(basename $f | sed s:.wav$::)
+ spkr_id=$(echo ${wav_id} | cut -d "-" -f1)
+ echo "${wav_id} $f" >> $WAVFILE
+ echo "${wav_id} ${spkr_id}" >> $SPKFILE
+ done
+ # utils/fix_data_dir.sh $this_out_dir
+done
+
+# Prepare test trials
+this_out_dir=$out_dir/eval_test/trials
+mkdir -p $out_dir/eval_test/trials
+this_in_dir=${in_dir}/eval/lists
+cat $this_in_dir/trials.lst | sed 's@-enroll@@g' | sed 's@test/@@g' | sed 's@.wav@@g' | \
+ awk '{if ($3 == "1")
+ {print $1,$2,"target"}
+ else
+ {print $1,$2,"nontarget"}
+ }'> $this_out_dir/trials.lst
diff --git a/egs/cnceleb/resnet34/local/make_cnceleb2.sh b/egs/cnceleb/resnet34/local/make_cnceleb2.sh
new file mode 100644
index 0000000..7bdad47
--- /dev/null
+++ b/egs/cnceleb/resnet34/local/make_cnceleb2.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright 2020 Jiawen Kang
+# Apache 2.0.
+#
+# This script prepares the CN-Celeb2 dataset.
+
+if [ $# != 2 ]; then
+ echo "Usage: make_cnceleb2.sh <CN-Celeb2_PATH> <out_dir>"
+ echo "E.g.: make_cnceleb2.sh /export/corpora/CN-Celeb2 data"
+ exit 1
+fi
+
+in_dir=$1
+out_dir=$2
+
+# Prepare the cnceleb2 training data
+this_out_dir=${out_dir}
+mkdir -p $this_out_dir 2>/dev/null
+WAVFILE=$this_out_dir/wav.scp
+SPKFILE=$this_out_dir/utt2spk
+rm $WAVFILE $SPKFILE 2>/dev/null
+this_in_dir=${in_dir}
+
+for spkr_id in `cat $this_in_dir/spk.lst`; do
+ for f in $in_dir/data/$spkr_id/*.wav; do
+ wav_id=$(basename $f | sed s:.wav$::)
+ echo "${spkr_id}-${wav_id} $f" >> $WAVFILE
+ echo "${spkr_id}-${wav_id} ${spkr_id}" >> $SPKFILE
+ done
+done
+
+# utils/fix_data_dir.sh $this_out_dir
\ No newline at end of file
diff --git a/egs_modelscope/aishell/paraformer/path.sh b/egs/cnceleb/resnet34/path.sh
similarity index 100%
rename from egs_modelscope/aishell/paraformer/path.sh
rename to egs/cnceleb/resnet34/path.sh
diff --git a/egs/cnceleb/resnet34/run.sh b/egs/cnceleb/resnet34/run.sh
new file mode 100644
index 0000000..cf1011d
--- /dev/null
+++ b/egs/cnceleb/resnet34/run.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+. ./path.sh || exit 1;
+
+# machines configuration
+gpu_devices="6,7"
+gpu_num=2
+count=1
+gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
+# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob
+njob=1
+train_cmd=utils/run.pl
+infer_cmd=utils/run.pl
+
+# general configuration
+feats_dir="." #feature output dictionary
+exp_dir="."
+lang=zh
+dumpdir=dump/fbank
+feats_type=fbank
+token_type=spk
+scp=feats.scp
+type=kaldi_ark
+stage=0
+stop_stage=4
+
+# feature configuration
+feats_dim=80
+sample_frequency=16000
+nj=32
+speed_perturb="0.9,1.0,1.1"
+
+# data
+data_cnceleb=
+
+# exp tag
+tag=""
+inference_tag="basic"
+inference_sv_model=sv.pb
+sv_config=sv.yaml
+
+. utils/parse_options.sh || exit 1;
+
+model_dir="baseline_$(basename "${sv_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=train
+valid_set=dev
+test_sets="eval_enroll eval_test"
+
+# you can set gpu num for decoding here
+gpuid_list=$gpu_devices # set gpus for decoding, the same as training stage by default
+ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
+
+if ${gpu_inference}; then
+ inference_nj=$[${ngpu}*${njob}]
+ _ngpu=1
+else
+ inference_nj=$njob
+ _ngpu=0
+fi
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+ echo "stage 0: Data preparation"
+ # Data preparation
+ bash local/make_cnceleb1.sh ${data_cnceleb}/CN-Celeb1 ${feats_dir}/data
+ # bash local/make_cnceleb2.sh ${data_cnceleb}/CN-Celeb2 ${feats_dir}/data/cnceleb2_train
+ grep speech ${feats_dir}/data/eval_test/trials/trials.lst > ${feats_dir}/data/eval_test/trials/trials.lst.speech
+ # local/combine_data.sh ${feats_dir}/data/train ${feats_dir}/data/cnceleb1_train ${feats_dir}/data/cnceleb2_train
+fi
+
+# Testing Stage
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+ echo "stage 1: Inference"
+ for dset in ${test_sets}; do
+ echo "extracting embedding for ${dset}"
+ asr_exp=${exp_dir}/exp/${model_dir}
+ _dir="${asr_exp}/${inference_tag}/${inference_sv_model}/${dset}"
+ _logdir="${_dir}/logdir"
+ if [ -d ${_dir} ]; then
+ echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
+ exit 0
+ fi
+ mkdir -p "${_logdir}"
+ _data="data/${dset}"
+ key_file=${_data}/wav.scp
+ num_scp_file="$(<${key_file} wc -l)"
+ _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
+ split_scps=
+ for n in $(seq "${_nj}"); do
+ split_scps+=" ${_logdir}/keys.${n}.scp"
+ done
+ # shellcheck disable=SC2086
+ utils/split_scp.pl "${key_file}" ${split_scps}
+
+ ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/sv_inference.JOB.log \
+ python -m funasr.bin.sv_inference \
+ --gpuid_list ${gpu_devices} \
+ --ngpu ${_ngpu} \
+ --key_file "${_logdir}"/keys.JOB.scp \
+ --data_path_and_name_and_type data/${dset}/wav.scp,speech,sound \
+ --allow_variable_data_keys true \
+ --sv_train_config ${sv_config} \
+ --sv_model_file ${inference_sv_model} \
+ --output_dir "${_logdir}"/output.JOB \
+ --num_workers 1
+
+ for f in xvector.ark; do
+ if [ -f "${_logdir}/output.1/${f}" ]; then
+ for i in $(seq "${_nj}"); do
+ echo "${_logdir}/output.${i}/${f}"
+ done > "${_dir}/${f}.flist"
+ fi
+ done
+ done
+
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+
+ # compute eer and minDCF results
+ echo "stage 2: computing EER and minDCF"
+ asr_exp=${exp_dir}/exp/${model_dir}
+ _dir="${asr_exp}/${inference_tag}/${inference_sv_model}"
+ mkdir -p ${_dir}/score
+ cp ${_dir}/eval_enroll/xvector.ark.flist ${_dir}/score/spk2xvec.flist
+ cp ${_dir}/eval_test/xvector.ark.flist ${_dir}/score/utt2xvec.flist
+ cp ${feats_dir}/data/eval_test/trials/trials.lst.speech ${_dir}/score/trials
+ python sid/calc_trial_scores.py --no_pbar ${_dir}/score ${_dir}/score/trials ${_dir}/score/trials.cos
+ python sid/compute_eer.py ${_dir}/score/trials ${_dir}/score/trials.cos
+ python sid/compute_min_dcf.py --c-miss 10 --p-target 0.01 \
+ ${_dir}/score/trials.cos ${_dir}/score/trials 2>/dev/null
+fi
\ No newline at end of file
diff --git a/egs/cnceleb/resnet34/sid/calc_trial_scores.py b/egs/cnceleb/resnet34/sid/calc_trial_scores.py
new file mode 100644
index 0000000..12bcde9
--- /dev/null
+++ b/egs/cnceleb/resnet34/sid/calc_trial_scores.py
@@ -0,0 +1,160 @@
+from __future__ import print_function
+import numpy as np
+import os
+import kaldiio
+from multiprocessing import Pool
+import argparse
+from tqdm import tqdm
+import math
+
+
+class MultiProcessRunner:
+ def __init__(self, fn):
+ self.args = None
+ self.process = fn
+
+ def run(self):
+ parser = argparse.ArgumentParser("")
+ # Task-independent options
+ parser.add_argument("--nj", type=int, default=16)
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("--no_pbar", action="store_true", default=False)
+ parser.add_argument("--verbose", action="store_ture", default=False)
+
+ task_list, args = self.prepare(parser)
+ result_list = self.pool_run(task_list, args)
+ self.post(result_list, args)
+
+ def prepare(self, parser):
+ raise NotImplementedError("Please implement the prepare function.")
+
+ def post(self, result_list, args):
+ raise NotImplementedError("Please implement the post function.")
+
+ def pool_run(self, tasks, args):
+ results = []
+ if args.debug:
+ one_result = self.process(tasks[0])
+ results.append(one_result)
+ else:
+ pool = Pool(args.nj)
+ for one_result in tqdm(pool.imap(self.process, tasks), total=len(tasks), ascii=True, disable=args.no_pbar):
+ results.append(one_result)
+ pool.close()
+
+ return results
+
+
+class MultiProcessRunnerV2:
+ def __init__(self, fn):
+ self.args = None
+ self.process = fn
+
+ def run(self):
+ parser = argparse.ArgumentParser("")
+ # Task-independent options
+ parser.add_argument("--nj", type=int, default=16)
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("--no_pbar", action="store_true", default=False)
+ parser.add_argument("--verbose", action="store_true", default=False)
+
+ task_list, args = self.prepare(parser)
+ chunk_size = int(math.ceil(float(len(task_list)) / args.nj))
+ if args.verbose:
+ print("Split {} tasks into {} sub-tasks with chunk_size {}".format(len(task_list), args.nj, chunk_size))
+ subtask_list = [task_list[i*chunk_size: (i+1)*chunk_size] for i in range(args.nj)]
+ result_list = self.pool_run(subtask_list, args)
+ self.post(result_list, args)
+
+ def prepare(self, parser):
+ raise NotImplementedError("Please implement the prepare function.")
+
+ def post(self, result_list, args):
+ raise NotImplementedError("Please implement the post function.")
+
+ def pool_run(self, tasks, args):
+ results = []
+ if args.debug:
+ one_result = self.process(tasks[0])
+ results.append(one_result)
+ else:
+ pool = Pool(args.nj)
+ for one_result in tqdm(pool.imap(self.process, tasks), total=len(tasks), ascii=True, disable=args.no_pbar):
+ results.append(one_result)
+ pool.close()
+
+ return results
+
+
+class MultiProcessRunnerV3(MultiProcessRunnerV2):
+ def run(self):
+ parser = argparse.ArgumentParser("")
+ # Task-independent options
+ parser.add_argument("--nj", type=int, default=16)
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("--no_pbar", action="store_true", default=False)
+ parser.add_argument("--verbose", action="store_true", default=False)
+ parser.add_argument("--sr", type=int, default=16000)
+
+ task_list, shared_param, args = self.prepare(parser)
+ chunk_size = int(math.ceil(float(len(task_list)) / args.nj))
+ if args.verbose:
+ print("Split {} tasks into {} sub-tasks with chunk_size {}".format(len(task_list), args.nj, chunk_size))
+ subtask_list = [(i, task_list[i * chunk_size: (i + 1) * chunk_size], shared_param, args)
+ for i in range(args.nj)]
+ result_list = self.pool_run(subtask_list, args)
+ self.post(result_list, args)
+
+
+
+class MyRunner(MultiProcessRunnerV3):
+ def prepare(self, parser):
+ assert isinstance(parser, argparse.ArgumentParser)
+ parser.add_argument("enroll_dir", type=str)
+ parser.add_argument("trial_in", type=str)
+ parser.add_argument("trial_out", type=str)
+ args = parser.parse_args()
+
+ if not os.path.exists(os.path.dirname(args.trial_out)):
+ os.makedirs(os.path.dirname(args.trial_out))
+
+ flist_path = os.path.join(args.enroll_dir, "spk2xvec.flist")
+ spk2xvec = {}
+ for _path in open(flist_path, "r"):
+ for key, value in kaldiio.load_ark(_path.strip()):
+ if "-enroll" in key:
+ key = key.replace("-enroll", "")
+ spk2xvec[key] = value
+
+ flist_path = os.path.join(args.enroll_dir, "utt2xvec.flist")
+ utt2xvec = {}
+ for _path in open(flist_path, 'r'):
+ for key, value in kaldiio.load_ark(_path.strip()):
+ utt2xvec[key] = value
+
+ task_list = [one_line.strip().split(" ") for one_line in open(args.trial_in, "rt")]
+ return task_list, [spk2xvec, utt2xvec], args
+
+ def post(self, results_list, args):
+ with open(args.trial_out, "wt") as fs:
+ for results in results_list:
+ for one_item in results:
+ fs.write(one_item+"\n")
+
+
+def process(task_args):
+ task_id, task_list, [spk2xvec, utt2xvec], args = task_args
+ results = []
+ for spk, utt, _ in task_list:
+ xvec = utt2xvec[utt]
+ normed_x = xvec / np.linalg.norm(xvec)
+ normed_y = spk2xvec[spk] / np.linalg.norm(spk2xvec[spk])
+ score = np.sum(normed_x * normed_y)
+ results.append("{} {} {:.5f}".format(spk, utt, score))
+
+ return results
+
+
+if __name__ == '__main__':
+ my_runner = MyRunner(process)
+ my_runner.run()
diff --git a/egs/cnceleb/resnet34/sid/compute_eer.py b/egs/cnceleb/resnet34/sid/compute_eer.py
new file mode 100644
index 0000000..0ca9280
--- /dev/null
+++ b/egs/cnceleb/resnet34/sid/compute_eer.py
@@ -0,0 +1,54 @@
+import numpy as np
+from sklearn.metrics import roc_curve
+import argparse
+
+
+def compute_eer(label, pred, positive_label=1):
+ """
+ Python compute equal error rate (eer)
+ ONLY tested on binary classification
+
+ :param label: ground-truth label, should be a 1-d list or np.array, each element represents the ground-truth label of one sample
+ :param pred: model prediction, should be a 1-d list or np.array, each element represents the model prediction of one sample
+ :param positive_label: the class that is viewed as positive class when computing EER
+ :return: equal error rate (EER)
+ """
+
+ # all fpr, tpr, fnr, fnr, threshold are lists (in the format of np.array)
+ fpr, tpr, threshold = roc_curve(label, pred, pos_label=positive_label)
+ fnr = 1 - tpr
+
+ # the threshold of fnr == fpr
+ eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
+
+ # theoretically eer from fpr and eer from fnr should be identical but they can be slightly differ in reality
+ eer_1 = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
+ eer_2 = fnr[np.nanargmin(np.absolute((fnr - fpr)))]
+
+ # return the mean of eer from fpr and from fnr
+ eer = (eer_1 + eer_2) / 2
+ return eer, eer_threshold
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("trials", help="trial list")
+ parser.add_argument("scores", help="score file, normalized to [0, 1]")
+ args = parser.parse_args()
+
+ labels = []
+ for one_line in open(args.trials, "r"):
+ labels.append(one_line.strip().rsplit(" ", 1)[-1] == "target")
+ labels = np.array(labels, dtype=int)
+
+ scores = []
+ for one_line in open(args.scores, "r"):
+ scores.append(float(one_line.strip().rsplit(" ", 1)[-1]))
+ scores = np.array(scores, dtype=float)
+
+ eer, threshold = compute_eer(labels, scores)
+ print("EER is {:.4f} at threshold {:.4f}".format(eer * 100.0, threshold))
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/egs/cnceleb/resnet34/sid/compute_min_dcf.py b/egs/cnceleb/resnet34/sid/compute_min_dcf.py
new file mode 100644
index 0000000..f2a6682
--- /dev/null
+++ b/egs/cnceleb/resnet34/sid/compute_min_dcf.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# Copyright 2018 David Snyder
+# Apache 2.0
+
+# This script computes the minimum detection cost function, which is a common
+# error metric used in speaker recognition. Compared to equal error-rate,
+# which assigns equal weight to false negatives and false positives, this
+# error-rate is usually used to assess performance in settings where achieving
+# a low false positive rate is more important than achieving a low false
+# negative rate. See the NIST 2016 Speaker Recognition Evaluation Plan at
+# https://www.nist.gov/sites/default/files/documents/2016/10/07/sre16_eval_plan_v1.3.pdf
+# for more details about the metric.
+from __future__ import print_function
+from operator import itemgetter
+import sys, argparse, os
+
+
+def GetArgs():
+ parser = argparse.ArgumentParser(description="Compute the minimum "
+ "detection cost function along with the threshold at which it occurs. "
+ "Usage: sid/compute_min_dcf.py [options...] <scores-file> "
+ "<trials-file> "
+ "E.g., sid/compute_min_dcf.py --p-target 0.01 --c-miss 1 --c-fa 1 "
+ "exp/scores/trials data/test/trials",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--p-target', type=float, dest="p_target",
+ default=0.01,
+ help='The prior probability of the target speaker in a trial.')
+ parser.add_argument('--c-miss', type=float, dest="c_miss", default=1,
+ help='Cost of a missed detection. This is usually not changed.')
+ parser.add_argument('--c-fa', type=float, dest="c_fa", default=1,
+ help='Cost of a spurious detection. This is usually not changed.')
+ parser.add_argument("scores_filename",
+ help="Input scores file, with columns of the form "
+ "<utt1> <utt2> <score>")
+ parser.add_argument("trials_filename",
+ help="Input trials file, with columns of the form "
+ "<utt1> <utt2> <target/nontarget>")
+ sys.stderr.write(' '.join(sys.argv) + "\n")
+ args = parser.parse_args()
+ args = CheckArgs(args)
+ return args
+
+
+def CheckArgs(args):
+ if args.c_fa <= 0:
+ raise Exception("--c-fa must be greater than 0")
+ if args.c_miss <= 0:
+ raise Exception("--c-miss must be greater than 0")
+ if args.p_target <= 0 or args.p_target >= 1:
+ raise Exception("--p-target must be greater than 0 and less than 1")
+ return args
+
+
+# Creates a list of false-negative rates, a list of false-positive rates
+# and a list of decision thresholds that give those error-rates.
+def ComputeErrorRates(scores, labels):
+
+ # Sort the scores from smallest to largest, and also get the corresponding
+ # indexes of the sorted scores. We will treat the sorted scores as the
+ # thresholds at which the the error-rates are evaluated.
+ sorted_indexes, thresholds = zip(*sorted(
+ [(index, threshold) for index, threshold in enumerate(scores)],
+ key=itemgetter(1)))
+ labels = [labels[i] for i in sorted_indexes]
+ fns = []
+ tns = []
+
+ # At the end of this loop, fns[i] is the number of errors made by
+ # incorrectly rejecting scores less than thresholds[i]. And, tns[i]
+ # is the total number of times that we have correctly rejected scores
+ # less than thresholds[i].
+ for i in range(0, len(labels)):
+ if i == 0:
+ fns.append(labels[i])
+ tns.append(1 - labels[i])
+ else:
+ fns.append(fns[i-1] + labels[i])
+ tns.append(tns[i-1] + 1 - labels[i])
+ positives = sum(labels)
+ negatives = len(labels) - positives
+
+ # Now divide the false negatives by the total number of
+ # positives to obtain the false negative rates across
+ # all thresholds
+ fnrs = [fn / float(positives) for fn in fns]
+
+ # Divide the true negatives by the total number of
+ # negatives to get the true negative rate. Subtract these
+ # quantities from 1 to get the false positive rates.
+ fprs = [1 - tn / float(negatives) for tn in tns]
+ return fnrs, fprs, thresholds
+
+
+# Computes the minimum of the detection cost function. The comments refer to
+# equations in Section 3 of the NIST 2016 Speaker Recognition Evaluation Plan.
+def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):
+ min_c_det = float("inf")
+ min_c_det_threshold = thresholds[0]
+ for i in range(0, len(fnrs)):
+ # See Equation (2). it is a weighted sum of false negative
+ # and false positive errors.
+ c_det = c_miss * fnrs[i] * p_target + c_fa * fprs[i] * (1 - p_target)
+ if c_det < min_c_det:
+ min_c_det = c_det
+ min_c_det_threshold = thresholds[i]
+ # See Equations (3) and (4). Now we normalize the cost.
+ c_def = min(c_miss * p_target, c_fa * (1 - p_target))
+ min_dcf = min_c_det / c_def
+ return min_dcf, min_c_det_threshold
+
+def main():
+ args = GetArgs()
+ scores_file = open(args.scores_filename, 'r').readlines()
+ trials_file = open(args.trials_filename, 'r').readlines()
+ c_miss = args.c_miss
+ c_fa = args.c_fa
+ p_target = args.p_target
+
+ scores = []
+ labels = []
+
+ trials = {}
+ for line in trials_file:
+ utt1, utt2, target = line.rstrip().split()
+ trial = utt1 + " " + utt2
+ trials[trial] = target
+
+ for line in scores_file:
+ utt1, utt2, score = line.rstrip().split()
+ trial = utt1 + " " + utt2
+ if trial in trials:
+ scores.append(float(score))
+ if trials[trial] == "target":
+ labels.append(1)
+ else:
+ labels.append(0)
+ else:
+ raise Exception("Missing entry for " + utt1 + " and " + utt2
+ + " " + args.scores_filename)
+
+ fnrs, fprs, thresholds = ComputeErrorRates(scores, labels)
+ mindcf, threshold = ComputeMinDcf(fnrs, fprs, thresholds, p_target,
+ c_miss, c_fa)
+ # sys.stdout.write("{0:.4f}\n".format(mindcf))
+ sys.stdout.write("minDCF is {0:.4f} at threshold {1:.4f} (p-target={2}, c-miss={3}, "
+ "c-fa={4})\n".format(mindcf, threshold, p_target,c_miss, c_fa))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/egs/cnceleb/resnet34/sid/speaker_verification.py b/egs/cnceleb/resnet34/sid/speaker_verification.py
new file mode 100644
index 0000000..53c0d25
--- /dev/null
+++ b/egs/cnceleb/resnet34/sid/speaker_verification.py
@@ -0,0 +1,167 @@
+from __future__ import print_function
+import numpy as np
+import os
+import kaldiio
+from multiprocessing import Pool
+import argparse
+from tqdm import tqdm
+import math
+from funasr.utils.types import str2triple_str
+import logging
+from typing import List, Union, Tuple, Sequence
+from funasr.bin.sv_inference import inference_modelscope
+import soundfile
+import torch
+
+
+class MultiProcessRunner:
+ def __init__(self, fn):
+ self.process = fn
+
+ def run(self):
+ parser = argparse.ArgumentParser("")
+ # Task-independent options
+ parser.add_argument("--njobs", type=int, default=16)
+ parser.add_argument("--debug", action="store_true", default=False)
+ parser.add_argument("--no_pbar", action="store_true", default=False)
+ parser.add_argument("--verbose", action="store_true", default=False)
+ parser.add_argument("--log_level", type=str, default="INFO")
+ parser.add_argument("--sr", type=int, default=16000)
+
+ task_list, shared_param, args = self.prepare(parser)
+ chunk_size = int(math.ceil(float(len(task_list)) / args.njobs))
+ if args.verbose:
+ print("Split {} tasks into {} sub-tasks with chunk_size {}".format(len(task_list), args.njobs, chunk_size))
+ subtask_list = [(i, task_list[i * chunk_size: (i + 1) * chunk_size], shared_param, args)
+ for i in range(args.njobs)]
+ result_list = self.pool_run(subtask_list, args)
+ self.post(result_list, args)
+
+ def prepare(self, parser: argparse.ArgumentParser):
+ raise NotImplementedError("Please implement the prepare function.")
+
+ def post(self, results_list: list, args: argparse.Namespace):
+ raise NotImplementedError("Please implement the post function.")
+
+ def pool_run(self, tasks: list, args: argparse.Namespace):
+ results = []
+ if args.debug:
+ one_result = self.process(tasks[0])
+ results.append(one_result)
+ else:
+ pool = Pool(args.njobs)
+ for one_result in tqdm(pool.imap(self.process, tasks), total=len(tasks), ascii=True, disable=args.no_pbar):
+ results.append(one_result)
+ pool.close()
+
+ return results
+
+
+class MyRunner(MultiProcessRunner):
+ def prepare(self, parser: argparse.ArgumentParser):
+ parser.add_argument(
+ "--gpu_inference",
+ type=bool,
+ default=False
+ )
+ parser.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=True,
+ action="append"
+ )
+ parser.add_argument(
+ "--gpu_devices",
+ type=lambda devices: devices.split(","),
+ default=None,
+ )
+ args = parser.parse_args()
+
+ logging.basicConfig(
+ level=args.log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if args.gpu_inference and (args.gpu_devices is None or len(args.gpu_devices) == 0):
+ logging.warning("gpu_inference is set to True, but gpu_devices is not given, use CPU instead.")
+ args.gpu_inference = False
+
+ if args.gpu_inference:
+ args.njobs = args.njobs * len(args.gpu_devices)
+
+ speech_dict = {}
+ ref_speech_dict = {}
+ for _path, _name, _type in args.data_path_and_name_and_type:
+ if _name == "speech":
+ speech_dict = self.read_data_path(_path)
+ elif _name == "ref_speech":
+ ref_speech_dict = self.read_data_path(_path)
+
+ task_list, args.njobs = self.get_key_list(args.data_path_and_name_and_type, args.njobs)
+
+ return task_list, [speech_dict, ref_speech_dict], args
+
+ def read_data_path(self, file_path):
+ results = {}
+ for line in open(file_path, "r"):
+ key, path = line.strip().split(" ", 1)
+ results[key] = path
+
+ return results
+
+ def get_key_list(
+ self,
+ data_path_and_name_and_type: Sequence[Tuple[str, str, str]],
+ njobs: int
+ ):
+ first_data = data_path_and_name_and_type[0]
+ content = open(first_data[0], "r").readlines()
+ line_number = len(content)
+ njobs = min(njobs, line_number)
+ logging.warning("njobs is reduced to {}, since only {} lines exist in {}".format(
+ njobs, line_number, first_data[0],
+ ))
+ key_list = [line.strip().split(" ", 1)[0] for line in content]
+ return key_list, njobs
+
+ def post(self, results_list: list, args: argparse.Namespace):
+ for results in results_list:
+ for key, value in results:
+ logging.info("{} {}".format(key, value))
+
+
+def process(task_args):
+ task_id, key_list, [speech_dict, ref_speech_dict], args = task_args
+ if args.gpu_inference:
+ device = args.gpu_devices[task_id % len(args.gpu_devices)]
+ torch.cuda.set_device("cuda:".format(device))
+ inference_func = inference_modelscope(
+ output_dir=None,
+ batch_size=1,
+ dtype="float32",
+ ngpu=1 if args.gpu_inference else 0,
+ seed=0,
+ num_workers=0,
+ log_level=logging.INFO,
+ key_file=None,
+ sv_train_config="sv.yaml",
+ sv_model_file="sv.pb",
+ model_tag=None,
+ allow_variable_data_keys=True,
+ streaming=False,
+ embedding_node="resnet1_dense",
+ sv_threshold=0.9465,
+ )
+ results = {}
+ for key in key_list:
+ speech = soundfile.read(speech_dict[key])[0]
+ ref_speech = soundfile.read(ref_speech_dict[key])[0]
+ ret = inference_func(None, (speech, ref_speech))
+ results[key] = ret["value"]
+
+ return results
+
+
+if __name__ == '__main__':
+ my_runner = MyRunner(process)
+ my_runner.run()
diff --git a/egs/cnceleb/resnet34/sv.yaml b/egs/cnceleb/resnet34/sv.yaml
new file mode 100644
index 0000000..adf3382
--- /dev/null
+++ b/egs/cnceleb/resnet34/sv.yaml
@@ -0,0 +1,3515 @@
+# network architecture
+input_size: null
+specaug: null
+normalize: null
+init: null
+model: espnet
+model_conf: {}
+# encoder related
+encoder: resnet34
+encoder_conf:
+ use_head_conv: true
+ batchnorm_momentum: 0.5
+ use_head_maxpool: false
+ num_nodes_pooling_layer: 256
+ layers_in_block:
+ - 3
+ - 4
+ - 6
+ - 3
+ filters_in_block:
+ - 32
+ - 64
+ - 128
+ - 256
+
+# Pooling related
+pooling_type: statistic
+
+# decoder related
+decoder: dense
+decoder_conf:
+ num_nodes_resnet1: 256
+ num_nodes_last_layer: 256
+ batchnorm_momentum: 0.5
+
+frontend: wav_frontend
+frontend_conf:
+ fs: 16000
+ window: povey
+ n_mels: 80
+ frame_length: 25
+ frame_shift: 10
+ filter_length_min: -1
+ filter_length_max: -1
+ lfr_m: 1
+ lfr_n: 1
+
+log_interval: 50
+
+token_list:
+- R0003_M0046_S001
+- R0003_M0046_S002
+- R0003_M0046_S003
+- R0003_M0046_S004
+- R0003_M0047_S001
+- R0003_M0047_S002
+- R0003_M0047_S003
+- R0003_M0047_S004
+- R0004_M0012_S001
+- R0004_M0012_S002
+- R0004_M0012_S003
+- R0004_M0012_S004
+- R0004_M0017_S001
+- R0004_M0017_S002
+- R0004_M0017_S003
+- R0004_M0017_S004
+- R0004_M0018_S001
+- R0004_M0018_S002
+- R0004_M0018_S003
+- R0004_M0018_S004
+- R0004_M0022_S001
+- R0004_M0022_S002
+- R0004_M0022_S003
+- R0004_M0022_S004
+- R0004_M0025_S001
+- R0004_M0025_S002
+- R0004_M0025_S003
+- R0004_M0025_S004
+- R0004_M0027_S001
+- R0004_M0027_S002
+- R0004_M0027_S003
+- R0004_M0027_S004
+- R0004_M0028_S001
+- R0004_M0028_S002
+- R0004_M0028_S003
+- R0004_M0028_S004
+- R0005_M0035_S001
+- R0005_M0035_S002
+- R0005_M0035_S003
+- R0005_M0035_S004
+- R0005_M0040_S001
+- R0005_M0040_S002
+- R0005_M0040_S003
+- R0005_M0040_S004
+- R0005_M0041_S001
+- R0005_M0041_S002
+- R0005_M0041_S003
+- R0005_M0041_S004
+- R0008_M0050_S001
+- R0008_M0050_S002
+- R0008_M0050_S003
+- R0008_M0050_S004
+- R0008_M0051_S001
+- R0008_M0051_S002
+- R0008_M0051_S003
+- R0008_M0051_S004
+- R0008_M0052_S001
+- R0008_M0052_S002
+- R0008_M0052_S003
+- R0008_M0052_S004
+- R0008_M0053_S001
+- R0008_M0053_S002
+- R0008_M0053_S003
+- R0008_M0053_S004
+- R0008_M0054_S001
+- R0008_M0054_S002
+- R0008_M0054_S003
+- R0008_M0054_S004
+- R0008_M0055_S001
+- R0008_M0055_S002
+- R0008_M0055_S003
+- R0008_M0055_S004
+- R0008_M0056_S001
+- R0008_M0056_S002
+- R0008_M0056_S003
+- R0008_M0056_S004
+- R0008_M0057_S001
+- R0008_M0057_S002
+- R0008_M0057_S003
+- R0008_M0057_S004
+- R0008_M0058_S001
+- R0008_M0058_S002
+- R0008_M0058_S003
+- R0008_M0058_S004
+- R0008_M0060_S001
+- R0008_M0060_S002
+- R0008_M0060_S003
+- R0008_M0060_S004
+- R0008_M0063_S001
+- R0008_M0063_S002
+- R0008_M0063_S003
+- R0008_M0063_S004
+- R0008_M0064_S001
+- R0008_M0064_S002
+- R0008_M0064_S003
+- R0008_M0064_S004
+- R0008_M0069_S001
+- R0008_M0069_S002
+- R0008_M0069_S003
+- R0008_M0069_S004
+- R0008_M0070_S001
+- R0008_M0070_S002
+- R0008_M0070_S003
+- R0008_M0070_S004
+- R0008_M0071_S001
+- R0008_M0071_S002
+- R0008_M0071_S003
+- R0008_M0071_S004
+- R0008_M0072_S001
+- R0008_M0072_S002
+- R0008_M0072_S003
+- R0008_M0072_S004
+- R0008_M0073_S001
+- R0008_M0073_S002
+- R0008_M0073_S003
+- R0008_M0073_S004
+- R0008_M0074_S001
+- R0008_M0074_S002
+- R0008_M0074_S003
+- R0008_M0074_S004
+- R0008_M0075_S001
+- R0008_M0075_S002
+- R0008_M0075_S003
+- R0008_M0075_S004
+- R0008_M0076_S001
+- R0008_M0076_S002
+- R0008_M0076_S003
+- R0008_M0076_S004
+- R0014_M0077_S001
+- R0014_M0077_S002
+- R0014_M0077_S003
+- R0014_M0077_S004
+- R0014_M0078_S001
+- R0014_M0078_S002
+- R0014_M0078_S003
+- R0014_M0078_S004
+- R0014_M0079_S001
+- R0014_M0079_S002
+- R0014_M0079_S003
+- R0014_M0079_S004
+- R0014_M0080_S001
+- R0014_M0080_S002
+- R0014_M0080_S003
+- R0014_M0080_S004
+- R0014_M0083_S001
+- R0014_M0083_S002
+- R0014_M0083_S003
+- R0014_M0083_S004
+- R0014_M0084_S001
+- R0014_M0084_S002
+- R0014_M0084_S003
+- R0014_M0084_S004
+- R0014_M0085_S001
+- R0014_M0085_S002
+- R0014_M0085_S003
+- R0014_M0085_S004
+- R0014_M0086_S001
+- R0014_M0086_S002
+- R0014_M0086_S003
+- R0014_M0086_S004
+- R0014_M0087_S001
+- R0014_M0087_S002
+- R0014_M0087_S003
+- R0014_M0087_S004
+- R0014_M0089_S001
+- R0014_M0089_S002
+- R0014_M0089_S003
+- R0014_M0089_S004
+- R0014_M0090_S001
+- R0014_M0090_S002
+- R0014_M0090_S003
+- R0014_M0090_S004
+- R0014_M0091_S001
+- R0014_M0091_S002
+- R0014_M0091_S003
+- R0014_M0091_S004
+- R0014_M0092_S001
+- R0014_M0092_S002
+- R0014_M0092_S003
+- R0014_M0092_S004
+- R0014_M0093_S001
+- R0014_M0093_S002
+- R0014_M0093_S003
+- R0014_M0093_S004
+- R0014_M0094_S001
+- R0014_M0094_S002
+- R0014_M0094_S003
+- R0014_M0094_S004
+- R0014_M0097_S001
+- R0014_M0097_S002
+- R0014_M0097_S003
+- R0014_M0097_S004
+- R0014_M0098_S001
+- R0014_M0098_S002
+- R0014_M0098_S003
+- R0014_M0098_S004
+- R0014_M0099_S001
+- R0014_M0099_S002
+- R0014_M0099_S003
+- R0014_M0099_S004
+- R0014_M0100_S001
+- R0014_M0100_S002
+- R0014_M0100_S003
+- R0014_M0100_S004
+- R0014_M0101_S001
+- R0014_M0101_S002
+- R0014_M0101_S003
+- R0014_M0101_S004
+- R0014_M0102_S001
+- R0014_M0102_S002
+- R0014_M0102_S003
+- R0014_M0102_S004
+- R0014_M0105_S001
+- R0014_M0105_S002
+- R0014_M0105_S003
+- R0014_M0105_S004
+- R0014_M0106_S001
+- R0014_M0106_S002
+- R0014_M0106_S003
+- R0014_M0106_S004
+- R0014_M0107_S001
+- R0014_M0107_S002
+- R0014_M0107_S003
+- R0014_M0107_S004
+- R0014_M0108_S001
+- R0014_M0108_S002
+- R0014_M0108_S003
+- R0014_M0108_S004
+- R0014_M0109_S001
+- R0014_M0109_S002
+- R0014_M0109_S003
+- R0014_M0109_S004
+- R0014_M0110_S001
+- R0014_M0110_S002
+- R0014_M0110_S003
+- R0014_M0110_S004
+- R0014_M0111_S001
+- R0014_M0111_S002
+- R0014_M0111_S003
+- R0014_M0111_S004
+- R0014_M0112_S001
+- R0014_M0112_S002
+- R0014_M0112_S003
+- R0014_M0112_S004
+- R0015_M0115_S001
+- R0015_M0115_S002
+- R0015_M0115_S003
+- R0015_M0115_S004
+- R0015_M0116_S001
+- R0015_M0116_S002
+- R0015_M0116_S003
+- R0015_M0116_S004
+- R0015_M0117_S001
+- R0015_M0117_S002
+- R0015_M0117_S003
+- R0015_M0117_S004
+- R0015_M0118_S001
+- R0015_M0118_S002
+- R0015_M0118_S003
+- R0015_M0118_S004
+- R0015_M0119_S001
+- R0015_M0119_S002
+- R0015_M0119_S003
+- R0015_M0119_S004
+- R0015_M0120_S001
+- R0015_M0120_S002
+- R0015_M0120_S003
+- R0015_M0120_S004
+- R0015_M0121_S001
+- R0015_M0121_S002
+- R0015_M0121_S003
+- R0015_M0121_S004
+- R0015_M0122_S001
+- R0015_M0122_S002
+- R0015_M0122_S003
+- R0015_M0122_S004
+- R0015_M0123_S001
+- R0015_M0123_S002
+- R0015_M0123_S003
+- R0015_M0123_S004
+- R0015_M0124_S001
+- R0015_M0124_S002
+- R0015_M0124_S003
+- R0015_M0124_S004
+- R0015_M0125_S001
+- R0015_M0125_S002
+- R0015_M0125_S003
+- R0015_M0125_S004
+- R0015_M0126_S001
+- R0015_M0126_S002
+- R0015_M0126_S003
+- R0015_M0126_S004
+- R0015_M0129_S001
+- R0015_M0129_S002
+- R0015_M0129_S003
+- R0015_M0129_S004
+- R0015_M0130_S001
+- R0015_M0130_S002
+- R0015_M0130_S003
+- R0015_M0130_S004
+- R0015_M0133_S001
+- R0015_M0133_S002
+- R0015_M0133_S003
+- R0015_M0133_S004
+- R0015_M0134_S001
+- R0015_M0134_S002
+- R0015_M0134_S003
+- R0015_M0134_S004
+- R0015_M0135_S001
+- R0015_M0135_S002
+- R0015_M0135_S003
+- R0015_M0135_S004
+- R0015_M0138_S001
+- R0015_M0138_S002
+- R0015_M0138_S003
+- R0015_M0138_S004
+- R0015_M0139_S001
+- R0015_M0139_S002
+- R0015_M0139_S003
+- R0015_M0139_S004
+- R0015_M0140_S001
+- R0015_M0140_S002
+- R0015_M0140_S003
+- R0015_M0140_S004
+- R0015_M0141_S001
+- R0015_M0141_S002
+- R0015_M0141_S003
+- R0015_M0141_S004
+- R0015_M0142_S001
+- R0015_M0142_S002
+- R0015_M0142_S003
+- R0015_M0142_S004
+- R0015_M0143_S001
+- R0015_M0143_S002
+- R0015_M0143_S003
+- R0015_M0143_S004
+- R0015_M0144_S001
+- R0015_M0144_S002
+- R0015_M0144_S003
+- R0015_M0144_S004
+- R0015_M0145_S001
+- R0015_M0145_S002
+- R0015_M0145_S003
+- R0015_M0145_S004
+- R0015_M0146_S001
+- R0015_M0146_S002
+- R0015_M0146_S003
+- R0015_M0146_S004
+- R0015_M0147_S001
+- R0015_M0147_S002
+- R0015_M0147_S003
+- R0015_M0147_S004
+- R0015_M0148_S001
+- R0015_M0148_S002
+- R0015_M0148_S003
+- R0015_M0148_S004
+- R0015_M0149_S001
+- R0015_M0149_S002
+- R0015_M0149_S003
+- R0015_M0149_S004
+- R0015_M0150_S001
+- R0015_M0150_S002
+- R0015_M0150_S003
+- R0015_M0150_S004
+- R0015_M0151_S001
+- R0015_M0151_S002
+- R0015_M0151_S003
+- R0015_M0151_S004
+- R0015_M0152_S002
+- R0015_M0152_S004
+- R0015_M0153_S001
+- R0015_M0153_S002
+- R0015_M0153_S003
+- R0015_M0153_S004
+- R0015_M0158_S001
+- R0015_M0158_S002
+- R0015_M0158_S003
+- R0015_M0158_S004
+- R0015_M0159_S001
+- R0015_M0159_S002
+- R0015_M0159_S003
+- R0015_M0159_S004
+- R0015_M0160_S001
+- R0015_M0160_S002
+- R0015_M0160_S003
+- R0015_M0160_S004
+- R0015_M0161_S001
+- R0015_M0161_S002
+- R0015_M0161_S003
+- R0015_M0161_S004
+- R0015_M0162_S001
+- R0015_M0162_S002
+- R0015_M0162_S003
+- R0015_M0162_S004
+- R0015_M0163_S001
+- R0015_M0163_S002
+- R0015_M0163_S003
+- R0015_M0163_S004
+- R0020_M0166_S001
+- R0020_M0166_S002
+- R0020_M0166_S003
+- R0020_M0166_S004
+- R0020_M0167_S001
+- R0020_M0167_S002
+- R0020_M0167_S003
+- R0020_M0167_S004
+- R0020_M0168_S001
+- R0020_M0168_S002
+- R0020_M0168_S003
+- R0020_M0168_S004
+- R0020_M0169_S001
+- R0020_M0169_S002
+- R0020_M0169_S003
+- R0020_M0169_S004
+- R0020_M0170_S001
+- R0020_M0170_S002
+- R0020_M0170_S003
+- R0020_M0170_S004
+- R0020_M0171_S001
+- R0020_M0171_S002
+- R0020_M0171_S003
+- R0020_M0171_S004
+- R0020_M0172_S001
+- R0020_M0172_S002
+- R0020_M0172_S003
+- R0020_M0172_S004
+- R0020_M0175_S001
+- R0020_M0175_S002
+- R0020_M0175_S003
+- R0020_M0175_S004
+- R0020_M0176_S001
+- R0020_M0176_S002
+- R0020_M0176_S003
+- R0020_M0176_S004
+- R0020_M0178_S001
+- R0020_M0178_S002
+- R0020_M0178_S003
+- R0020_M0178_S004
+- R0020_M0179_S001
+- R0020_M0179_S002
+- R0020_M0179_S003
+- R0020_M0179_S004
+- R0020_M0183_S001
+- R0020_M0183_S002
+- R0020_M0183_S003
+- R0020_M0183_S004
+- R0020_M0184_S001
+- R0020_M0184_S002
+- R0020_M0184_S003
+- R0020_M0184_S004
+- R0020_M0185_S001
+- R0020_M0185_S002
+- R0020_M0185_S003
+- R0020_M0185_S004
+- R0020_M0186_S001
+- R0020_M0186_S002
+- R0020_M0186_S003
+- R0020_M0186_S004
+- R0020_M0190_S001
+- R0020_M0190_S002
+- R0020_M0190_S003
+- R0020_M0190_S004
+- R0020_M0191_S001
+- R0020_M0191_S002
+- R0020_M0191_S003
+- R0020_M0191_S004
+- R0020_M0192_S001
+- R0020_M0192_S002
+- R0020_M0192_S003
+- R0020_M0192_S004
+- R0020_M0193_S001
+- R0020_M0193_S002
+- R0020_M0193_S003
+- R0020_M0193_S004
+- R1019_M1928_S001
+- R1019_M1928_S002
+- R1019_M1929_S001
+- R1019_M1929_S002
+- R1019_M1945_S001
+- R1019_M1945_S002
+- R1019_M1945_S003
+- R1019_M1945_S004
+- R1019_M1946_S001
+- R1019_M1946_S002
+- R1019_M1946_S003
+- R1019_M1946_S004
+- R1019_M1948_S001
+- R1019_M1948_S002
+- R1019_M1949_S001
+- R1019_M1949_S002
+- R1019_M1950_S001
+- R1019_M1950_S002
+- R1019_M1953_S001
+- R1019_M1953_S002
+- R1019_M1954_S001
+- R1019_M1954_S002
+- R1019_M1956_S001
+- R1019_M1956_S002
+- R1019_M1960_S001
+- R1019_M1960_S002
+- R1019_M1963_S001
+- R1019_M1963_S002
+- R1019_M1965_S001
+- R1019_M1965_S002
+- R1019_M1967_S001
+- R1019_M1967_S002
+- R1019_M1969_S001
+- R1019_M1969_S002
+- R1019_M1971_S001
+- R1019_M1971_S002
+- R1019_M1972_S001
+- R1019_M1972_S002
+- R1019_M1975_S001
+- R1019_M1975_S002
+- R1019_M1977_S001
+- R1019_M1977_S002
+- R1019_M1979_S001
+- R1019_M1979_S002
+- R1019_M1981_S001
+- R1019_M1981_S002
+- R1019_M1987_S001
+- R1019_M1987_S002
+- R1019_M1989_S001
+- R1019_M1989_S002
+- R1019_M1993_S001
+- R1019_M1993_S002
+- R1019_M1995_S001
+- R1019_M1995_S002
+- R1019_M1997_S001
+- R1019_M1997_S002
+- R1019_M1998_S001
+- R1019_M1998_S002
+- R1019_M4004_S001
+- R1019_M4004_S002
+- R1019_M4005_S001
+- R1019_M4005_S002
+- R1019_M4007_S001
+- R1019_M4007_S002
+- R1019_M4011_S001
+- R1019_M4011_S002
+- R1019_M4013_S001
+- R1019_M4013_S002
+- R1019_M4015_S001
+- R1019_M4015_S002
+- R1019_M4017_S001
+- R1019_M4017_S002
+- R1019_M4020_S001
+- R1019_M4020_S002
+- R1019_M4021_S001
+- R1019_M4021_S002
+- R1019_M4026_S001
+- R1019_M4026_S002
+- R1019_M4028_S001
+- R1019_M4028_S002
+- R1019_M4030_S001
+- R1019_M4030_S002
+- R1019_M4031_S001
+- R1019_M4031_S002
+- R1019_M4031_S003
+- R1019_M4031_S004
+- R1019_M4033_S001
+- R1019_M4033_S002
+- R1019_M4035_S001
+- R1019_M4035_S002
+- R1019_M4039_S001
+- R1019_M4039_S002
+- R1019_M4041_S001
+- R1019_M4041_S002
+- R1019_M4043_S001
+- R1019_M4043_S002
+- R1019_M4051_S001
+- R1019_M4051_S002
+- R1019_M4052_S001
+- R1019_M4052_S002
+- R1019_M4052_S003
+- R1019_M4052_S004
+- R1019_M4055_S001
+- R1019_M4055_S002
+- R1019_M4056_S001
+- R1019_M4056_S002
+- R1019_M4056_S003
+- R1019_M4057_S001
+- R1019_M4057_S002
+- R1019_M4059_S001
+- R1019_M4059_S002
+- R1019_M4063_S001
+- R1019_M4063_S002
+- R1019_M4070_S001
+- R1019_M4070_S002
+- R1019_M4071_S001
+- R1019_M4071_S002
+- R1019_M4075_S001
+- R1019_M4075_S002
+- R1021_M1932_S001
+- R1021_M1932_S002
+- R1021_M1933_S001
+- R1021_M1933_S002
+- R1021_M1935_S001
+- R1021_M1935_S002
+- R1021_M1938_S001
+- R1021_M1938_S002
+- R1021_M1940_S001
+- R1021_M1940_S002
+- R1021_M1942_S001
+- R1021_M1942_S002
+- R1021_M1944_S001
+- R1021_M1944_S002
+- R1021_M1947_S001
+- R1021_M1947_S002
+- R1021_M1959_S001
+- R1021_M1959_S002
+- R1021_M1961_S001
+- R1021_M1961_S002
+- R1021_M1986_S001
+- R1021_M1986_S002
+- R1021_M4012_S001
+- R1021_M4012_S002
+- R1021_M4025_S001
+- R1021_M4025_S002
+- R1021_M4032_S001
+- R1021_M4032_S002
+- R1021_M4032_S003
+- R1021_M4032_S004
+- R1021_M4062_S001
+- R1021_M4062_S002
+- R1021_M4073_S001
+- R1021_M4073_S002
+- R1021_M4078_S001
+- R1021_M4078_S002
+- R1021_M4080_S001
+- R1021_M4080_S002
+- R1021_M4082_S001
+- R1021_M4082_S002
+- R1021_M4083_S001
+- R1021_M4083_S002
+- R2001_M2205_S001
+- R2001_M2205_S002
+- R2001_M2205_S003
+- R2001_M2206_S001
+- R2001_M2206_S002
+- R2001_M2206_S003
+- R2105_M3164_S001
+- R2105_M3164_S002
+- R2105_M3164_S003
+- R2105_M3175_S001
+- R2105_M3175_S002
+- R2105_M3175_S003
+- R2105_M3318_S001
+- R2105_M3318_S002
+- R2105_M3318_S003
+- R2108_M2692_S001
+- R2108_M2692_S002
+- R2108_M2692_S003
+- R2108_M3199_S001
+- R2108_M3199_S002
+- R2108_M3199_S003
+- R2108_M3200_S001
+- R2108_M3200_S002
+- R2108_M3200_S003
+- R2108_M3206_S001
+- R2108_M3206_S002
+- R2108_M3206_S003
+- R2108_M3210_S001
+- R2108_M3210_S002
+- R2108_M3210_S003
+- R2108_M3223_S001
+- R2108_M3223_S002
+- R2108_M3223_S003
+- R2108_M3225_S001
+- R2108_M3225_S002
+- R2108_M3225_S003
+- R2108_M3230_S001
+- R2108_M3230_S002
+- R2108_M3230_S003
+- R2108_M3243_S001
+- R2108_M3243_S002
+- R2108_M3243_S003
+- R2108_M3244_S001
+- R2108_M3244_S002
+- R2108_M3244_S003
+- id00000
+- id00001
+- id00002
+- id00003
+- id00004
+- id00005
+- id00006
+- id00007
+- id00008
+- id00009
+- id00010
+- id00011
+- id00012
+- id00013
+- id00014
+- id00015
+- id00016
+- id00017
+- id00018
+- id00019
+- id00020
+- id00021
+- id00022
+- id00023
+- id00024
+- id00025
+- id00026
+- id00027
+- id00028
+- id00029
+- id00030
+- id00031
+- id00032
+- id00033
+- id00034
+- id00035
+- id00036
+- id00037
+- id00038
+- id00039
+- id00040
+- id00041
+- id00042
+- id00043
+- id00044
+- id00045
+- id00046
+- id00047
+- id00048
+- id00049
+- id00050
+- id00051
+- id00052
+- id00053
+- id00054
+- id00055
+- id00056
+- id00057
+- id00058
+- id00059
+- id00060
+- id00061
+- id00062
+- id00063
+- id00064
+- id00065
+- id00066
+- id00067
+- id00068
+- id00069
+- id00070
+- id00071
+- id00072
+- id00073
+- id00074
+- id00075
+- id00076
+- id00077
+- id00078
+- id00079
+- id00080
+- id00081
+- id00082
+- id00083
+- id00084
+- id00085
+- id00086
+- id00087
+- id00088
+- id00089
+- id00090
+- id00091
+- id00092
+- id00093
+- id00094
+- id00095
+- id00096
+- id00097
+- id00098
+- id00099
+- id00100
+- id00101
+- id00102
+- id00103
+- id00104
+- id00105
+- id00106
+- id00107
+- id00108
+- id00109
+- id00110
+- id00111
+- id00112
+- id00113
+- id00114
+- id00115
+- id00116
+- id00117
+- id00118
+- id00119
+- id00120
+- id00121
+- id00122
+- id00123
+- id00124
+- id00125
+- id00126
+- id00127
+- id00128
+- id00129
+- id00130
+- id00131
+- id00132
+- id00133
+- id00134
+- id00135
+- id00136
+- id00137
+- id00138
+- id00139
+- id00140
+- id00141
+- id00142
+- id00145
+- id00146
+- id00148
+- id00149
+- id00151
+- id00152
+- id00153
+- id00154
+- id00155
+- id00156
+- id00157
+- id00158
+- id00159
+- id00160
+- id00161
+- id00162
+- id00163
+- id00164
+- id00165
+- id00166
+- id00167
+- id00169
+- id00170
+- id00171
+- id00173
+- id00174
+- id00175
+- id00176
+- id00177
+- id00178
+- id00179
+- id00180
+- id00181
+- id00182
+- id00183
+- id00184
+- id00185
+- id00186
+- id00187
+- id00188
+- id00189
+- id00190
+- id00191
+- id00192
+- id00193
+- id00194
+- id00195
+- id00196
+- id00197
+- id00198
+- id00199
+- id00200
+- id00201
+- id00203
+- id00204
+- id00206
+- id00207
+- id00208
+- id00209
+- id00210
+- id00211
+- id00213
+- id00214
+- id00215
+- id00216
+- id00217
+- id00218
+- id00219
+- id00220
+- id00221
+- id00222
+- id00223
+- id00224
+- id00225
+- id00226
+- id00228
+- id00230
+- id00231
+- id00233
+- id00234
+- id00235
+- id00236
+- id00237
+- id00238
+- id00240
+- id00241
+- id00242
+- id00243
+- id00244
+- id00245
+- id00246
+- id00247
+- id00248
+- id00249
+- id00250
+- id00251
+- id00252
+- id00253
+- id00254
+- id00256
+- id00257
+- id00258
+- id00259
+- id00260
+- id00261
+- id00262
+- id00263
+- id00264
+- id00265
+- id00266
+- id00267
+- id00268
+- id00269
+- id00270
+- id00271
+- id00272
+- id00274
+- id00275
+- id00276
+- id00278
+- id00279
+- id00280
+- id00281
+- id00282
+- id00283
+- id00284
+- id00285
+- id00286
+- id00287
+- id00288
+- id00289
+- id00290
+- id00291
+- id00292
+- id00293
+- id00294
+- id00295
+- id00296
+- id00297
+- id00298
+- id00299
+- id00300
+- id00301
+- id00302
+- id00303
+- id00304
+- id00305
+- id00306
+- id00307
+- id00308
+- id00309
+- id00310
+- id00311
+- id00312
+- id00313
+- id00314
+- id00315
+- id00316
+- id00318
+- id00319
+- id00320
+- id00322
+- id00323
+- id00324
+- id00325
+- id00326
+- id00327
+- id00328
+- id00329
+- id00330
+- id00331
+- id00332
+- id00333
+- id00334
+- id00335
+- id00336
+- id00337
+- id00338
+- id00339
+- id00340
+- id00341
+- id00342
+- id00343
+- id00344
+- id00345
+- id00346
+- id00347
+- id00348
+- id00349
+- id00350
+- id00351
+- id00352
+- id00353
+- id00354
+- id00355
+- id00356
+- id00357
+- id00358
+- id00359
+- id00360
+- id00361
+- id00362
+- id00363
+- id00364
+- id00365
+- id00366
+- id00367
+- id00368
+- id00369
+- id00370
+- id00371
+- id00372
+- id00373
+- id00374
+- id00375
+- id00376
+- id00377
+- id00378
+- id00379
+- id00380
+- id00381
+- id00382
+- id00383
+- id00384
+- id00385
+- id00386
+- id00387
+- id00388
+- id00389
+- id00390
+- id00391
+- id00392
+- id00393
+- id00394
+- id00395
+- id00396
+- id00397
+- id00398
+- id00399
+- id00400
+- id00401
+- id00402
+- id00403
+- id00404
+- id00405
+- id00406
+- id00407
+- id00408
+- id00409
+- id00410
+- id00411
+- id00412
+- id00413
+- id00414
+- id00415
+- id00416
+- id00417
+- id00418
+- id00419
+- id00420
+- id00421
+- id00422
+- id00423
+- id00424
+- id00425
+- id00426
+- id00427
+- id00428
+- id00429
+- id00430
+- id00431
+- id00432
+- id00433
+- id00434
+- id00435
+- id00436
+- id00437
+- id00438
+- id00439
+- id00440
+- id00441
+- id00442
+- id00443
+- id00444
+- id00445
+- id00446
+- id00447
+- id00448
+- id00449
+- id00450
+- id00451
+- id00452
+- id00453
+- id00454
+- id00455
+- id00456
+- id00458
+- id00459
+- id00460
+- id00461
+- id00462
+- id00463
+- id00464
+- id00465
+- id00466
+- id00467
+- id00468
+- id00469
+- id00470
+- id00472
+- id00473
+- id00474
+- id00475
+- id00476
+- id00477
+- id00478
+- id00479
+- id00480
+- id00481
+- id00482
+- id00483
+- id00484
+- id00485
+- id00486
+- id00487
+- id00488
+- id00489
+- id00490
+- id00491
+- id00492
+- id00493
+- id00494
+- id00495
+- id00496
+- id00497
+- id00498
+- id00499
+- id00500
+- id00501
+- id00502
+- id00503
+- id00504
+- id00505
+- id00506
+- id00507
+- id00508
+- id00509
+- id00510
+- id00511
+- id00512
+- id00513
+- id00514
+- id00515
+- id00516
+- id00517
+- id00518
+- id00519
+- id00520
+- id00521
+- id00522
+- id00523
+- id00524
+- id00525
+- id00526
+- id00527
+- id00528
+- id00529
+- id00530
+- id00531
+- id00532
+- id00533
+- id00534
+- id00535
+- id00536
+- id00537
+- id00538
+- id00539
+- id00540
+- id00541
+- id00542
+- id00543
+- id00544
+- id00545
+- id00546
+- id00547
+- id00549
+- id00550
+- id00551
+- id00552
+- id00553
+- id00554
+- id00555
+- id00556
+- id00557
+- id00558
+- id00559
+- id00560
+- id00561
+- id00562
+- id00563
+- id00564
+- id00565
+- id00566
+- id00567
+- id00568
+- id00569
+- id00570
+- id00571
+- id00572
+- id00573
+- id00574
+- id00575
+- id00576
+- id00577
+- id00578
+- id00579
+- id00580
+- id00581
+- id00582
+- id00583
+- id00584
+- id00585
+- id00586
+- id00587
+- id00588
+- id00589
+- id00590
+- id00591
+- id00592
+- id00593
+- id00594
+- id00595
+- id00596
+- id00597
+- id00598
+- id00599
+- id00600
+- id00601
+- id00603
+- id00604
+- id00605
+- id00606
+- id00607
+- id00608
+- id00609
+- id00610
+- id00611
+- id00612
+- id00613
+- id00614
+- id00615
+- id00616
+- id00619
+- id00620
+- id00621
+- id00622
+- id00623
+- id00624
+- id00625
+- id00626
+- id00627
+- id00628
+- id00629
+- id00630
+- id00631
+- id00632
+- id00633
+- id00634
+- id00635
+- id00636
+- id00637
+- id00638
+- id00639
+- id00640
+- id00641
+- id00642
+- id00643
+- id00644
+- id00645
+- id00646
+- id00647
+- id00648
+- id00649
+- id00650
+- id00651
+- id00652
+- id00653
+- id00654
+- id00655
+- id00656
+- id00657
+- id00658
+- id00659
+- id00661
+- id00662
+- id00663
+- id00664
+- id00665
+- id00666
+- id00667
+- id00668
+- id00669
+- id00670
+- id00671
+- id00672
+- id00673
+- id00674
+- id00675
+- id00676
+- id00678
+- id00679
+- id00680
+- id00681
+- id00682
+- id00683
+- id00684
+- id00685
+- id00686
+- id00687
+- id00688
+- id00689
+- id00690
+- id00691
+- id00692
+- id00693
+- id00694
+- id00695
+- id00696
+- id00697
+- id00698
+- id00699
+- id00701
+- id00702
+- id00703
+- id00704
+- id00705
+- id00706
+- id00707
+- id00708
+- id00709
+- id00710
+- id00711
+- id00712
+- id00713
+- id00714
+- id00715
+- id00716
+- id00717
+- id00718
+- id00719
+- id00720
+- id00721
+- id00722
+- id00723
+- id00724
+- id00725
+- id00726
+- id00727
+- id00728
+- id00729
+- id00730
+- id00731
+- id00732
+- id00733
+- id00734
+- id00735
+- id00737
+- id00738
+- id00739
+- id00740
+- id00741
+- id00742
+- id00743
+- id00744
+- id00745
+- id00746
+- id00747
+- id00748
+- id00749
+- id00750
+- id00751
+- id00752
+- id00754
+- id00755
+- id00756
+- id00757
+- id00758
+- id00759
+- id00760
+- id00761
+- id00762
+- id00763
+- id00764
+- id00765
+- id00766
+- id00767
+- id00768
+- id00769
+- id00770
+- id00771
+- id00772
+- id00773
+- id00774
+- id00775
+- id00776
+- id00777
+- id00778
+- id00779
+- id00780
+- id00781
+- id00782
+- id00783
+- id00784
+- id00785
+- id00786
+- id00787
+- id00788
+- id00789
+- id00790
+- id00791
+- id00792
+- id00793
+- id00794
+- id00795
+- id00796
+- id00797
+- id00798
+- id00799
+- id10000
+- id10001
+- id10002
+- id10003
+- id10004
+- id10005
+- id10006
+- id10007
+- id10008
+- id10009
+- id10010
+- id10011
+- id10012
+- id10013
+- id10014
+- id10015
+- id10016
+- id10017
+- id10018
+- id10019
+- id10020
+- id10021
+- id10022
+- id10023
+- id10024
+- id10025
+- id10026
+- id10027
+- id10028
+- id10029
+- id10030
+- id10031
+- id10032
+- id10033
+- id10034
+- id10035
+- id10036
+- id10037
+- id10038
+- id10039
+- id10040
+- id10041
+- id10042
+- id10043
+- id10044
+- id10045
+- id10046
+- id10047
+- id10048
+- id10049
+- id10050
+- id10051
+- id10052
+- id10053
+- id10054
+- id10055
+- id10056
+- id10057
+- id10058
+- id10059
+- id10060
+- id10061
+- id10062
+- id10063
+- id10064
+- id10065
+- id10066
+- id10067
+- id10068
+- id10069
+- id10070
+- id10071
+- id10072
+- id10073
+- id10074
+- id10075
+- id10076
+- id10077
+- id10078
+- id10079
+- id10080
+- id10081
+- id10082
+- id10083
+- id10084
+- id10085
+- id10086
+- id10087
+- id10088
+- id10089
+- id10090
+- id10091
+- id10092
+- id10093
+- id10094
+- id10095
+- id10096
+- id10097
+- id10098
+- id10099
+- id10100
+- id10101
+- id10102
+- id10103
+- id10104
+- id10105
+- id10106
+- id10107
+- id10108
+- id10109
+- id10110
+- id10111
+- id10112
+- id10113
+- id10114
+- id10115
+- id10116
+- id10117
+- id10118
+- id10119
+- id10120
+- id10121
+- id10122
+- id10123
+- id10124
+- id10125
+- id10126
+- id10127
+- id10128
+- id10129
+- id10130
+- id10131
+- id10132
+- id10133
+- id10134
+- id10135
+- id10136
+- id10137
+- id10138
+- id10139
+- id10140
+- id10141
+- id10142
+- id10143
+- id10144
+- id10145
+- id10146
+- id10147
+- id10148
+- id10149
+- id10150
+- id10151
+- id10152
+- id10153
+- id10154
+- id10155
+- id10156
+- id10157
+- id10158
+- id10159
+- id10160
+- id10161
+- id10162
+- id10163
+- id10164
+- id10165
+- id10167
+- id10168
+- id10169
+- id10170
+- id10171
+- id10172
+- id10173
+- id10174
+- id10175
+- id10176
+- id10177
+- id10178
+- id10179
+- id10180
+- id10181
+- id10182
+- id10183
+- id10184
+- id10185
+- id10186
+- id10187
+- id10188
+- id10189
+- id10190
+- id10191
+- id10192
+- id10193
+- id10194
+- id10195
+- id10196
+- id10197
+- id10198
+- id10199
+- id10200
+- id10201
+- id10202
+- id10203
+- id10204
+- id10205
+- id10206
+- id10207
+- id10208
+- id10209
+- id10210
+- id10211
+- id10212
+- id10213
+- id10214
+- id10215
+- id10216
+- id10217
+- id10218
+- id10219
+- id10220
+- id10221
+- id10222
+- id10223
+- id10224
+- id10225
+- id10226
+- id10227
+- id10228
+- id10229
+- id10230
+- id10231
+- id10232
+- id10233
+- id10234
+- id10235
+- id10236
+- id10237
+- id10238
+- id10239
+- id10240
+- id10241
+- id10242
+- id10243
+- id10244
+- id10245
+- id10246
+- id10247
+- id10248
+- id10249
+- id10250
+- id10251
+- id10252
+- id10253
+- id10254
+- id10255
+- id10256
+- id10257
+- id10258
+- id10259
+- id10260
+- id10261
+- id10262
+- id10263
+- id10264
+- id10265
+- id10266
+- id10267
+- id10268
+- id10269
+- id10270
+- id10271
+- id10272
+- id10273
+- id10274
+- id10275
+- id10276
+- id10277
+- id10278
+- id10279
+- id10280
+- id10281
+- id10282
+- id10283
+- id10284
+- id10285
+- id10286
+- id10287
+- id10288
+- id10289
+- id10290
+- id10291
+- id10292
+- id10293
+- id10294
+- id10295
+- id10296
+- id10297
+- id10298
+- id10299
+- id10300
+- id10301
+- id10302
+- id10303
+- id10304
+- id10305
+- id10306
+- id10307
+- id10308
+- id10309
+- id10310
+- id10311
+- id10312
+- id10313
+- id10314
+- id10315
+- id10316
+- id10317
+- id10318
+- id10319
+- id10320
+- id10321
+- id10322
+- id10323
+- id10324
+- id10325
+- id10326
+- id10327
+- id10328
+- id10329
+- id10330
+- id10331
+- id10332
+- id10333
+- id10334
+- id10335
+- id10336
+- id10337
+- id10338
+- id10339
+- id10340
+- id10341
+- id10342
+- id10343
+- id10344
+- id10345
+- id10346
+- id10347
+- id10348
+- id10349
+- id10350
+- id10351
+- id10352
+- id10353
+- id10354
+- id10355
+- id10356
+- id10357
+- id10358
+- id10359
+- id10360
+- id10361
+- id10362
+- id10363
+- id10364
+- id10365
+- id10366
+- id10367
+- id10368
+- id10369
+- id10370
+- id10371
+- id10372
+- id10373
+- id10374
+- id10375
+- id10376
+- id10377
+- id10378
+- id10379
+- id10380
+- id10381
+- id10382
+- id10383
+- id10384
+- id10385
+- id10386
+- id10387
+- id10388
+- id10389
+- id10390
+- id10391
+- id10392
+- id10393
+- id10394
+- id10395
+- id10396
+- id10397
+- id10398
+- id10399
+- id10400
+- id10401
+- id10402
+- id10403
+- id10404
+- id10405
+- id10406
+- id10407
+- id10408
+- id10409
+- id10410
+- id10411
+- id10412
+- id10413
+- id10414
+- id10415
+- id10416
+- id10417
+- id10418
+- id10419
+- id10420
+- id10421
+- id10422
+- id10423
+- id10424
+- id10425
+- id10426
+- id10427
+- id10428
+- id10429
+- id10430
+- id10431
+- id10432
+- id10433
+- id10434
+- id10435
+- id10436
+- id10437
+- id10438
+- id10439
+- id10440
+- id10441
+- id10442
+- id10443
+- id10444
+- id10445
+- id10446
+- id10447
+- id10448
+- id10449
+- id10450
+- id10451
+- id10452
+- id10453
+- id10454
+- id10455
+- id10456
+- id10457
+- id10458
+- id10459
+- id10460
+- id10461
+- id10462
+- id10463
+- id10464
+- id10465
+- id10466
+- id10467
+- id10468
+- id10469
+- id10470
+- id10471
+- id10472
+- id10473
+- id10474
+- id10475
+- id10476
+- id10477
+- id10478
+- id10479
+- id10480
+- id10481
+- id10482
+- id10483
+- id10484
+- id10485
+- id10486
+- id10487
+- id10488
+- id10489
+- id10490
+- id10491
+- id10492
+- id10493
+- id10494
+- id10495
+- id10496
+- id10497
+- id10498
+- id10499
+- id10500
+- id10501
+- id10502
+- id10503
+- id10504
+- id10505
+- id10506
+- id10508
+- id10509
+- id10510
+- id10511
+- id10512
+- id10513
+- id10514
+- id10515
+- id10516
+- id10517
+- id10518
+- id10519
+- id10520
+- id10521
+- id10522
+- id10523
+- id10524
+- id10525
+- id10526
+- id10527
+- id10528
+- id10529
+- id10530
+- id10531
+- id10532
+- id10533
+- id10534
+- id10535
+- id10536
+- id10537
+- id10538
+- id10539
+- id10540
+- id10541
+- id10542
+- id10543
+- id10544
+- id10545
+- id10546
+- id10547
+- id10548
+- id10549
+- id10550
+- id10551
+- id10552
+- id10553
+- id10554
+- id10555
+- id10556
+- id10557
+- id10558
+- id10559
+- id10560
+- id10561
+- id10562
+- id10563
+- id10564
+- id10565
+- id10566
+- id10567
+- id10568
+- id10569
+- id10570
+- id10571
+- id10572
+- id10573
+- id10574
+- id10575
+- id10576
+- id10577
+- id10578
+- id10579
+- id10580
+- id10581
+- id10582
+- id10583
+- id10584
+- id10585
+- id10586
+- id10587
+- id10588
+- id10589
+- id10590
+- id10591
+- id10592
+- id10593
+- id10594
+- id10595
+- id10596
+- id10597
+- id10598
+- id10599
+- id10600
+- id10601
+- id10602
+- id10603
+- id10604
+- id10605
+- id10606
+- id10607
+- id10608
+- id10609
+- id10610
+- id10611
+- id10612
+- id10613
+- id10614
+- id10615
+- id10616
+- id10617
+- id10618
+- id10619
+- id10620
+- id10621
+- id10622
+- id10623
+- id10624
+- id10625
+- id10626
+- id10627
+- id10628
+- id10629
+- id10630
+- id10631
+- id10632
+- id10633
+- id10634
+- id10635
+- id10637
+- id10638
+- id10639
+- id10640
+- id10641
+- id10642
+- id10643
+- id10644
+- id10645
+- id10646
+- id10647
+- id10648
+- id10649
+- id10650
+- id10651
+- id10652
+- id10653
+- id10654
+- id10655
+- id10656
+- id10657
+- id10658
+- id10659
+- id10660
+- id10661
+- id10662
+- id10663
+- id10664
+- id10665
+- id10666
+- id10668
+- id10669
+- id10670
+- id10671
+- id10672
+- id10673
+- id10674
+- id10675
+- id10676
+- id10677
+- id10678
+- id10679
+- id10680
+- id10681
+- id10682
+- id10683
+- id10684
+- id10685
+- id10686
+- id10687
+- id10688
+- id10689
+- id10690
+- id10691
+- id10692
+- id10693
+- id10694
+- id10695
+- id10696
+- id10697
+- id10698
+- id10699
+- id10700
+- id10701
+- id10702
+- id10703
+- id10704
+- id10705
+- id10706
+- id10707
+- id10708
+- id10709
+- id10710
+- id10711
+- id10712
+- id10713
+- id10714
+- id10715
+- id10716
+- id10717
+- id10718
+- id10719
+- id10720
+- id10721
+- id10722
+- id10723
+- id10724
+- id10725
+- id10726
+- id10727
+- id10728
+- id10729
+- id10730
+- id10731
+- id10732
+- id10733
+- id10734
+- id10735
+- id10736
+- id10737
+- id10738
+- id10739
+- id10740
+- id10741
+- id10742
+- id10743
+- id10744
+- id10745
+- id10746
+- id10747
+- id10748
+- id10749
+- id10750
+- id10751
+- id10752
+- id10753
+- id10754
+- id10755
+- id10756
+- id10757
+- id10758
+- id10759
+- id10760
+- id10761
+- id10762
+- id10763
+- id10764
+- id10765
+- id10766
+- id10767
+- id10768
+- id10769
+- id10770
+- id10771
+- id10772
+- id10773
+- id10774
+- id10775
+- id10776
+- id10777
+- id10778
+- id10779
+- id10780
+- id10781
+- id10782
+- id10783
+- id10784
+- id10785
+- id10786
+- id10787
+- id10788
+- id10789
+- id10790
+- id10791
+- id10792
+- id10793
+- id10794
+- id10795
+- id10796
+- id10797
+- id10798
+- id10799
+- id10800
+- id10801
+- id10802
+- id10803
+- id10804
+- id10805
+- id10806
+- id10807
+- id10808
+- id10809
+- id10810
+- id10812
+- id10813
+- id10814
+- id10815
+- id10816
+- id10817
+- id10818
+- id10819
+- id10820
+- id10821
+- id10822
+- id10823
+- id10824
+- id10825
+- id10826
+- id10827
+- id10828
+- id10829
+- id10830
+- id10831
+- id10832
+- id10833
+- id10834
+- id10835
+- id10836
+- id10837
+- id10838
+- id10839
+- id10840
+- id10841
+- id10842
+- id10843
+- id10844
+- id10845
+- id10846
+- id10847
+- id10848
+- id10849
+- id10850
+- id10851
+- id10852
+- id10853
+- id10854
+- id10855
+- id10856
+- id10857
+- id10858
+- id10859
+- id10860
+- id10861
+- id10862
+- id10863
+- id10864
+- id10865
+- id10866
+- id10867
+- id10868
+- id10869
+- id10870
+- id10871
+- id10872
+- id10873
+- id10874
+- id10875
+- id10876
+- id10877
+- id10878
+- id10879
+- id10880
+- id10881
+- id10882
+- id10883
+- id10884
+- id10885
+- id10886
+- id10887
+- id10888
+- id10889
+- id10890
+- id10891
+- id10892
+- id10893
+- id10894
+- id10895
+- id10896
+- id10897
+- id10898
+- id10899
+- id10900
+- id10901
+- id10902
+- id10903
+- id10904
+- id10905
+- id10906
+- id10907
+- id10908
+- id10909
+- id10910
+- id10911
+- id10912
+- id10913
+- id10914
+- id10915
+- id10916
+- id10917
+- id10918
+- id10919
+- id10920
+- id10921
+- id10922
+- id10923
+- id10924
+- id10925
+- id10926
+- id10927
+- id10928
+- id10929
+- id10930
+- id10931
+- id10932
+- id10933
+- id10934
+- id10935
+- id10936
+- id10937
+- id10938
+- id10939
+- id10940
+- id10941
+- id10942
+- id10943
+- id10944
+- id10945
+- id10946
+- id10947
+- id10948
+- id10949
+- id10950
+- id10951
+- id10952
+- id10953
+- id10954
+- id10955
+- id10956
+- id10957
+- id10958
+- id10959
+- id10960
+- id10961
+- id10962
+- id10963
+- id10964
+- id10965
+- id10966
+- id10967
+- id10968
+- id10969
+- id10970
+- id10971
+- id10972
+- id10973
+- id10974
+- id10975
+- id10976
+- id10977
+- id10978
+- id10979
+- id10980
+- id10981
+- id10982
+- id10983
+- id10984
+- id10985
+- id10986
+- id10987
+- id10988
+- id10989
+- id10990
+- id10991
+- id10992
+- id10993
+- id10994
+- id10995
+- id10996
+- id10997
+- id10998
+- id10999
+- id11000
+- id11001
+- id11002
+- id11004
+- id11005
+- id11006
+- id11007
+- id11008
+- id11009
+- id11011
+- id11012
+- id11013
+- id11014
+- id11015
+- id11016
+- id11017
+- id11018
+- id11019
+- id11020
+- id11021
+- id11022
+- id11023
+- id11024
+- id11025
+- id11026
+- id11027
+- id11028
+- id11029
+- id11030
+- id11031
+- id11032
+- id11033
+- id11034
+- id11035
+- id11036
+- id11037
+- id11038
+- id11039
+- id11040
+- id11041
+- id11042
+- id11043
+- id11044
+- id11045
+- id11046
+- id11047
+- id11048
+- id11049
+- id11050
+- id11051
+- id11052
+- id11053
+- id11054
+- id11055
+- id11056
+- id11057
+- id11058
+- id11059
+- id11060
+- id11061
+- id11062
+- id11063
+- id11064
+- id11065
+- id11066
+- id11067
+- id11068
+- id11069
+- id11070
+- id11071
+- id11072
+- id11073
+- id11074
+- id11075
+- id11076
+- id11077
+- id11078
+- id11079
+- id11080
+- id11081
+- id11082
+- id11083
+- id11084
+- id11085
+- id11086
+- id11087
+- id11088
+- id11089
+- id11090
+- id11091
+- id11092
+- id11093
+- id11094
+- id11095
+- id11096
+- id11097
+- id11098
+- id11099
+- id11100
+- id11101
+- id11102
+- id11103
+- id11104
+- id11105
+- id11106
+- id11107
+- id11108
+- id11109
+- id11110
+- id11111
+- id11112
+- id11113
+- id11114
+- id11115
+- id11116
+- id11117
+- id11118
+- id11119
+- id11120
+- id11121
+- id11122
+- id11123
+- id11124
+- id11125
+- id11126
+- id11127
+- id11128
+- id11129
+- id11130
+- id11131
+- id11132
+- id11133
+- id11134
+- id11135
+- id11136
+- id11137
+- id11138
+- id11139
+- id11140
+- id11141
+- id11142
+- id11143
+- id11144
+- id11145
+- id11146
+- id11147
+- id11148
+- id11149
+- id11150
+- id11151
+- id11152
+- id11153
+- id11154
+- id11155
+- id11156
+- id11157
+- id11158
+- id11159
+- id11160
+- id11161
+- id11162
+- id11163
+- id11164
+- id11165
+- id11166
+- id11167
+- id11168
+- id11169
+- id11170
+- id11171
+- id11172
+- id11173
+- id11174
+- id11175
+- id11176
+- id11177
+- id11178
+- id11179
+- id11180
+- id11181
+- id11182
+- id11183
+- id11184
+- id11185
+- id11186
+- id11187
+- id11188
+- id11189
+- id11190
+- id11191
+- id11192
+- id11193
+- id11194
+- id11195
+- id11196
+- id11197
+- id11198
+- id11199
+- id11200
+- id11201
+- id11202
+- id11203
+- id11204
+- id11205
+- id11206
+- id11207
+- id11208
+- id11209
+- id11210
+- id11211
+- id11212
+- id11213
+- id11214
+- id11215
+- id11216
+- id11217
+- id11218
+- id11219
+- id11220
+- id11221
+- id11222
+- id11223
+- id11224
+- id11225
+- id11226
+- id11227
+- id11228
+- id11229
+- id11230
+- id11231
+- id11232
+- id11233
+- id11234
+- id11235
+- id11236
+- id11237
+- id11238
+- id11239
+- id11240
+- id11241
+- id11242
+- id11243
+- id11244
+- id11245
+- id11246
+- id11247
+- id11248
+- id11249
+- id11250
+- id11251
+- id11252
+- id11254
+- id11255
+- id11256
+- id11257
+- id11258
+- id11259
+- id11260
+- id11261
+- id11262
+- id11263
+- id11264
+- id11265
+- id11266
+- id11267
+- id11268
+- id11269
+- id11270
+- id11271
+- id11272
+- id11273
+- id11274
+- id11275
+- id11276
+- id11277
+- id11278
+- id11279
+- id11280
+- id11281
+- id11282
+- id11283
+- id11284
+- id11285
+- id11286
+- id11287
+- id11288
+- id11289
+- id11290
+- id11291
+- id11292
+- id11293
+- id11294
+- id11295
+- id11296
+- id11297
+- id11298
+- id11299
+- id11300
+- id11301
+- id11302
+- id11303
+- id11304
+- id11305
+- id11306
+- id11307
+- id11308
+- id11309
+- id11310
+- id11311
+- id11312
+- id11313
+- id11314
+- id11315
+- id11316
+- id11317
+- id11318
+- id11319
+- id11320
+- id11321
+- id11322
+- id11323
+- id11324
+- id11325
+- id11326
+- id11327
+- id11328
+- id11329
+- id11330
+- id11331
+- id11332
+- id11333
+- id11334
+- id11335
+- id11336
+- id11337
+- id11338
+- id11339
+- id11340
+- id11341
+- id11342
+- id11343
+- id11344
+- id11345
+- id11346
+- id11347
+- id11348
+- id11349
+- id11350
+- id11351
+- id11352
+- id11353
+- id11354
+- id11355
+- id11356
+- id11357
+- id11358
+- id11359
+- id11360
+- id11361
+- id11362
+- id11363
+- id11364
+- id11365
+- id11366
+- id11367
+- id11368
+- id11369
+- id11370
+- id11371
+- id11372
+- id11373
+- id11374
+- id11375
+- id11376
+- id11377
+- id11379
+- id11380
+- id11381
+- id11382
+- id11383
+- id11384
+- id11385
+- id11386
+- id11387
+- id11388
+- id11389
+- id11390
+- id11391
+- id11392
+- id11393
+- id11394
+- id11395
+- id11396
+- id11397
+- id11398
+- id11399
+- id11400
+- id11401
+- id11402
+- id11403
+- id11404
+- id11405
+- id11406
+- id11407
+- id11408
+- id11409
+- id11410
+- id11411
+- id11412
+- id11413
+- id11414
+- id11415
+- id11416
+- id11417
+- id11418
+- id11419
+- id11420
+- id11421
+- id11422
+- id11423
+- id11424
+- id11425
+- id11426
+- id11427
+- id11428
+- id11429
+- id11430
+- id11431
+- id11432
+- id11433
+- id11434
+- id11435
+- id11436
+- id11437
+- id11438
+- id11439
+- id11440
+- id11441
+- id11442
+- id11443
+- id11444
+- id11445
+- id11446
+- id11447
+- id11448
+- id11449
+- id11450
+- id11451
+- id11452
+- id11453
+- id11454
+- id11455
+- id11456
+- id11457
+- id11458
+- id11459
+- id11460
+- id11461
+- id11462
+- id11463
+- id11464
+- id11465
+- id11466
+- id11467
+- id11468
+- id11469
+- id11470
+- id11471
+- id11472
+- id11473
+- id11474
+- id11475
+- id11476
+- id11477
+- id11478
+- id11479
+- id11480
+- id11481
+- id11482
+- id11483
+- id11484
+- id11485
+- id11486
+- id11487
+- id11488
+- id11489
+- id11490
+- id11491
+- id11492
+- id11493
+- id11495
+- id11496
+- id11497
+- id11498
+- id11499
+- id11500
+- id11501
+- id11502
+- id11503
+- id11504
+- id11505
+- id11506
+- id11507
+- id11508
+- id11509
+- id11510
+- id11511
+- id11512
+- id11513
+- id11514
+- id11515
+- id11516
+- id11517
+- id11518
+- id11519
+- id11520
+- id11521
+- id11522
+- id11523
+- id11524
+- id11525
+- id11526
+- id11527
+- id11528
+- id11529
+- id11530
+- id11531
+- id11532
+- id11533
+- id11534
+- id11535
+- id11536
+- id11537
+- id11538
+- id11539
+- id11540
+- id11541
+- id11542
+- id11543
+- id11544
+- id11545
+- id11546
+- id11547
+- id11548
+- id11549
+- id11550
+- id11551
+- id11552
+- id11553
+- id11554
+- id11555
+- id11556
+- id11557
+- id11558
+- id11559
+- id11560
+- id11561
+- id11562
+- id11563
+- id11564
+- id11565
+- id11566
+- id11567
+- id11568
+- id11569
+- id11570
+- id11571
+- id11572
+- id11573
+- id11574
+- id11575
+- id11576
+- id11577
+- id11578
+- id11579
+- id11580
+- id11581
+- id11582
+- id11583
+- id11584
+- id11585
+- id11586
+- id11587
+- id11588
+- id11589
+- id11590
+- id11591
+- id11592
+- id11593
+- id11594
+- id11595
+- id11596
+- id11597
+- id11598
+- id11599
+- id11600
+- id11601
+- id11602
+- id11603
+- id11604
+- id11605
+- id11606
+- id11607
+- id11608
+- id11609
+- id11610
+- id11611
+- id11612
+- id11613
+- id11614
+- id11615
+- id11616
+- id11617
+- id11618
+- id11619
+- id11620
+- id11621
+- id11622
+- id11623
+- id11624
+- id11625
+- id11626
+- id11627
+- id11628
+- id11629
+- id11630
+- id11631
+- id11632
+- id11633
+- id11634
+- id11635
+- id11636
+- id11637
+- id11638
+- id11639
+- id11640
+- id11641
+- id11642
+- id11643
+- id11644
+- id11645
+- id11646
+- id11647
+- id11648
+- id11649
+- id11650
+- id11651
+- id11652
+- id11653
+- id11654
+- id11655
+- id11656
+- id11657
+- id11658
+- id11659
+- id11660
+- id11661
+- id11662
+- id11663
+- id11664
+- id11665
+- id11666
+- id11667
+- id11668
+- id11669
+- id11670
+- id11671
+- id11672
+- id11673
+- id11674
+- id11675
+- id11676
+- id11677
+- id11678
+- id11679
+- id11680
+- id11681
+- id11682
+- id11683
+- id11684
+- id11685
+- id11686
+- id11687
+- id11688
+- id11689
+- id11690
+- id11691
+- id11692
+- id11693
+- id11694
+- id11695
+- id11696
+- id11697
+- id11698
+- id11699
+- id11700
+- id11701
+- id11702
+- id11703
+- id11704
+- id11705
+- id11706
+- id11707
+- id11708
+- id11709
+- id11710
+- id11711
+- id11712
+- id11713
+- id11714
+- id11715
+- id11716
+- id11717
+- id11718
+- id11719
+- id11720
+- id11721
+- id11722
+- id11723
+- id11724
+- id11725
+- id11726
+- id11727
+- id11728
+- id11729
+- id11730
+- id11731
+- id11732
+- id11733
+- id11734
+- id11735
+- id11736
+- id11737
+- id11738
+- id11739
+- id11740
+- id11741
+- id11742
+- id11744
+- id11745
+- id11746
+- id11747
+- id11748
+- id11749
+- id11750
+- id11751
+- id11752
+- id11753
+- id11754
+- id11755
+- id11756
+- id11757
+- id11758
+- id11759
+- id11760
+- id11761
+- id11762
+- id11763
+- id11764
+- id11765
+- id11766
+- id11767
+- id11768
+- id11769
+- id11770
+- id11771
+- id11772
+- id11773
+- id11774
+- id11775
+- id11776
+- id11777
+- id11778
+- id11779
+- id11780
+- id11781
+- id11782
+- id11783
+- id11784
+- id11785
+- id11786
+- id11787
+- id11788
+- id11789
+- id11790
+- id11791
+- id11792
+- id11793
+- id11794
+- id11795
+- id11796
+- id11797
+- id11798
+- id11799
+- id11800
+- id11801
+- id11802
+- id11803
+- id11804
+- id11805
+- id11806
+- id11807
+- id11808
+- id11809
+- id11810
+- id11811
+- id11812
+- id11813
+- id11814
+- id11815
+- id11816
+- id11817
+- id11818
+- id11819
+- id11820
+- id11821
+- id11822
+- id11823
+- id11824
+- id11825
+- id11826
+- id11827
+- id11828
+- id11829
+- id11830
+- id11831
+- id11832
+- id11833
+- id11834
+- id11835
+- id11836
+- id11837
+- id11838
+- id11839
+- id11840
+- id11841
+- id11842
+- id11843
+- id11844
+- id11845
+- id11846
+- id11847
+- id11848
+- id11849
+- id11850
+- id11851
+- id11852
+- id11853
+- id11854
+- id11855
+- id11856
+- id11857
+- id11858
+- id11859
+- id11860
+- id11861
+- id11862
+- id11863
+- id11864
+- id11865
+- id11866
+- id11867
+- id11868
+- id11869
+- id11870
+- id11871
+- id11872
+- id11873
+- id11874
+- id11875
+- id11876
+- id11877
+- id11878
+- id11879
+- id11880
+- id11881
+- id11882
+- id11883
+- id11884
+- id11885
+- id11886
+- id11887
+- id11888
+- id11889
+- id11890
+- id11891
+- id11892
+- id11893
+- id11894
+- id11895
+- id11896
+- id11897
+- id11898
+- id11899
+- id11900
+- id11901
+- id11902
+- id11903
+- id11904
+- id11905
+- id11906
+- id11907
+- id11908
+- id11909
+- id11910
+- id11911
+- id11912
+- id11913
+- id11914
+- id11915
+- id11916
+- id11917
+- id11918
+- id11919
+- id11920
+- id11921
+- id11922
+- id11923
+- id11924
+- id11925
+- id11926
+- id11927
+- id11928
+- id11929
+- id11930
+- id11931
+- id11932
+- id11933
+- id11934
+- id11935
+- id11936
+- id11937
+- id11938
+- id11939
+- id11940
+- id11941
+- id11942
+- id11943
+- id11944
+- id11945
+- id11946
+- id11947
+- id11948
+- id11949
+- id11950
+- id11951
+- id11952
+- id11953
+- id11954
+- id11955
+- id11956
+- id11957
+- id11958
+- id11959
+- id11960
+- id11961
+- id11962
+- id11963
+- id11964
+- id11965
+- id11966
+- id11967
+- id11968
+- id11969
+- id11970
+- id11971
+- id11972
+- id11973
+- id11974
+- id11975
+- id11976
+- id11977
+- id11978
+- id11979
+- id11980
+- id11981
+- id11982
+- id11983
+- id11984
+- id11985
+- id11986
+- id11987
+- id11988
+- id11989
+- id11990
+- id11991
+- id11992
+- id11993
+- id11994
+- id11995
+- id11996
+- id11997
+- id11998
+- id11999
+- R8001_M8004_MS801_S001
+- R8001_M8004_MS801_S002
+- R8001_M8004_MS801_S003
+- R8001_M8004_MS801_S004
+- R8003_M8001_MS801_S001
+- R8003_M8001_MS801_S002
+- R8003_M8001_MS801_S003
+- R8003_M8001_MS801_S004
+- R8007_M8010_MS803_S001
+- R8007_M8010_MS803_S002
+- R8007_M8010_MS803_S003
+- R8007_M8010_MS803_S004
+- R8007_M8011_MS806_S001
+- R8007_M8011_MS806_S002
+- R8007_M8011_MS806_S003
+- R8007_M8011_MS806_S004
+- R8008_M8013_MS807_S001
+- R8008_M8013_MS807_S002
+- R8008_M8013_MS807_S003
+- R8009_M8018_MS809_S001
+- R8009_M8018_MS809_S002
+- R8009_M8019_MS810_S001
+- R8009_M8019_MS810_S002
+- R8009_M8020_MS810_S001
+- R8009_M8020_MS810_S002
diff --git a/egs/cnceleb/resnet34/utils b/egs/cnceleb/resnet34/utils
new file mode 120000
index 0000000..f245098
--- /dev/null
+++ b/egs/cnceleb/resnet34/utils
@@ -0,0 +1 @@
+../../aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs_modelscope/aishell/paraformer/README.md b/egs_modelscope/aishell/paraformer/README.md
deleted file mode 100644
index 48a5621..0000000
--- a/egs_modelscope/aishell/paraformer/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-# ModelScope: Paraformer-large Model
-
-## Highlight
-
-### ModelScope: Paraformer-Large Model
-- <strong>Fast</strong>: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup.
-- <strong>Accurate</strong>: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation.
-- <strong>Convenient</strong>: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference.
- - Support finetuning and inference on AISHELL-1 and AISHELL-2.
- - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio.
-
-## How to finetune and infer using a pretrained ModelScope Paraformer-large Model
-
-### Finetune
-- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml`
-- Setting parameters in `paraformer_large_finetune.sh`
- - <strong>data_aishell:</strong> please set the aishell data path
- - <strong>tag:</strong> exp tag
- - <strong>init_model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-- Then you can run the pipeline to finetune with our model download from modelscope and infer after finetune:
-```sh
- sh ./paraformer_large_finetune.sh
-```
-
-### Inference
-
-Or you can download the model from ModelScope for inference directly.
-
-- Setting parameters in `paraformer_large_infer.sh`
- - <strong>ori_data:</strong> please set the aishell raw data path
- - <strong>data_dir:</strong> data output dictionary
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope
- - <strong>test_sets:</strong> please set the testsets name
-- Then you can run the pipeline to infer with:
-```sh
- sh ./paraformer_large_infer.sh
-```
diff --git a/egs_modelscope/aishell/paraformer/RESULTS.md b/egs_modelscope/aishell/paraformer/RESULTS.md
deleted file mode 100644
index 5167504..0000000
--- a/egs_modelscope/aishell/paraformer/RESULTS.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# Paraformer-Large
-- Model link: <https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary>
-- Model size: 220M
-- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-
-# Environments
-- date: `Tue Nov 22 18:48:39 CST 2022`
-- python version: `3.7.12`
-- FunASR version: `0.1.0`
-- pytorch version: `pytorch 1.7.0`
-- Git hash: ``
-- Commit date: ``
-
-# Beachmark Results
-
-## AISHELL-1
-- Decode config: conf/decode_asr_transformer_noctc_1best.yaml
- - Decode without CTC
- - Decode without LM
-
-| testset | CER(%)|
-|:---------:|:-----:|
-| dev | 1.75 |
-| test | 1.95 |
diff --git a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
deleted file mode 100644
index 22f02d9..0000000
--- a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 10
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.15
diff --git a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
deleted file mode 100644
index e623192..0000000
--- a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 1
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.0
diff --git a/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
deleted file mode 100644
index cb8b0af..0000000
--- a/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
+++ /dev/null
@@ -1,91 +0,0 @@
-# network architecture
-# encoder related
-encoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- attention_dropout_rate: 0.1
-
-# decoder related
-decoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- self_attention_dropout_rate: 0.1
- src_attention_dropout_rate: 0.1
-
-predictor_conf:
- threshold: 1.0
- l_order: 1
- r_order: 1
- tail_threshold: 0.45
-
-# hybrid CTC/attention
-model_conf:
- ctc_weight: 0.0
- lsm_weight: 0.1 # label smoothing option
- length_normalized_loss: true
- predictor_weight: 1.0
- predictor_bias: 1
- sampling_ratio: 0.75
-
-# minibatch related
-# dataset_type: small
-batch_type: length
-batch_bins: 2000
-num_workers: 16
-# dataset_type: large
-dataset_conf:
- filter_conf:
- min_length: 10
- max_length: 250
- min_token_length: 1
- max_token_length: 200
- shuffle: true
- shuffle_conf:
- shuffle_size: 10240
- sort_size: 500
- batch_conf:
- batch_type: 'token'
- batch_size: 6000
- num_workers: 16
-
-# optimization related
-accum_grad: 1
-grad_clip: 5
-max_epoch: 20
-val_scheduler_criterion:
- - valid
- - acc
-best_model_criterion:
-- - valid
- - acc
- - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
- lr: 0.0005
-scheduler: warmuplr
-scheduler_conf:
- warmup_steps: 50000
-
-specaug: specaug_lfr
-specaug_conf:
- apply_time_warp: false
- time_warp_window: 5
- time_warp_mode: bicubic
- apply_freq_mask: true
- freq_mask_width_range:
- - 0
- - 30
- lfr_rate: 6
- num_freq_mask: 1
- apply_time_mask: true
- time_mask_width_range:
- - 0
- - 12
- num_time_mask: 1
-
-unused_parameters: true
-log_interval: 50
-normalize: None
-split_with_space: true
diff --git a/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh b/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh
deleted file mode 100755
index 83f489b..0000000
--- a/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 Xingyu Na
-# Apache 2.0
-
-#. ./path.sh || exit 1;
-
-if [ $# != 3 ]; then
- echo "Usage: $0 <audio-path> <text-path> <output-path>"
- echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data"
- exit 1;
-fi
-
-aishell_audio_dir=$1
-aishell_text=$2/aishell_transcript_v0.8.txt
-output_dir=$3
-
-train_dir=$output_dir/data/local/train
-dev_dir=$output_dir/data/local/dev
-test_dir=$output_dir/data/local/test
-tmp_dir=$output_dir/data/local/tmp
-
-mkdir -p $train_dir
-mkdir -p $dev_dir
-mkdir -p $test_dir
-mkdir -p $tmp_dir
-
-# data directory check
-if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then
- echo "Error: $0 requires two directory arguments"
- exit 1;
-fi
-
-# find wav audio file for train, dev and test resp.
-find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist
-n=`cat $tmp_dir/wav.flist | wc -l`
-[ $n -ne 141925 ] && \
- echo Warning: expected 141925 data data files, found $n
-
-grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1;
-grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1;
-grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1;
-
-rm -r $tmp_dir
-
-# Transcriptions preparation
-for dir in $train_dir $dev_dir $test_dir; do
- echo Preparing $dir transcriptions
- sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list
- paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all
- utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt
- awk '{print $1}' $dir/transcripts.txt > $dir/utt.list
- utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp
- sort -u $dir/transcripts.txt > $dir/text
-done
-
-mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test
-
-for f in wav.scp text; do
- cp $train_dir/$f $output_dir/data/train/$f || exit 1;
- cp $dev_dir/$f $output_dir/data/dev/$f || exit 1;
- cp $test_dir/$f $output_dir/data/test/$f || exit 1;
-done
-
-echo "$0: AISHELL data preparation succeeded"
-exit 0;
diff --git a/egs_modelscope/aishell/paraformer/modelscope_utils b/egs_modelscope/aishell/paraformer/modelscope_utils
deleted file mode 120000
index fc97768..0000000
--- a/egs_modelscope/aishell/paraformer/modelscope_utils
+++ /dev/null
@@ -1 +0,0 @@
-../../common/modelscope_utils
\ No newline at end of file
diff --git a/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh b/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh
deleted file mode 100755
index 5e1f01a..0000000
--- a/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh
+++ /dev/null
@@ -1,267 +0,0 @@
-#!/usr/bin/env bash
-
-. ./path.sh || exit 1;
-
-# machines configuration
-CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1"
-gpu_num=2
-count=1
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-njob=1 # the number of jobs for each gpu
-train_cmd=utils/run.pl
-infer_cmd=utils/run.pl
-
-# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
-exp_dir="."
-lang=zh
-dumpdir=dump/fbank
-feats_type=fbank
-token_type=char
-scp=feats.scp
-type=kaldi_ark
-stage=0
-stop_stage=4
-
-# feature configuration
-feats_dim=560
-sample_frequency=16000
-nj=32
-speed_perturb="1.0"
-lfr=True
-lfr_m=7
-lfr_n=6
-
-init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-model_revision="v1.0.4" # please do not modify the model revision
-cmvn_file=init_model/${init_model_name}/am.mvn
-seg_file=init_model/${init_model_name}/seg_dict
-vocab=init_model/${init_model_name}/tokens.txt
-
-# data
-data_aishell=
-
-# exp tag
-tag=""
-
-# Set bash to 'debug' mode, it will exit on :
-# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
-set -e
-set -u
-set -o pipefail
-
-train_set=train
-valid_set=dev
-test_sets="dev test"
-
-asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-init_param="init_model/${init_model_name}/model.pb"
-
-inference_config=conf/decode_asr_transformer_noctc_1best.yaml
-inference_asr_model=valid.acc.ave_10best.pth
-
-. utils/parse_options.sh || exit 1;
-
-# download model from modelscope
-python modelscope_utils/download_model.py --model_name ${init_model_name} --model_revision ${model_revision}
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist"
- exit 1
-else
- if [ -d init_model/${init_model_name} ]; then
- echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first."
- else
- mkdir -p init_model/${init_model_name}
- cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name}
- fi
-fi
-
-model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
-
-# you can set gpu num for decoding here
-gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-
-if ${gpu_inference}; then
- inference_nj=$njob
- _ngpu=1
-else
- inference_nj=$njob
- _ngpu=0
-fi
-
-if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
- echo "stage 0: Data preparation"
- # Data preparation
- local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript ${feats_dir}
- for x in train dev test; do
- cp ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org
- paste -d " " <(cut -f 1 -d" " ${feats_dir}/data/${x}/text.org) <(cut -f 2- -d" " ${feats_dir}/data/${x}/text.org | tr -d " ") \
- > ${feats_dir}/data/${x}/text
- rm ${feats_dir}/data/${x}/text.org
- done
-fi
-
-feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir}
-feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir}
-feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir}
-if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
- echo "stage 1: Feature Generation"
- # compute fbank features
- fbankdir=${feats_dir}/fbank
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
- ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
- utils/fix_data_feat.sh ${fbankdir}/train
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
- ${feats_dir}/data/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev
- utils/fix_data_feat.sh ${fbankdir}/dev
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
- ${feats_dir}/data/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test
- utils/fix_data_feat.sh ${fbankdir}/test
-
- echo "apply low_frame_rate and cmvn"
- [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1;
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/train ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/dev ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/test ${cmvn_file} ${exp_dir}/exp/make_fbank/test ${feat_test_dir}
-
- echo "Text Tokenize"
- # 鎴戠埍reading->鎴� 鐖� read@@ ing
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/train ${seg_file} ${feat_train_dir}/log ${feat_train_dir}
- utils/fix_data_feat.sh ${feat_train_dir}
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/dev ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir}
- utils/fix_data_feat.sh ${feat_dev_dir}
- cp ${fbankdir}/test/text ${feat_test_dir}
-fi
-
-token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
-echo "dictionary: ${token_list}"
-if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
- echo "stage 2: Dictionary Preparation"
- mkdir -p ${feats_dir}/data/${lang}_token_list/char/
- cp $vocab ${token_list}
-
- vocab_size=$(wc -l <${token_list})
- awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
- awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev
- cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train
- cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev
-fi
-
-# Training Stage
-world_size=$gpu_num # run on one machine
-if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
- echo "stage 3: Training"
- # update asr train config.yaml
- python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/finetune.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml
- finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml
-
- mkdir -p ${exp_dir}/exp/${model_dir}
- mkdir -p ${exp_dir}/exp/${model_dir}/log
- INIT_FILE=$exp_dir/ddp_init
- if [ -f $INIT_FILE ];then
- rm -f $INIT_FILE
- fi
- init_method=file://$(readlink -f $INIT_FILE)
- echo "$0: init method is $init_method"
- for ((i = 0; i < $gpu_num; ++i)); do
- {
- rank=$i
- local_rank=$i
- gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
- asr_train_paraformer.py \
- --gpu_id $gpu_id \
- --use_preprocessor true \
- --token_type $token_type \
- --token_list $token_list \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \
- --resume true \
- --output_dir ${exp_dir}/exp/${model_dir} \
- --init_param $init_param \
- --config $finetune_config \
- --input_size $feats_dim \
- --ngpu $gpu_num \
- --num_worker_count $count \
- --multiprocessing_distributed true \
- --dist_init_method $init_method \
- --dist_world_size $world_size \
- --dist_rank $rank \
- --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
- } &
- done
- wait
-fi
-
-# Testing Stage
-if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
- echo "stage 4: Inference"
- for dset in ${test_sets}; do
- asr_exp=${exp_dir}/exp/${model_dir}
- inference_tag="$(basename "${inference_config}" .yaml)"
- _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
- _logdir="${_dir}/logdir"
- if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
- exit 0
- fi
- mkdir -p "${_logdir}"
- _data="${feats_dir}/${dumpdir}/${dset}"
- key_file=${_data}/${scp}
- num_scp_file="$(<${key_file} wc -l)"
- _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
- split_scps=
- for n in $(seq "${_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
- done
- # shellcheck disable=SC2086
- utils/split_scp.pl "${key_file}" ${split_scps}
- _opts=
- if [ -n "${inference_config}" ]; then
- _opts+="--config ${inference_config} "
- fi
- ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.asr_inference_launch \
- --batch_size 64 \
- --ngpu "${_ngpu}" \
- --njob ${njob} \
- --gpuid_list ${gpuid_list:0:1} \
- --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
- --key_file "${_logdir}"/keys.JOB.scp \
- --asr_train_config "${asr_exp}"/config.yaml \
- --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
- --output_dir "${_logdir}"/output.JOB \
- --mode paraformer \
- ${_opts}
-
- for f in token token_int score text; do
- if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
- for i in $(seq "${_nj}"); do
- cat "${_logdir}/output.${i}/1best_recog/${f}"
- done | sort -k1 >"${_dir}/${f}"
- fi
- done
- python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
- python utils/proce_text.py ${_data}/text ${_data}/text.proc
- python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
- tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
- cat ${_dir}/text.cer.txt
- done
-fi
-
-
diff --git a/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh b/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh
deleted file mode 100755
index 711f6fc..0000000
--- a/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-ori_data=
-data_dir=
-exp_dir=
-model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch
-model_revision="v1.0.4" # please do not modify the model revision
-inference_nj=32
-gpuid_list="0" # set gpus, e.g., gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1 # the number of jobs for each gpu
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
-else
- inference_nj=$njob
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-test_sets="dev test"
-
-. utils/parse_options.sh
-
-aishell_audio_dir=$ori_data/data_aishell/wav
-aishell_text=$ori_data/data_aishell/transcript/aishell_transcript_v0.8.txt
-dev_dir=${data_dir}/aishell/dev
-test_dir=${data_dir}/aishell/test
-tmp_dir=${data_dir}/aishell/tmp
-
-mkdir -p ${dev_dir}
-mkdir -p ${test_dir}
-mkdir -p ${tmp_dir}
-
-find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist
-grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1;
-grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1;
-
-rm -r $tmp_dir
-
-for dir in $dev_dir $test_dir; do
- sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list
- paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all
- utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt
- awk '{print $1}' $dir/transcripts.txt > $dir/utt.list
- utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp
- sort -u $dir/transcripts.txt > $dir/text
-done
-
-mkdir -p ${exp_dir}/aishell
-
-modelscope_utils/modelscope_infer.sh \
- --data_dir ${data_dir}/aishell \
- --exp_dir ${exp_dir}/aishell \
- --test_sets "${test_sets}" \
- --model_name ${model_name} \
- --model_revision ${model_revision} \
- --inference_nj ${inference_nj} \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --gpu_inference ${gpu_inference} \
- --use_lm ${use_lm} \
- --beam_size ${beam_size} \
- --lm_weight ${lm_weight}
diff --git a/egs_modelscope/aishell/paraformer/utils b/egs_modelscope/aishell/paraformer/utils
deleted file mode 120000
index 4e7a6ce..0000000
--- a/egs_modelscope/aishell/paraformer/utils
+++ /dev/null
@@ -1 +0,0 @@
-../../../egs/aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs_modelscope/aishell2/paraformer/README.md b/egs_modelscope/aishell2/paraformer/README.md
deleted file mode 100644
index 46bd3ad..0000000
--- a/egs_modelscope/aishell2/paraformer/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# ModelScope: Paraformer-large Model
-
-## Highlight
-
-### ModelScope: Paraformer-Large Model
-- <strong>Fast</strong>: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup.
-- <strong>Accurate</strong>: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation.
-- <strong>Convenient</strong>: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference.
- - Support finetuning and inference on AISHELL-1 and AISHELL-2.
- - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio.
-
-## How to finetune and infer using a pretrained ModelScope Paraformer-large Model
-
-### Finetune
-- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml`
-- Setting parameters in `paraformer_large_finetune.sh`
- - <strong>tr_dir:</strong> please set the aishell2 train data path
- - <strong>dev_tst_dir:</strong> please set the aishell2 dev/test data path
- - <strong>tag:</strong> exp tag
- - <strong>init_model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-- Then you can run the pipeline to finetune with our model download from modelscope and infer after finetune:
-```sh
- sh ./paraformer_large_finetune.sh
-```
-
-### Inference
-
-Or you can download the model from ModelScope for inference directly.
-
-- Setting parameters in `paraformer_large_infer.sh`
- - <strong>ori_data:</strong> please set the aishell2 dev/test raw data path
- - <strong>data_dir:</strong> data output dictionary
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope
- - <strong>test_sets:</strong> please set the testsets name
-- Then you can run the pipeline to infer with:
-```sh
- sh ./paraformer_large_infer.sh
-```
diff --git a/egs_modelscope/aishell2/paraformer/RESULTS.md b/egs_modelscope/aishell2/paraformer/RESULTS.md
deleted file mode 100644
index a265a74..0000000
--- a/egs_modelscope/aishell2/paraformer/RESULTS.md
+++ /dev/null
@@ -1,26 +0,0 @@
-# Paraformer-Large
-- Model link: <https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary>
-- Model size: 220M
-- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-
-# Environments
-- date: `Tue Nov 22 18:48:39 CST 2022`
-- python version: `3.7.12`
-- FunASR version: `0.1.0`
-- pytorch version: `pytorch 1.7.0`
-- Git hash: ``
-- Commit date: ``
-
-# Beachmark Results
-
-## AISHELL-2
-- Decode config: conf/decode_asr_transformer_noctc_1best.yaml
- - Decode without CTC
- - Decode without LM
-
-| testset | CER(%)|
-|:------------:|:-----:|
-| dev_ios | 2.80 |
-| test_android | 3.13 |
-| test_ios | 2.85 |
-| test_mic | 3.06 |
diff --git a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
deleted file mode 100644
index 22f02d9..0000000
--- a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 10
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.15
diff --git a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
deleted file mode 100644
index e623192..0000000
--- a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 1
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.0
diff --git a/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
deleted file mode 100644
index c8990eb..0000000
--- a/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
+++ /dev/null
@@ -1,92 +0,0 @@
-# network architecture
-# encoder related
-encoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- attention_dropout_rate: 0.1
-
-# decoder related
-decoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- self_attention_dropout_rate: 0.1
- src_attention_dropout_rate: 0.1
-
-predictor_conf:
- threshold: 1.0
- l_order: 1
- r_order: 1
- tail_threshold: 0.45
-
-# hybrid CTC/attention
-model_conf:
- ctc_weight: 0.0
- lsm_weight: 0.1 # label smoothing option
- length_normalized_loss: true
- predictor_weight: 1.0
- predictor_bias: 1
- sampling_ratio: 0.75
-
-
-# minibatch related
-# dataset_type: small
-batch_type: length
-batch_bins: 2000
-num_workers: 16
-# dataset_type: large
-dataset_conf:
- filter_conf:
- min_length: 10
- max_length: 250
- min_token_length: 1
- max_token_length: 200
- shuffle: true
- shuffle_conf:
- shuffle_size: 10240
- sort_size: 500
- batch_conf:
- batch_type: 'token'
- batch_size: 6000
- num_workers: 16
-
-# optimization related
-accum_grad: 1
-grad_clip: 5
-max_epoch: 20
-val_scheduler_criterion:
- - valid
- - acc
-best_model_criterion:
-- - valid
- - acc
- - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
- lr: 0.0005
-scheduler: warmuplr
-scheduler_conf:
- warmup_steps: 30000
-
-specaug: specaug_lfr
-specaug_conf:
- apply_time_warp: false
- time_warp_window: 5
- time_warp_mode: bicubic
- apply_freq_mask: true
- freq_mask_width_range:
- - 0
- - 30
- lfr_rate: 6
- num_freq_mask: 1
- apply_time_mask: true
- time_mask_width_range:
- - 0
- - 12
- num_time_mask: 1
-
-unused_parameters: true
-log_interval: 50
-normalize: None
-split_with_space: true
diff --git a/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh b/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh
deleted file mode 100755
index 77791f9..0000000
--- a/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG)
-# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU)
-# Apache 2.0
-
-# transform raw AISHELL-2 data to kaldi format
-
-. ./path.sh || exit 1;
-
-tmp=
-dir=
-
-if [ $# != 3 ]; then
- echo "Usage: $0 <corpus-data-dir> <tmp-dir> <output-dir>"
- echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train"
- exit 1;
-fi
-
-corpus=$1
-tmp=$2
-dir=$3
-
-echo "prepare_data.sh: Preparing data in $corpus"
-
-mkdir -p $tmp
-mkdir -p $dir
-
-# corpus check
-if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then
- echo "Error: $0 requires wav.scp and trans.txt under $corpus directory."
- exit 1;
-fi
-
-# validate utt-key list, IC0803W0380 is a bad utterance
-awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list
-awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list
-utils/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list
-
-# wav.scp
-awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp
-utils/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp
-
-# text
-utils/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text
-
-# copy prepared resources from tmp_dir to target dir
-mkdir -p $dir
-for f in wav.scp text; do
- cp $tmp/$f $dir/$f || exit 1;
-done
-
-echo "local/prepare_data.sh succeeded"
-exit 0;
diff --git a/egs_modelscope/aishell2/paraformer/modelscope_utils b/egs_modelscope/aishell2/paraformer/modelscope_utils
deleted file mode 120000
index fc97768..0000000
--- a/egs_modelscope/aishell2/paraformer/modelscope_utils
+++ /dev/null
@@ -1 +0,0 @@
-../../common/modelscope_utils
\ No newline at end of file
diff --git a/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh b/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh
deleted file mode 100755
index 55935f0..0000000
--- a/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh
+++ /dev/null
@@ -1,281 +0,0 @@
-#!/usr/bin/env bash
-
-. ./path.sh || exit 1;
-
-# machines configuration
-CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1"
-gpu_num=2
-count=1
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-njob=1 # the number of jobs for each gpu
-train_cmd=utils/run.pl
-infer_cmd=utils/run.pl
-
-# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
-exp_dir="."
-lang=zh
-dumpdir=dump/fbank
-feats_type=fbank
-token_type=char
-scp=feats.scp
-type=kaldi_ark
-stage=0
-stop_stage=4
-
-# feature configuration
-feats_dim=560
-sample_frequency=16000
-nj=100
-speed_perturb="1.0"
-lfr=True
-lfr_m=7
-lfr_n=6
-
-init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-model_revision="v1.0.4" # please do not modify the model revision
-cmvn_file=init_model/${init_model_name}/am.mvn
-seg_file=init_model/${init_model_name}/seg_dict
-vocab=init_model/${init_model_name}/tokens.txt
-
-# data
-tr_dir=
-dev_tst_dir=
-
-# exp tag
-tag=""
-
-# Set bash to 'debug' mode, it will exit on :
-# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
-set -e
-set -u
-set -o pipefail
-
-train_set=train
-valid_set=dev_ios
-test_sets="dev_ios test_android test_ios test_mic"
-
-asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-init_param="init_model/${init_model_name}/model.pb"
-
-inference_config=conf/decode_asr_transformer_noctc_1best.yaml
-inference_asr_model=valid.acc.ave_10best.pth
-
-. utils/parse_options.sh || exit 1;
-
-# download model from modelscope
-python modelscope_utils/download_model.py --model_name ${init_model_name} --model_revision ${model_revision}
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist"
- exit 1
-else
- if [ -d init_model/${init_model_name} ]; then
- echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first."
- else
- mkdir -p init_model/${init_model_name}
- cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name}
- fi
-fi
-
-model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
-
-# you can set gpu num for decoding here
-gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-
-if ${gpu_inference}; then
- inference_nj=$njob
- _ngpu=1
-else
- inference_nj=$njob
- _ngpu=0
-fi
-
-if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
- echo "stage 0: Data preparation"
- # For training set
- local/aishell2_data_prep.sh ${tr_dir} ${feats_dir}/data/local/train ${feats_dir}/data/train || exit 1;
- # # For dev and test set
- for x in Android iOS Mic; do
- local/aishell2_data_prep.sh ${dev_tst_dir}/${x}/dev ${feats_dir}/data/local/dev_${x,,} ${feats_dir}/data/dev_${x,,} || exit 1;
- local/aishell2_data_prep.sh ${dev_tst_dir}/${x}/test ${feats_dir}/data/local/test_${x,,} ${feats_dir}/data/test_${x,,} || exit 1;
- done
- # Normalize text to capital letters
- for x in train dev_android dev_ios dev_mic test_android test_ios test_mic; do
- mv ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org
- paste -d " " <(cut -f 1 ${feats_dir}/data/${x}/text.org) <(cut -f 2- ${feats_dir}/data/${x}/text.org \
- | tr 'A-Z' 'a-z' | tr -d " ") \
- > ${feats_dir}/data/${x}/text
- rm ${feats_dir}/data/${x}/text.org
- done
-fi
-
-feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir}
-feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir}
-if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
- echo "stage 1: Feature Generation"
- # compute fbank features
- fbankdir=${feats_dir}/fbank
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \
- ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
- utils/fix_data_feat.sh ${fbankdir}/train
- for x in android ios mic; do
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
- ${feats_dir}/data/dev_${x} ${exp_dir}/exp/make_fbank/dev_${x} ${fbankdir}/dev_${x}
- utils/fix_data_feat.sh ${fbankdir}/dev_${x}
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \
- ${feats_dir}/data/test_${x} ${exp_dir}/exp/make_fbank/test_${x} ${fbankdir}/test_${x}
- utils/fix_data_feat.sh ${fbankdir}/test_${x}
- done
-
- echo "apply low_frame_rate and cmvn"
- [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1;
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/${train_set} ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/${valid_set} ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir}
- for x in android ios mic; do
- feat_test_dir=${feats_dir}/${dumpdir}/test_${x}; mkdir ${feat_test_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/test_${x} ${cmvn_file} ${exp_dir}/exp/make_fbank/test_${x} ${feat_test_dir}
- done
-
- echo "Text Tokenize"
- # 鎴戠埍reading->鎴� 鐖� read@@ ing
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/${train_set} ${seg_file} ${feat_train_dir}/log ${feat_train_dir}
- utils/fix_data_feat.sh ${feat_train_dir}
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/${valid_set} ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir}
- utils/fix_data_feat.sh ${feat_dev_dir}
- for x in android ios mic; do
- feat_test_dir=${feats_dir}/${dumpdir}/test_${x}
- cp ${fbankdir}/test_${x}/text ${feat_test_dir}
- done
-fi
-
-token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
-echo "dictionary: ${token_list}"
-if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
- echo "stage 2: Dictionary Preparation"
- mkdir -p ${feats_dir}/data/${lang}_token_list/char/
- cp $vocab ${token_list}
-
- vocab_size=$(wc -l <${token_list})
- awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
- awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev_ios
- cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${train_set}
- cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}
-fi
-
-# Training Stage
-world_size=$gpu_num # run on one machine
-if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
- echo "stage 3: Training"
- # update asr train config.yaml
- python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/finetune.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml
- finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml
-
- mkdir -p ${exp_dir}/exp/${model_dir}
- mkdir -p ${exp_dir}/exp/${model_dir}/log
- INIT_FILE=$exp_dir/ddp_init
- if [ -f $INIT_FILE ];then
- rm -f $INIT_FILE
- fi
- init_method=file://$(readlink -f $INIT_FILE)
- echo "$0: init method is $init_method"
- for ((i = 0; i < $gpu_num; ++i)); do
- {
- rank=$i
- local_rank=$i
- gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
- asr_train_paraformer.py \
- --gpu_id $gpu_id \
- --use_preprocessor true \
- --token_type $token_type \
- --token_list $token_list \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \
- --resume true \
- --output_dir ${exp_dir}/exp/${model_dir} \
- --init_param $init_param \
- --config $finetune_config \
- --input_size $feats_dim \
- --ngpu $gpu_num \
- --num_worker_count $count \
- --multiprocessing_distributed true \
- --dist_init_method $init_method \
- --dist_world_size $world_size \
- --dist_rank $rank \
- --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
- } &
- done
- wait
-fi
-
-# Testing Stage
-if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
- echo "stage 4: Inference"
- for dset in ${test_sets}; do
- asr_exp=${exp_dir}/exp/${model_dir}
- inference_tag="$(basename "${inference_config}" .yaml)"
- _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
- _logdir="${_dir}/logdir"
- if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
- exit 0
- fi
- mkdir -p "${_logdir}"
- _data="${feats_dir}/${dumpdir}/${dset}"
- key_file=${_data}/${scp}
- num_scp_file="$(<${key_file} wc -l)"
- _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
- split_scps=
- for n in $(seq "${_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
- done
- # shellcheck disable=SC2086
- utils/split_scp.pl "${key_file}" ${split_scps}
- _opts=
- if [ -n "${inference_config}" ]; then
- _opts+="--config ${inference_config} "
- fi
- ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.asr_inference_launch \
- --batch_size 64 \
- --ngpu "${_ngpu}" \
- --njob ${njob} \
- --gpuid_list ${gpuid_list:0:1} \
- --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
- --key_file "${_logdir}"/keys.JOB.scp \
- --asr_train_config "${asr_exp}"/config.yaml \
- --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
- --output_dir "${_logdir}"/output.JOB \
- --mode paraformer \
- ${_opts}
-
- for f in token token_int score text; do
- if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
- for i in $(seq "${_nj}"); do
- cat "${_logdir}/output.${i}/1best_recog/${f}"
- done | sort -k1 >"${_dir}/${f}"
- fi
- done
- python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
- python utils/proce_text.py ${_data}/text ${_data}/text.proc
- python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
- tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
- cat ${_dir}/text.cer.txt
- done
-fi
-
diff --git a/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh b/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh
deleted file mode 100755
index 86c9673..0000000
--- a/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-ori_data=
-data_dir=
-exp_dir=
-model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch
-model_revision="v1.0.4" # please do not modify the model revision
-inference_nj=32
-gpuid_list="0" # set gpus, e.g., gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1 # the number of jobs for each gpu
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
-else
- inference_nj=$njob
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-test_sets="dev_ios test_android test_ios test_mic"
-
-. utils/parse_options.sh
-
-for x in Android iOS Mic; do
- local/aishell2_data_prep.sh ${ori_data}/${x}/dev ${data_dir}/aishell2/local/dev_${x,,} ${data_dir}/aishell2/dev_${x,,} || exit 1;
- local/aishell2_data_prep.sh ${ori_data}/${x}/test ${data_dir}/aishell2/local/test_${x,,} ${data_dir}/aishell2/test_${x,,} || exit 1;
-done
-for x in dev_android dev_ios dev_mic test_android test_ios test_mic; do
- mv ${data_dir}/aishell2/${x}/text ${data_dir}/aishell2/${x}/text.org
- paste -d " " <(cut -f 1 ${data_dir}/aishell2/${x}/text.org) <(cut -f 2- ${data_dir}/aishell2/${x}/text.org \
- | tr 'A-Z' 'a-z' | tr -d " ") \
- > ${data_dir}/aishell2/${x}/text
- rm ${data_dir}/aishell2/${x}/text.org
-done
-
-mkdir -p ${exp_dir}/aishell2
-
-modelscope_utils/modelscope_infer.sh \
- --data_dir ${data_dir}/aishell2 \
- --exp_dir ${exp_dir}/aishell2 \
- --test_sets "${test_sets}" \
- --model_name ${model_name} \
- --model_revision ${model_revision} \
- --inference_nj ${inference_nj} \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --gpu_inference ${gpu_inference} \
- --use_lm ${use_lm} \
- --beam_size ${beam_size} \
- --lm_weight ${lm_weight}
diff --git a/egs_modelscope/aishell2/paraformer/path.sh b/egs_modelscope/aishell2/paraformer/path.sh
deleted file mode 100755
index 7972642..0000000
--- a/egs_modelscope/aishell2/paraformer/path.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-export FUNASR_DIR=$PWD/../../..
-
-# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8
-export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs_modelscope/aishell2/paraformer/utils b/egs_modelscope/aishell2/paraformer/utils
deleted file mode 120000
index 4e7a6ce..0000000
--- a/egs_modelscope/aishell2/paraformer/utils
+++ /dev/null
@@ -1 +0,0 @@
-../../../egs/aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py
new file mode 100644
index 0000000..6b221c4
--- /dev/null
+++ b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 25000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
new file mode 100644
index 0000000..3594815
--- /dev/null
+++ b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py
new file mode 100644
index 0000000..7932045
--- /dev/null
+++ b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch",
+ data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 25000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.0005 # 璁剧疆瀛︿範鐜�
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
new file mode 100644
index 0000000..b55b59f
--- /dev/null
+++ b/egs_modelscope/asr/conformer/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://modelscope.oss-cn-beijing.aliyuncs.com/test/audios/asr_example.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/README.md b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/README.md
new file mode 100644
index 0000000..c2e4354
--- /dev/null
+++ b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/README.md
@@ -0,0 +1,53 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: `train/wav.scp`, `train/text`; `validation/wav.scp`, `validation/text`
+ - <strong>dataset_type:</strong> # for dataset larger than 1000 hours, set as `large`, otherwise set as `small`
+ - <strong>batch_bins:</strong> # batch size. For dataset_type is `small`, `batch_bins` indicates the feature frames. For dataset_type is `large`, `batch_bins` indicates the duration in ms
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>data_dir:</strong> # the dataset dir needs to include `test/wav.scp`. If `test/text` is also exists, CER will be computed
+ - <strong>output_dir:</strong> # result dir
+ - <strong>ngpu:</strong> # the number of GPUs for decoding
+ - <strong>njob:</strong> # the number of jobs for each GPU
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
+
+- Results
+
+The decoding results can be found in `$output_dir/1best_recog/text.cer`, which includes recognition results of each sample and the CER metric of the whole test set.
+
+### Inference using local finetuned model
+
+- Modify inference related parameters in `infer_after_finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include `test/wav.scp`. If `test/text` is also exists, CER will be computed~~~~
+ - <strong>decoding_model_name:</strong> # set the checkpoint name for decoding, e.g., `valid.cer_ctc.ave.pth`
+
+- Then you can run the pipeline to finetune with:
+```python
+ python infer_after_finetune.py
+```
+
+- Results
+
+The decoding results can be found in `$output_dir/decoding_results/text.cer`, which includes recognition results of each sample and the CER metric of the whole test set.
diff --git a/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/finetune.py b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/finetune.py
new file mode 100644
index 0000000..005cae6
--- /dev/null
+++ b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/finetune.py
@@ -0,0 +1,37 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch",
+ data_path="./data")
+ params.output_dir = "./checkpoint"
+ params.data_path = "./example_data/"
+ params.dataset_type = "small"
+ params.batch_bins = 16000
+ params.max_epoch = 50
+ params.lr = 0.00005
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer.py b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer.py
new file mode 100644
index 0000000..54cfec0
--- /dev/null
+++ b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer.py
@@ -0,0 +1,87 @@
+import os
+import shutil
+from multiprocessing import Pool
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_core(output_dir, split_dir, njob, idx):
+ output_dir_job = os.path.join(output_dir, "output.{}".format(idx))
+ gpu_id = (int(idx) - 1) // njob
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[gpu_id])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch",
+ output_dir=output_dir_job,
+ )
+ audio_in = os.path.join(split_dir, "wav.{}.scp".format(idx))
+ inference_pipline(audio_in=audio_in)
+
+
+def modelscope_infer(params):
+ # prepare for multi-GPU decoding
+ ngpu = params["ngpu"]
+ njob = params["njob"]
+ output_dir = params["output_dir"]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.mkdir(output_dir)
+ split_dir = os.path.join(output_dir, "split")
+ os.mkdir(split_dir)
+ nj = ngpu * njob
+ wav_scp_file = os.path.join(params["data_dir"], "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(split_dir, "wav.{}.scp".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(modelscope_infer_core,
+ args=(output_dir, split_dir, njob, str(i + 1)))
+ p.close()
+ p.join()
+
+ # combine decoding results
+ best_recog_path = os.path.join(output_dir, "1best_recog")
+ os.mkdir(best_recog_path)
+ files = ["text", "token", "score"]
+ for file in files:
+ with open(os.path.join(best_recog_path, file), "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(output_dir, "output.{}/1best_recog".format(str(i + 1)), file)
+ with open(job_file) as f_job:
+ lines = f_job.readlines()
+ f.writelines(lines)
+
+ # If text exists, compute CER
+ text_in = os.path.join(params["data_dir"], "text")
+ if os.path.exists(text_in):
+ text_proc_file = os.path.join(best_recog_path, "token")
+ compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))
+
+
+if __name__ == "__main__":
+ params = {}
+ params["data_dir"] = "./data/test"
+ params["output_dir"] = "./results"
+ params["ngpu"] = 2
+ params["njob"] = 5
+ modelscope_infer(params)
diff --git a/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer_after_finetune.py b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer_after_finetune.py
new file mode 100644
index 0000000..b37fc97
--- /dev/null
+++ b/egs_modelscope/asr/data2vec/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch/infer_after_finetune.py
@@ -0,0 +1,52 @@
+import json
+import os
+import shutil
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_after_finetune(params):
+ # prepare for decoding
+ pretrained_model_path = os.path.join(os.environ["HOME"], ".cache/modelscope/hub", params["modelscope_model_name"])
+ for file_name in params["required_files"]:
+ if file_name == "configuration.json":
+ with open(os.path.join(pretrained_model_path, file_name)) as f:
+ config_dict = json.load(f)
+ config_dict["model"]["am_model_name"] = params["decoding_model_name"]
+ with open(os.path.join(params["output_dir"], "configuration.json"), "w") as f:
+ json.dump(config_dict, f, indent=4, separators=(',', ': '))
+ else:
+ shutil.copy(os.path.join(pretrained_model_path, file_name),
+ os.path.join(params["output_dir"], file_name))
+ decoding_path = os.path.join(params["output_dir"], "decode_results")
+ if os.path.exists(decoding_path):
+ shutil.rmtree(decoding_path)
+ os.mkdir(decoding_path)
+
+ # decoding
+ inference_pipeline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model=params["output_dir"],
+ output_dir=decoding_path,
+ )
+ audio_in = os.path.join(params["data_dir"], "wav.scp")
+ inference_pipeline(audio_in=audio_in)
+
+ # computer CER if GT text is set
+ text_in = os.path.join(params["data_dir"], "text")
+ if text_in is not None:
+ text_proc_file = os.path.join(decoding_path, "1best_recog/token")
+ compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))
+
+
+if __name__ == '__main__':
+ params = {}
+ params["modelscope_model_name"] = "damo/speech_data2vec_pretrain-zh-cn-aishell2-16k-pytorch"
+ params["required_files"] = ["am.mvn", "decoding.yaml", "configuration.json"]
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data/test"
+ params["decoding_model_name"] = "valid.cer_ctc.ave.pth"
+ modelscope_infer_after_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
new file mode 100644
index 0000000..b238123
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
new file mode 100644
index 0000000..2703c2a
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
@@ -0,0 +1,15 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ output_dir=output_dir,
+ batch_size=1,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/README.md b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/README.md
new file mode 100644
index 0000000..1587d3d
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>data_dir:</strong> # the dataset dir
+ - <strong>output_dir:</strong> # result dir
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/finetune.py
new file mode 100644
index 0000000..5817f0e
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer.py
new file mode 100644
index 0000000..1aab1a0
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer.py
@@ -0,0 +1,88 @@
+import os
+import shutil
+from multiprocessing import Pool
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_core(output_dir, split_dir, njob, idx):
+ output_dir_job = os.path.join(output_dir, "output.{}".format(idx))
+ gpu_id = (int(idx) - 1) // njob
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[gpu_id])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch",
+ output_dir=output_dir_job,
+ batch_size=64
+ )
+ audio_in = os.path.join(split_dir, "wav.{}.scp".format(idx))
+ inference_pipline(audio_in=audio_in)
+
+
+def modelscope_infer(params):
+ # prepare for multi-GPU decoding
+ ngpu = params["ngpu"]
+ njob = params["njob"]
+ output_dir = params["output_dir"]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.mkdir(output_dir)
+ split_dir = os.path.join(output_dir, "split")
+ os.mkdir(split_dir)
+ nj = ngpu * njob
+ wav_scp_file = os.path.join(params["data_dir"], "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(split_dir, "wav.{}.scp".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(modelscope_infer_core,
+ args=(output_dir, split_dir, njob, str(i + 1)))
+ p.close()
+ p.join()
+
+ # combine decoding results
+ best_recog_path = os.path.join(output_dir, "1best_recog")
+ os.mkdir(best_recog_path)
+ files = ["text", "token", "score"]
+ for file in files:
+ with open(os.path.join(best_recog_path, file), "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(output_dir, "output.{}".format(str(i + 1)), file)
+ with open(job_file) as f_job:
+ lines = f_job.readlines()
+ f.writelines(lines)
+
+ # If text exists, compute CER
+ text_in = os.path.join(params["data_dir"], "text")
+ if os.path.exists(text_in):
+ text_proc_file = os.path.join(best_recog_path, "token")
+ compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))
+
+
+if __name__ == "__main__":
+ params = {}
+ params["data_dir"] = "./data/test"
+ params["output_dir"] = "./results"
+ params["ngpu"] = 1
+ params["njob"] = 1
+ modelscope_infer(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer_after_finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer_after_finetune.py
new file mode 100644
index 0000000..abe4d56
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch/infer_after_finetune.py
@@ -0,0 +1,53 @@
+import json
+import os
+import shutil
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_after_finetune(params):
+ # prepare for decoding
+ pretrained_model_path = os.path.join(os.environ["HOME"], ".cache/modelscope/hub", params["modelscope_model_name"])
+ for file_name in params["required_files"]:
+ if file_name == "configuration.json":
+ with open(os.path.join(pretrained_model_path, file_name)) as f:
+ config_dict = json.load(f)
+ config_dict["model"]["am_model_name"] = params["decoding_model_name"]
+ with open(os.path.join(params["output_dir"], "configuration.json"), "w") as f:
+ json.dump(config_dict, f, indent=4, separators=(',', ': '))
+ else:
+ shutil.copy(os.path.join(pretrained_model_path, file_name),
+ os.path.join(params["output_dir"], file_name))
+ decoding_path = os.path.join(params["output_dir"], "decode_results")
+ if os.path.exists(decoding_path):
+ shutil.rmtree(decoding_path)
+ os.mkdir(decoding_path)
+
+ # decoding
+ inference_pipeline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model=params["output_dir"],
+ output_dir=decoding_path,
+ batch_size=64
+ )
+ audio_in = os.path.join(params["data_dir"], "wav.scp")
+ inference_pipeline(audio_in=audio_in)
+
+ # computer CER if GT text is set
+ text_in = os.path.join(params["data_dir"], "text")
+ if text_in is not None:
+ text_proc_file = os.path.join(decoding_path, "1best_recog/token")
+ compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))
+
+
+if __name__ == '__main__':
+ params = {}
+ params["modelscope_model_name"] = "damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell1-vocab8404-pytorch"
+ params["required_files"] = ["am.mvn", "decoding.yaml", "configuration.json"]
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data/test"
+ params["decoding_model_name"] = "valid.acc.ave_10best.pth"
+ modelscope_infer_after_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/README.md b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/README.md
new file mode 100644
index 0000000..1587d3d
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>data_dir:</strong> # the dataset dir
+ - <strong>output_dir:</strong> # result dir
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/finetune.py
new file mode 100644
index 0000000..c46d676
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer.py
new file mode 100644
index 0000000..a625202
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer.py
@@ -0,0 +1,88 @@
+import os
+import shutil
+from multiprocessing import Pool
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_core(output_dir, split_dir, njob, idx):
+ output_dir_job = os.path.join(output_dir, "output.{}".format(idx))
+ gpu_id = (int(idx) - 1) // njob
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[gpu_id])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch",
+ output_dir=output_dir_job,
+ batch_size=64
+ )
+ audio_in = os.path.join(split_dir, "wav.{}.scp".format(idx))
+ inference_pipline(audio_in=audio_in)
+
+
+def modelscope_infer(params):
+ # prepare for multi-GPU decoding
+ ngpu = params["ngpu"]
+ njob = params["njob"]
+ output_dir = params["output_dir"]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.mkdir(output_dir)
+ split_dir = os.path.join(output_dir, "split")
+ os.mkdir(split_dir)
+ nj = ngpu * njob
+ wav_scp_file = os.path.join(params["data_dir"], "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(split_dir, "wav.{}.scp".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(modelscope_infer_core,
+ args=(output_dir, split_dir, njob, str(i + 1)))
+ p.close()
+ p.join()
+
+ # combine decoding results
+ best_recog_path = os.path.join(output_dir, "1best_recog")
+ os.mkdir(best_recog_path)
+ files = ["text", "token", "score"]
+ for file in files:
+ with open(os.path.join(best_recog_path, file), "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(output_dir, "output.{}".format(str(i + 1)), file)
+ with open(job_file) as f_job:
+ lines = f_job.readlines()
+ f.writelines(lines)
+
+ # If text exists, compute CER
+ text_in = os.path.join(params["data_dir"], "text")
+ if os.path.exists(text_in):
+ text_proc_file = os.path.join(best_recog_path, "token")
+ compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))
+
+
+if __name__ == "__main__":
+ params = {}
+ params["data_dir"] = "./data/test"
+ params["output_dir"] = "./results"
+ params["ngpu"] = 1
+ params["njob"] = 1
+ modelscope_infer(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer_after_finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer_after_finetune.py
new file mode 100644
index 0000000..547b223
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch/infer_after_finetune.py
@@ -0,0 +1,53 @@
+import json
+import os
+import shutil
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_after_finetune(params):
+ # prepare for decoding
+ pretrained_model_path = os.path.join(os.environ["HOME"], ".cache/modelscope/hub", params["modelscope_model_name"])
+ for file_name in params["required_files"]:
+ if file_name == "configuration.json":
+ with open(os.path.join(pretrained_model_path, file_name)) as f:
+ config_dict = json.load(f)
+ config_dict["model"]["am_model_name"] = params["decoding_model_name"]
+ with open(os.path.join(params["output_dir"], "configuration.json"), "w") as f:
+ json.dump(config_dict, f, indent=4, separators=(',', ': '))
+ else:
+ shutil.copy(os.path.join(pretrained_model_path, file_name),
+ os.path.join(params["output_dir"], file_name))
+ decoding_path = os.path.join(params["output_dir"], "decode_results")
+ if os.path.exists(decoding_path):
+ shutil.rmtree(decoding_path)
+ os.mkdir(decoding_path)
+
+ # decoding
+ inference_pipeline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model=params["output_dir"],
+ output_dir=decoding_path,
+ batch_size=64
+ )
+ audio_in = os.path.join(params["data_dir"], "wav.scp")
+ inference_pipeline(audio_in=audio_in)
+
+ # computer CER if GT text is set
+ text_in = os.path.join(params["data_dir"], "text")
+ if text_in is not None:
+ text_proc_file = os.path.join(decoding_path, "1best_recog/token")
+ compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))
+
+
+if __name__ == '__main__':
+ params = {}
+ params["modelscope_model_name"] = "damo/speech_paraformer-large_asr_nat-zh-cn-16k-aishell2-vocab8404-pytorch"
+ params["required_files"] = ["am.mvn", "decoding.yaml", "configuration.json"]
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data/test"
+ params["decoding_model_name"] = "valid.acc.ave_10best.pth"
+ modelscope_infer_after_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
new file mode 100644
index 0000000..1587d3d
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>data_dir:</strong> # the dataset dir
+ - <strong>output_dir:</strong> # result dir
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
new file mode 100644
index 0000000..1935258
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
new file mode 100644
index 0000000..dc37081
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
@@ -0,0 +1,88 @@
+import os
+import shutil
+from multiprocessing import Pool
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_core(output_dir, split_dir, njob, idx):
+ output_dir_job = os.path.join(output_dir, "output.{}".format(idx))
+ gpu_id = (int(idx) - 1) // njob
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[gpu_id])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ output_dir=output_dir_job,
+ batch_size=64
+ )
+ audio_in = os.path.join(split_dir, "wav.{}.scp".format(idx))
+ inference_pipline(audio_in=audio_in)
+
+
+def modelscope_infer(params):
+ # prepare for multi-GPU decoding
+ ngpu = params["ngpu"]
+ njob = params["njob"]
+ output_dir = params["output_dir"]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.mkdir(output_dir)
+ split_dir = os.path.join(output_dir, "split")
+ os.mkdir(split_dir)
+ nj = ngpu * njob
+ wav_scp_file = os.path.join(params["data_dir"], "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(split_dir, "wav.{}.scp".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(modelscope_infer_core,
+ args=(output_dir, split_dir, njob, str(i + 1)))
+ p.close()
+ p.join()
+
+ # combine decoding results
+ best_recog_path = os.path.join(output_dir, "1best_recog")
+ os.mkdir(best_recog_path)
+ files = ["text", "token", "score"]
+ for file in files:
+ with open(os.path.join(best_recog_path, file), "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(output_dir, "output.{}".format(str(i + 1)), file)
+ with open(job_file) as f_job:
+ lines = f_job.readlines()
+ f.writelines(lines)
+
+ # If text exists, compute CER
+ text_in = os.path.join(params["data_dir"], "text")
+ if os.path.exists(text_in):
+ text_proc_file = os.path.join(best_recog_path, "token")
+ compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))
+
+
+if __name__ == "__main__":
+ params = {}
+ params["data_dir"] = "./data/test"
+ params["output_dir"] = "./results"
+ params["ngpu"] = 1
+ params["njob"] = 1
+ modelscope_infer(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer_after_finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer_after_finetune.py
new file mode 100644
index 0000000..30483a0
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer_after_finetune.py
@@ -0,0 +1,53 @@
+import json
+import os
+import shutil
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_after_finetune(params):
+ # prepare for decoding
+ pretrained_model_path = os.path.join(os.environ["HOME"], ".cache/modelscope/hub", params["modelscope_model_name"])
+ for file_name in params["required_files"]:
+ if file_name == "configuration.json":
+ with open(os.path.join(pretrained_model_path, file_name)) as f:
+ config_dict = json.load(f)
+ config_dict["model"]["am_model_name"] = params["decoding_model_name"]
+ with open(os.path.join(params["output_dir"], "configuration.json"), "w") as f:
+ json.dump(config_dict, f, indent=4, separators=(',', ': '))
+ else:
+ shutil.copy(os.path.join(pretrained_model_path, file_name),
+ os.path.join(params["output_dir"], file_name))
+ decoding_path = os.path.join(params["output_dir"], "decode_results")
+ if os.path.exists(decoding_path):
+ shutil.rmtree(decoding_path)
+ os.mkdir(decoding_path)
+
+ # decoding
+ inference_pipeline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model=params["output_dir"],
+ output_dir=decoding_path,
+ batch_size=64
+ )
+ audio_in = os.path.join(params["data_dir"], "wav.scp")
+ inference_pipeline(audio_in=audio_in)
+
+ # computer CER if GT text is set
+ text_in = os.path.join(params["data_dir"], "text")
+ if text_in is not None:
+ text_proc_file = os.path.join(decoding_path, "1best_recog/token")
+ compute_wer(text_in, text_proc_file, os.path.join(decoding_path, "text.cer"))
+
+
+if __name__ == '__main__':
+ params = {}
+ params["modelscope_model_name"] = "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+ params["required_files"] = ["am.mvn", "decoding.yaml", "configuration.json"]
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data/test"
+ params["decoding_model_name"] = "valid.acc.ave_10best.pth"
+ modelscope_infer_after_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/local_infer.py b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/local_infer.py
new file mode 100644
index 0000000..ea92552
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/local_infer.py
@@ -0,0 +1,88 @@
+import os
+import shutil
+from multiprocessing import Pool
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+from funasr.utils.compute_wer import compute_wer
+
+
+def modelscope_infer_core(output_dir, split_dir, njob, idx):
+ output_dir_job = os.path.join(output_dir, "output.{}".format(idx))
+ gpu_id = (int(idx) - 1) // njob
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[gpu_id])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ output_dir=output_dir_job,
+ batch_size=32
+ )
+ audio_in = os.path.join(split_dir, "wav.{}.scp".format(idx))
+ inference_pipline(audio_in=audio_in)
+
+
+def modelscope_infer(params):
+ # prepare for multi-GPU decoding
+ ngpu = params["ngpu"]
+ njob = params["njob"]
+ output_dir = params["output_dir"]
+ if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+ os.mkdir(output_dir)
+ split_dir = os.path.join(output_dir, "split")
+ os.mkdir(split_dir)
+ nj = ngpu * njob
+ wav_scp_file = os.path.join(params["data_dir"], "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(split_dir, "wav.{}.scp".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(modelscope_infer_core,
+ args=(output_dir, split_dir, njob, str(i + 1)))
+ p.close()
+ p.join()
+
+ # combine decoding results
+ best_recog_path = os.path.join(output_dir, "1best_recog")
+ os.mkdir(best_recog_path)
+ files = ["text", "token", "score"]
+ for file in files:
+ with open(os.path.join(best_recog_path, file), "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(output_dir, "output.{}".format(str(i + 1)), file)
+ with open(job_file) as f_job:
+ lines = f_job.readlines()
+ f.writelines(lines)
+
+ # If text exists, compute CER
+ text_in = os.path.join(params["data_dir"], "text")
+ if os.path.exists(text_in):
+ text_proc_file = os.path.join(best_recog_path, "token")
+ compute_wer(text_in, text_proc_file, os.path.join(best_recog_path, "text.cer"))
+
+
+if __name__ == "__main__":
+ params = {}
+ params["data_dir"] = "/mnt/beinian.lzr/workspace/local_dataset/data/aishell-1/DATA/data/local/test"
+ params["output_dir"] = "./results"
+ params["ngpu"] = 2
+ params["njob"] = 1
+ modelscope_infer(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py
new file mode 100644
index 0000000..41e0337
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 25000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
new file mode 100644
index 0000000..8a6c87b
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
@@ -0,0 +1,15 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch",
+ output_dir=output_dir,
+ batch_size=32,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py
new file mode 100644
index 0000000..7932045
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/finetune.py
@@ -0,0 +1,36 @@
+import os
+
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+
+from funasr.datasets.ms_dataset import MsDataset
+from funasr.utils.modelscope_param import modelscope_args
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = modelscope_args(model="damo/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch",
+ data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 25000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.0005 # 璁剧疆瀛︿範鐜�
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
new file mode 100644
index 0000000..dec7de0
--- /dev/null
+++ b/egs_modelscope/asr/paraformer/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://modelscope.oss-cn-beijing.aliyuncs.com/test/audios/asr_example.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
new file mode 100644
index 0000000..df18903
--- /dev/null
+++ b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformerbert_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
new file mode 100644
index 0000000..83d6805
--- /dev/null
+++ b/egs_modelscope/asr/paraformerbert/speech_paraformerbert_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://modelscope.oss-cn-beijing.aliyuncs.com/test/audios/asr_example.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformerbert_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..3fa3f9d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..c06f934
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_cantonese-CHS.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..3fa3f9d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/infer.py
new file mode 100644
index 0000000..c06f934
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_cantonese-CHS.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-cantonese-CHS-16k-common-vocab1468-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..f15e3b9
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..227f4bf
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-offline",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..0d17e4d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/infer.py
new file mode 100644
index 0000000..74d9764
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-cn-dialect-16k-vocab8358-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..6998f0c
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..201f794
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..6998f0c
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/infer.py
new file mode 100644
index 0000000..201f794
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-en-16k-common-vocab1080-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..79ef4b6
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..cfd9e9d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_es.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..79ef4b6
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/infer.py
new file mode 100644
index 0000000..cfd9e9d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_es.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-es-16k-common-vocab3445-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..985b838
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..82d19d0
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_id.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..985b838
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/infer.py
new file mode 100644
index 0000000..82d19d0
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_id.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-id-16k-common-vocab1067-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..4a5efdb
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..f1e156b
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ja.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..4a5efdb
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/infer.py
new file mode 100644
index 0000000..f1e156b
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ja.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ja-16k-common-vocab93-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..249aa28
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..5fbfdbb
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ko.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..249aa28
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/infer.py
new file mode 100644
index 0000000..5fbfdbb
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ko.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ko-16k-common-vocab6400-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..60f3c82
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..fdab5a3
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_pt.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..60f3c82
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/infer.py
new file mode 100644
index 0000000..fdab5a3
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_pt.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-pt-16k-common-vocab1617-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..e8d61d9
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..1051b1f
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-offline/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ru.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..e8d61d9
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params["output_dir"]):
+ os.makedirs(params["output_dir"], exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params["data_dir"])
+ kwargs = dict(
+ model=params["model"],
+ model_revision=params["model_revision"],
+ data_dir=ds_dict,
+ dataset_type=params["dataset_type"],
+ work_dir=params["output_dir"],
+ batch_bins=params["batch_bins"],
+ max_epoch=params["max_epoch"],
+ lr=params["lr"])
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ params = {}
+ params["output_dir"] = "./checkpoint"
+ params["data_dir"] = "./data"
+ params["batch_bins"] = 2000
+ params["dataset_type"] = "small"
+ params["max_epoch"] = 50
+ params["lr"] = 0.00005
+ params["model"] = "damo/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online"
+ params["model_revision"] = None
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/infer.py
new file mode 100644
index 0000000..1051b1f
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online/infer.py
@@ -0,0 +1,13 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == "__main__":
+ audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_ru.wav"
+ output_dir = "./results"
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-ru-16k-common-vocab1664-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..73aae7d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..40686ac
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-offline",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..71a68d0
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/infer.py
new file mode 100644
index 0000000..dfe934d
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-16k-common-vocab8358-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/finetune.py
new file mode 100644
index 0000000..fe88cdf
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer.py
new file mode 100644
index 0000000..27d7903
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-offline",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/finetune.py
new file mode 100644
index 0000000..6341caf
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer.py
new file mode 100644
index 0000000..3b2964c
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/finetune.py
new file mode 100644
index 0000000..b18296e
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/infer.py
new file mode 100644
index 0000000..1c1e303
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-offline",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/README.md b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/finetune.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/finetune.py
new file mode 100644
index 0000000..435db14
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 20 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/infer.py b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/infer.py
new file mode 100644
index 0000000..94c1b68
--- /dev/null
+++ b/egs_modelscope/asr/uniasr/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online/infer.py
@@ -0,0 +1,14 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_UniASR_asr_2pass-zh-cn-8k-common-vocab8358-tensorflow1-online",
+ output_dir=output_dir,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
new file mode 100644
index 0000000..c68a8cd
--- /dev/null
+++ b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/README.md
@@ -0,0 +1,30 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained Paraformer-large Model
+
+### Finetune
+
+- Modify finetune training related parameters in `finetune.py`
+ - <strong>output_dir:</strong> # result dir
+ - <strong>data_dir:</strong> # the dataset dir needs to include files: train/wav.scp, train/text; validation/wav.scp, validation/text.
+ - <strong>batch_bins:</strong> # batch size
+ - <strong>max_epoch:</strong> # number of training epoch
+ - <strong>lr:</strong> # learning rate
+
+- Then you can run the pipeline to finetune with:
+```python
+ python finetune.py
+```
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
diff --git a/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
new file mode 100644
index 0000000..8c053d1
--- /dev/null
+++ b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/finetune.py
@@ -0,0 +1,35 @@
+import os
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from funasr.datasets.ms_dataset import MsDataset
+
+
+def modelscope_finetune(params):
+ if not os.path.exists(params.output_dir):
+ os.makedirs(params.output_dir, exist_ok=True)
+ # dataset split ["train", "validation"]
+ ds_dict = MsDataset.load(params.data_path)
+ kwargs = dict(
+ model=params.model,
+ model_revision=params.model_revision,
+ data_dir=ds_dict,
+ dataset_type=params.dataset_type,
+ work_dir=params.output_dir,
+ batch_bins=params.batch_bins,
+ max_epoch=params.max_epoch,
+ lr=params.lr)
+ trainer = build_trainer(Trainers.speech_asr_trainer, default_args=kwargs)
+ trainer.train()
+
+
+if __name__ == '__main__':
+ from funasr.utils.modelscope_param import modelscope_args
+ params = modelscope_args(model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", data_path="./data")
+ params.output_dir = "./checkpoint" # m妯″瀷淇濆瓨璺緞
+ params.data_path = "./example_data/" # 鏁版嵁璺緞
+ params.dataset_type = "small" # 灏忔暟鎹噺璁剧疆small锛岃嫢鏁版嵁閲忓ぇ浜�1000灏忔椂锛岃浣跨敤large
+ params.batch_bins = 2000 # batch size锛屽鏋渄ataset_type="small"锛宐atch_bins鍗曚綅涓篺bank鐗瑰緛甯ф暟锛屽鏋渄ataset_type="large"锛宐atch_bins鍗曚綅涓烘绉掞紝
+ params.max_epoch = 50 # 鏈�澶ц缁冭疆鏁�
+ params.lr = 0.00005 # 璁剧疆瀛︿範鐜�
+
+ modelscope_finetune(params)
diff --git a/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
new file mode 100644
index 0000000..2703c2a
--- /dev/null
+++ b/egs_modelscope/asr_vad_punc/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/infer.py
@@ -0,0 +1,15 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ output_dir=output_dir,
+ batch_size=1,
+ )
+ rec_result = inference_pipline(audio_in=audio_in)
+ print(rec_result)
+
diff --git a/egs_modelscope/common/README.md b/egs_modelscope/common/README.md
deleted file mode 100644
index f2049e2..0000000
--- a/egs_modelscope/common/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-# ModelScope Model
-
-## How to finetune and infer using a pretrained ModelScope Model
-
-### Finetune
-- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml`
-- Setting parameters in `modelscope_common_finetune.sh`
- - <strong>dataset:</strong> the dataset dir needs to include files: train/wav.scp, train/text; optional dev/wav.scp, dev/text, test/wav.scp test/text
- - <strong>tag:</strong> exp tag
- - <strong>init_model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-- Then you can run the pipeline to finetune with our model download from modelscope:
-```sh
- sh ./modelscope_common_finetune.sh
-```
-
-### Inference
-
-Or you can use the finetuned model for inference directly.
-
-- Setting parameters in `modelscope_common_infer.sh`
- - <strong>data_dir:</strong> # wav list, ${data_dir}/wav.scp
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope
-- Then you can run the pipeline to infer with:
-```sh
- sh ./modelscope_common_infer.sh
-```
diff --git a/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
deleted file mode 100644
index 22f02d9..0000000
--- a/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 10
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.15
diff --git a/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml
deleted file mode 100644
index e623192..0000000
--- a/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-beam_size: 1
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.0
diff --git a/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
deleted file mode 100644
index e9210f3..0000000
--- a/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
+++ /dev/null
@@ -1,91 +0,0 @@
-# network architecture
-# encoder related
-encoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- attention_dropout_rate: 0.1
-
-# decoder related
-decoder_conf:
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- self_attention_dropout_rate: 0.1
- src_attention_dropout_rate: 0.1
-
-predictor_conf:
- threshold: 1.0
- l_order: 1
- r_order: 1
- tail_threshold: 0.45
-
-# hybrid CTC/attention
-model_conf:
- ctc_weight: 0.0
- lsm_weight: 0.1 # label smoothing option
- length_normalized_loss: true
- predictor_weight: 1.0
- predictor_bias: 1
- sampling_ratio: 0.75
-
-# minibatch related
-# dataset_type: small
-batch_type: length
-batch_bins: 2000
-num_workers: 16
-# dataset_type: large
-dataset_conf:
- filter_conf:
- min_length: 10
- max_length: 250
- min_token_length: 1
- max_token_length: 200
- shuffle: true
- shuffle_conf:
- shuffle_size: 10240
- sort_size: 500
- batch_conf:
- batch_type: 'token'
- batch_size: 6000
- num_workers: 16
-
-# optimization related
-accum_grad: 1
-grad_clip: 5
-max_epoch: 20
-val_scheduler_criterion:
- - valid
- - acc
-best_model_criterion:
-- - valid
- - acc
- - max
-keep_nbest_models: 10
-
-optim: adam
-optim_conf:
- lr: 0.0005
-scheduler: warmuplr
-scheduler_conf:
- warmup_steps: 30000
-
-specaug: specaug_lfr
-specaug_conf:
- apply_time_warp: false
- time_warp_window: 5
- time_warp_mode: bicubic
- apply_freq_mask: true
- freq_mask_width_range:
- - 0
- - 30
- lfr_rate: 6
- num_freq_mask: 1
- apply_time_mask: true
- time_mask_width_range:
- - 0
- - 12
- num_time_mask: 1
-
-unused_parameters: true
-log_interval: 50
-normalize: None
-split_with_space: true
diff --git a/egs_modelscope/common/modelscope_common_finetune.sh b/egs_modelscope/common/modelscope_common_finetune.sh
deleted file mode 100755
index 8feacd0..0000000
--- a/egs_modelscope/common/modelscope_common_finetune.sh
+++ /dev/null
@@ -1,272 +0,0 @@
-#!/usr/bin/env bash
-
-. ./path.sh || exit 1;
-
-# machines configuration
-CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1"
-gpu_num=2
-count=1
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-njob=1 # the number of jobs for each gpu
-train_cmd=utils/run.pl
-infer_cmd=utils/run.pl
-
-# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
-exp_dir="."
-lang=zh
-dumpdir=dump/fbank
-feats_type=fbank
-token_type=char
-scp=feats.scp
-type=kaldi_ark
-stage=1
-stop_stage=4
-
-# feature configuration
-feats_dim=560
-sample_frequency=16000
-nj=32
-speed_perturb="1.0"
-lfr=True
-lfr_m=7
-lfr_n=6
-
-init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning
-model_revision="v1.0.4" # please do not modify the model revision
-cmvn_file=init_model/${init_model_name}/am.mvn
-seg_file=init_model/${init_model_name}/seg_dict
-vocab=init_model/${init_model_name}/tokens.txt
-
-# data
-dataset= # dataset (include train/wav.scp, train/text, dev/wav.scp, dev/text, optional test/wav.scp test/text)
-
-# exp tag
-tag=""
-
-# Set bash to 'debug' mode, it will exit on :
-# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
-set -e
-set -u
-set -o pipefail
-
-train_set=train
-valid_set=dev
-test_sets="dev test"
-
-asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-init_param="init_model/${init_model_name}/model.pb"
-
-inference_config=conf/decode_asr_transformer_noctc_1best.yaml
-inference_asr_model=valid.acc.ave_10best.pth
-
-. utils/parse_options.sh || exit 1;
-
-# download model from modelscope
-python modelscope_utils/download_model.py --model_name ${init_model_name} --model_revision ${model_revision}
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist"
- exit 1
-else
- if [ -d init_model/${init_model_name} ]; then
- echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first."
- else
- mkdir -p init_model/${init_model_name}
- cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name}
- fi
-fi
-
-model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
-
-# you can set gpu num for decoding here
-gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-
-if ${gpu_inference}; then
- inference_nj=$njob
- _ngpu=1
-else
- inference_nj=$njob
- _ngpu=0
-fi
-
-[ ! -d ${dataset} ] && echo "$0: Training data is required" && exit 1;
-[ ! -f ${dataset}/train/wav.scp ] && [ ! -f ${dataset}/train/text ] && echo "$0: Training data wav.scp or text is not found" && exit 1;
-
-if [ ! -d "${dataset}/dev" ]; then
- utils/fix_data.sh ${dataset}/train
- utils/subset_data_dir_tr_cv.sh --dev-num-utt 1000 ${dataset}/train ${dataset}
-fi
-if [ ! -d "${dataset}/test" ]; then
- test_sets="dev"
-fi
-
-feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir}
-feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir}
-feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir}
-
-if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
- echo "stage 1: Feature Generation"
- # compute fbank features
- fbankdir=${feats_dir}/fbank
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} --speed_perturb ${speed_perturb} \
- ${dataset}/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
- utils/fix_data_feat.sh ${fbankdir}/train
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} \
- ${dataset}/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev
- utils/fix_data_feat.sh ${fbankdir}/dev
- if [ -d "${dataset}/test" ]; then
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} \
- ${dataset}/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test
- utils/fix_data_feat.sh ${fbankdir}/test
- fi
-
- echo "apply low_frame_rate and cmvn"
- [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1;
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/train ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/dev ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir}
- if [ -d "${dataset}/test" ]; then
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/test ${cmvn_file} ${exp_dir}/exp/make_fbank/test ${feat_test_dir}
- fi
-
- echo "Text Tokenize"
- # 鎴戠埍reading->鎴� 鐖� read@@ ing
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/train ${seg_file} ${feat_train_dir}/log ${feat_train_dir}
- utils/fix_data_feat.sh ${feat_train_dir}
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/dev ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir}
- utils/fix_data_feat.sh ${feat_dev_dir}
- if [ -d "${dataset}/test" ]; then
- cp ${fbankdir}/test/text ${feat_test_dir}
- fi
-fi
-
-token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
-echo "dictionary: ${token_list}"
-if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
- echo "stage 2: Dictionary Preparation"
- mkdir -p ${feats_dir}/data/${lang}_token_list/char/
- cp $vocab ${token_list}
-
- vocab_size=$(wc -l <${token_list})
- awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
- awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev
- cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train
- cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev
-fi
-
-# Training Stage
-world_size=$gpu_num # run on one machine
-if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
- echo "stage 3: Training"
- # update asr train config.yaml
- python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/finetune.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml
- finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml
-
- mkdir -p ${exp_dir}/exp/${model_dir}
- mkdir -p ${exp_dir}/exp/${model_dir}/log
- INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
- if [ -f $INIT_FILE ];then
- rm -f $INIT_FILE
- fi
- init_method=file://$(readlink -f $INIT_FILE)
- echo "$0: init method is $init_method"
- for ((i = 0; i < $gpu_num; ++i)); do
- {
- rank=$i
- local_rank=$i
- gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
- asr_train_paraformer.py \
- --gpu_id $gpu_id \
- --use_preprocessor true \
- --token_type $token_type \
- --token_list $token_list \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \
- --resume true \
- --output_dir ${exp_dir}/exp/${model_dir} \
- --init_param $init_param \
- --config $finetune_config \
- --input_size $feats_dim \
- --ngpu $gpu_num \
- --num_worker_count $count \
- --multiprocessing_distributed true \
- --dist_init_method $init_method \
- --dist_world_size $world_size \
- --dist_rank $rank \
- --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
- } &
- done
- wait
-fi
-
-# Testing Stage
-# Testing Stage
-if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
- echo "stage 4: Inference"
- for dset in ${test_sets}; do
- asr_exp=${exp_dir}/exp/${model_dir}
- inference_tag="$(basename "${inference_config}" .yaml)"
- _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
- _logdir="${_dir}/logdir"
- if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete this dir first."
- exit 0
- fi
- mkdir -p "${_logdir}"
- _data="${feats_dir}/${dumpdir}/${dset}"
- key_file=${_data}/${scp}
- num_scp_file="$(<${key_file} wc -l)"
- _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
- split_scps=
- for n in $(seq "${_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
- done
- # shellcheck disable=SC2086
- utils/split_scp.pl "${key_file}" ${split_scps}
- _opts=
- if [ -n "${inference_config}" ]; then
- _opts+="--config ${inference_config} "
- fi
- ${infer_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.asr_inference_launch \
- --batch_size 64 \
- --ngpu "${_ngpu}" \
- --njob ${njob} \
- --gpuid_list ${gpuid_list:0:1} \
- --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
- --key_file "${_logdir}"/keys.JOB.scp \
- --asr_train_config "${asr_exp}"/config.yaml \
- --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
- --output_dir "${_logdir}"/output.JOB \
- --mode paraformer \
- ${_opts}
-
- for f in token token_int score text; do
- if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
- for i in $(seq "${_nj}"); do
- cat "${_logdir}/output.${i}/1best_recog/${f}"
- done | sort -k1 >"${_dir}/${f}"
- fi
- done
- python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
- python utils/proce_text.py ${_data}/text ${_data}/text.proc
- python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
- tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
- cat ${_dir}/text.cer.txt
- done
-fi
diff --git a/egs_modelscope/common/modelscope_common_infer.sh b/egs_modelscope/common/modelscope_common_infer.sh
deleted file mode 100755
index af3e7e5..0000000
--- a/egs_modelscope/common/modelscope_common_infer.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope
-model_revision="v1.0.4" # please do not modify the model revision
-data_dir= # wav list, ${data_dir}/wav.scp
-exp_dir="exp"
-gpuid_list="0"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1
-gpu_inference=true
-decode_cmd=utils/run.pl
-
-. utils/parse_options.sh
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
- _ngpu=1
-else
- inference_nj=${njob}
- _ngpu=0
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-python modelscope_utils/download_model.py \
- --model_name ${model_name} --model_revision ${model_revision}
-
-if [ -d ${exp_dir} ]; then
- echo "${exp_dir} is already exists. if you want to decode again, please delete ${exp_dir} first."
- exit 1
-else
- mkdir -p ${exp_dir}/${model_name}
- cp ${HOME}/.cache/modelscope/hub/damo/${model_name}/* ${exp_dir}/${model_name}/. -r
- _dir=${exp_dir}/decode_asr
- _logdir=${_dir}/logdir
- mkdir -p "${_dir}"
- mkdir -p "${_logdir}"
-fi
-
-for n in $(seq "${inference_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
-done
-# shellcheck disable=SC2086
-utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps}
-
-if "${use_lm}"; then
- cp ${exp_dir}/${model_name}/decoding.yaml ${exp_dir}/${model_name}/decoding.yaml.back
- sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${exp_dir}/${model_name}/decoding.yaml
- sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${exp_dir}/${model_name}/decoding.yaml
-fi
-
-echo "Decoding started... log: '${_logdir}/asr_inference.*.log'"
-# shellcheck disable=SC2086
-${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.modelscope_infer \
- --local_model_path ${exp_dir}/${model_name} \
- --wav_list ${_logdir}/keys.JOB.scp \
- --output_file ${_logdir}/text.JOB \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --ngpu ${_ngpu} \
-
- for i in $(seq ${inference_nj}); do
- cat ${_logdir}/text.${i}
- done | sort -k1 >${_dir}/text
-
-if "${use_lm}"; then
- mv ${exp_dir}/${model_name}/decoding.yaml.back ${exp_dir}/${model_name}/decoding.yaml
-fi
diff --git a/egs_modelscope/common/modelscope_common_infer_after_finetune.sh b/egs_modelscope/common/modelscope_common_infer_after_finetune.sh
deleted file mode 100755
index 3a11f69..0000000
--- a/egs_modelscope/common/modelscope_common_infer_after_finetune.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-pretrained_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope
-data_dir= # wav list, ${data_dir}/wav.scp
-finetune_model_name= # fine-tuning model name
-finetune_exp_dir= # fine-tuning model experiment result path
-gpuid_list="0"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1
-gpu_inference=true
-decode_cmd=utils/run.pl
-
-. utils/parse_options.sh
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
- _ngpu=1
-else
- inference_nj=${njob}
- _ngpu=0
-fi
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} must exist."
- exit 1
-else
- exp_dir=${finetune_exp_dir}/${finetune_model_name}.modelscope
- mkdir -p $exp_dir
- cp ${finetune_exp_dir}/${finetune_model_name} ${exp_dir}/${finetune_model_name}.modelscope
- cp ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name}/* ${exp_dir}/. -r
-fi
-
-_dir=${exp_dir}/decode_asr
-_logdir=${_dir}/logdir
-if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete ${_dir} first."
-else
- mkdir -p "${_dir}"
- mkdir -p "${_logdir}"
-fi
-
-for n in $(seq "${inference_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
-done
-# shellcheck disable=SC2086
-utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps}
-
-echo "Decoding started... log: '${_logdir}/asr_inference.*.log'"
-# shellcheck disable=SC2086
-${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.modelscope_infer \
- --local_model_path ${exp_dir} \
- --wav_list ${_logdir}/keys.JOB.scp \
- --output_file ${_logdir}/text.JOB \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --ngpu ${_ngpu} \
-
- for i in $(seq ${inference_nj}); do
- cat ${_logdir}/text.${i}
- done | sort -k1 >${_dir}/text
diff --git a/egs_modelscope/common/modelscope_utils/download_model.py b/egs_modelscope/common/modelscope_utils/download_model.py
deleted file mode 100755
index 51ba6b8..0000000
--- a/egs_modelscope/common/modelscope_utils/download_model.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-
-from modelscope.pipelines import pipeline
-from modelscope.utils.constant import Tasks
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description="download model configs",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- )
- parser.add_argument("--model_name",
- type=str,
- default="speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
- help="model name in modelscope")
- parser.add_argument("--model_revision",
- type=str,
- default="v1.0.3",
- help="model revision in modelscope")
- args = parser.parse_args()
-
- inference_pipeline = pipeline(
- task=Tasks.auto_speech_recognition,
- model='damo/{}'.format(args.model_name),
- model_revision=args.model_revision)
diff --git a/egs_modelscope/common/modelscope_utils/modelscope_infer.sh b/egs_modelscope/common/modelscope_utils/modelscope_infer.sh
deleted file mode 100755
index b9b3b6f..0000000
--- a/egs_modelscope/common/modelscope_utils/modelscope_infer.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-data_dir=
-exp_dir=
-model_name=
-model_revision=
-inference_nj=32
-gpuid_list="0,1,2,3"
-njob=32
-gpu_inference=true
-
-test_sets="dev test"
-decode_cmd=utils/run.pl
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-. utils/parse_options.sh
-
-if ${gpu_inference}; then
- _ngpu=1
-else
- _ngpu=0
-fi
-
-# download model from modelscope
-python modelscope_utils/download_model.py \
- --model_name ${model_name} --model_revision ${model_revision}
-
-modelscope_dir=${HOME}/.cache/modelscope/hub/damo/${model_name}
-
-
-for dset in ${test_sets}; do
- _dir=${exp_dir}/${model_name}/decode_asr/${dset}
- _logdir=${_dir}/logdir
- _data=${data_dir}/${dset}
- if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete ${_dir} first."
- exit 1
- else
- mkdir -p "${_dir}"
- mkdir -p "${_logdir}"
- fi
-
- if "${use_lm}"; then
- cp ${modelscope_dir}/decoding.yaml ${modelscope_dir}/decoding.yaml.back
- sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${modelscope_dir}/decoding.yaml
- sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${modelscope_dir}/decoding.yaml
- fi
-
- split_scps=
- for n in $(seq "${inference_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
- done
- # shellcheck disable=SC2086
- utils/split_scp.pl "${data_dir}/${dset}/wav.scp" ${split_scps}
-
- echo "Decoding started... log: '${_logdir}/asr_inference.*.log'"
- # shellcheck disable=SC2086
- ${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.modelscope_infer \
- --model_name ${model_name} \
- --model_revision ${model_revision} \
- --wav_list ${_logdir}/keys.JOB.scp \
- --output_file ${_logdir}/text.JOB \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --ngpu ${_ngpu} \
-
- for i in $(seq ${inference_nj}); do
- cat ${_logdir}/text.${i}
- done | sort -k1 >${_dir}/text
-
- python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
- python utils/proce_text.py ${_data}/text ${_data}/text.proc
- python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
- tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
- cat ${_dir}/text.cer.txt
-done
-
-if "${use_lm}"; then
- mv ${modelscope_dir}/decoding.yaml.back ${modelscope_dir}/decoding.yaml
-fi
diff --git a/egs_modelscope/common/modelscope_utils/update_config.py b/egs_modelscope/common/modelscope_utils/update_config.py
deleted file mode 100644
index 88466ed..0000000
--- a/egs_modelscope/common/modelscope_utils/update_config.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import yaml
-import argparse
-
-def update_dct(fin_configs, root):
- if root == {}:
- return {}
- for root_key, root_value in root.items():
- if not isinstance(root[root_key],dict):
- fin_configs[root_key] = root[root_key]
- else:
- result = update_dct(fin_configs[root_key], root[root_key])
- fin_configs[root_key] = result
- return fin_configs
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description="update configs",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter,
- )
- parser.add_argument("--modelscope_config",
- type=str,
- help="modelscope config file")
- parser.add_argument("--finetune_config",
- type=str,
- help="finetune config file")
- parser.add_argument("--output_config",
- type=str,
- help="output config file")
- args = parser.parse_args()
-
- with open(args.modelscope_config) as f:
- modelscope_configs = yaml.safe_load(f)
-
- with open(args.finetune_config) as f:
- finetune_configs = yaml.safe_load(f)
-
- # update configs, e.g., lr, batch_size, ...
- modelscope_configs = update_dct(modelscope_configs, finetune_configs)
-
- with open(args.output_config, "w") as f:
- yaml.dump(modelscope_configs, f, indent=4)
diff --git a/egs_modelscope/common/path.sh b/egs_modelscope/common/path.sh
deleted file mode 100755
index c340218..0000000
--- a/egs_modelscope/common/path.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-export FUNASR_DIR=$PWD/../..
-
-# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8
-export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs_modelscope/common_uniasr/README.md b/egs_modelscope/common_uniasr/README.md
deleted file mode 100644
index bd14334..0000000
--- a/egs_modelscope/common_uniasr/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-# ModelScope Model
-
-## How to finetune and infer using a pretrained ModelScope Model
-
-### Finetune
-- Modify finetune training related parameters in `conf/train_asr_uniasr_40e1_12d1_20e2_12d2_1280_320_lfr6.yaml`
-- Setting parameters in `modelscope_common_finetune.sh`
- - <strong>dataset:</strong> the dataset dir needs to include files: train/wav.scp, train/text; optional dev/wav.scp, dev/text, test/wav.scp test/text
- - <strong>tag:</strong> exp tag
- - <strong>init_model_name:</strong> speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online # pre-trained model, download from modelscope during fine-tuning
-- Then you can run the pipeline to finetune with our model download from modelscope:
-```sh
- sh ./modelscope_common_finetune.sh
-```
-
-### Inference
-
-Or you can use the finetuned model for inference directly.
-
-- Setting parameters in `modelscope_common_infer.sh`
- - <strong>data_dir:</strong> # wav list, ${data_dir}/wav.scp
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online # pre-trained model, download from modelscope
-- Then you can run the pipeline to infer with:
-```sh
- sh ./modelscope_common_infer.sh
-```
diff --git a/egs_modelscope/common_uniasr/conf/decode_asr_uniasr.yaml b/egs_modelscope/common_uniasr/conf/decode_asr_uniasr.yaml
deleted file mode 100644
index f723dd6..0000000
--- a/egs_modelscope/common_uniasr/conf/decode_asr_uniasr.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-beam_size: 5
-penalty: 0.0
-maxlenratio: 0.0
-minlenratio: 0.0
-ctc_weight: 0.0
-lm_weight: 0.0
-token_num_relax: 5
-decoding_ind: 0
-decoding_mode: model2
\ No newline at end of file
diff --git a/egs_modelscope/common_uniasr/conf/train_asr_uniasr_40e1_12d1_20e2_12d2_1280_320_lfr6.yaml b/egs_modelscope/common_uniasr/conf/train_asr_uniasr_40e1_12d1_20e2_12d2_1280_320_lfr6.yaml
deleted file mode 100644
index 9885f47..0000000
--- a/egs_modelscope/common_uniasr/conf/train_asr_uniasr_40e1_12d1_20e2_12d2_1280_320_lfr6.yaml
+++ /dev/null
@@ -1,192 +0,0 @@
-# encoder related
-encoder: sanm_chunk_opt
-encoder_conf:
- output_size: 320 # dimension of attention
- attention_heads: 4
- linear_units: 1280 # the number of units of position-wise feed forward
- num_blocks: 40 # the number of encoder blocks
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- attention_dropout_rate: 0.1
- input_layer: pe # encoder architecture type
- pos_enc_class: SinusoidalPositionEncoder
- normalize_before: true
- kernel_size: 11
- sanm_shfit: 0
- selfattention_layer_type: sanm
- chunk_size:
- - 20
- - 60
- stride:
- - 10
- - 40
- pad_left:
- - 5
- - 10
- encoder_att_look_back_factor:
- - 0
- - 0
- decoder_att_look_back_factor:
- - 0
- - 0
-
-# encoder related
-encoder2: sanm_chunk_opt
-encoder2_conf:
- output_size: 320 # dimension of attention
- attention_heads: 4
- linear_units: 1280 # the number of units of position-wise feed forward
- num_blocks: 20 # the number of encoder blocks
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- attention_dropout_rate: 0.1
- input_layer: pe # encoder architecture type
- pos_enc_class: SinusoidalPositionEncoder
- normalize_before: true
- kernel_size: 21
- sanm_shfit: 0
- selfattention_layer_type: sanm
- chunk_size:
- - 45
- - 70
- stride:
- - 35
- - 50
- pad_left:
- - 5
- - 10
- encoder_att_look_back_factor:
- - 0
- - 0
- decoder_att_look_back_factor:
- - 0
- - 0
-
-# decoder related
-decoder: fsmn_scama_opt
-decoder_conf:
- attention_dim: 256
- attention_heads: 4
- linear_units: 1024
- num_blocks: 12
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- self_attention_dropout_rate: 0.1
- src_attention_dropout_rate: 0.1
- att_layer_num: 6
- kernel_size: 11
- concat_embeds: true
-
-# decoder related
-decoder2: fsmn_scama_opt
-decoder2_conf:
- attention_dim: 320
- attention_heads: 4
- linear_units: 1280
- num_blocks: 12
- dropout_rate: 0.1
- positional_dropout_rate: 0.1
- self_attention_dropout_rate: 0.1
- src_attention_dropout_rate: 0.1
- att_layer_num: 6
- kernel_size: 11
- concat_embeds: true
-
-stride_conv: stride_conv1d
-stride_conv_conf:
- kernel_size: 2
- stride: 2
- pad:
- - 0
- - 1
-
-predictor: cif_predictor_v2
-predictor_conf:
- idim: 320
- threshold: 1.0
- l_order: 1
- r_order: 1
-
-predictor2: cif_predictor_v2
-predictor2_conf:
- idim: 320
- threshold: 1.0
- l_order: 1
- r_order: 1
-
-# hybrid CTC/attention
-model: uniasr
-model_conf:
- ctc_weight: 0.0
- lsm_weight: 0.1 # label smoothing option
- length_normalized_loss: true
- predictor_weight: 1.0
- decoder_attention_chunk_type: chunk
- ctc_weight2: 0.0
- predictor_weight2: 1.0
- decoder_attention_chunk_type2: chunk
- loss_weight_model1: 0.5
- enable_maas_finetune: true
-
-# minibatch related
-batch_type: length
-batch_bins: 2000
-num_workers: 16
-
-dataset_conf:
- filter_conf:
- min_length: 10
- max_length: 250
- min_token_length: 1
- max_token_length: 200
- shuffle: True
- shuffle_conf:
- shuffle_size: 10240
- sort_size: 500
- batch_conf:
- batch_type: token
- batch_size: 2000
- num_workers: 16
-
-# optimization related
-accum_grad: 1
-grad_clip: 5
-max_epoch: 20
-val_scheduler_criterion:
- - valid
- - acc
-best_model_criterion:
-- - valid
- - acc
- - max
-keep_nbest_models: 20
-
-optim: adam
-optim_conf:
- lr: 0.0001
-scheduler: warmuplr
-scheduler_conf:
- warmup_steps: 30000
-
-specaug: specaug_lfr
-specaug_conf:
- apply_time_warp: false
- time_warp_window: 5
- time_warp_mode: bicubic
- apply_freq_mask: true
- freq_mask_width_range:
- - 0
- - 30
- lfr_rate: 6
- num_freq_mask: 1
- apply_time_mask: true
- time_mask_width_range:
- - 0
- - 12
- num_time_mask: 1
-
-
-log_interval: 50
-normalize: None
-split_with_space: true
-unused_parameters: true
diff --git a/egs_modelscope/common_uniasr/modelscope_common_finetune.sh b/egs_modelscope/common_uniasr/modelscope_common_finetune.sh
deleted file mode 100755
index dfc1fdb..0000000
--- a/egs_modelscope/common_uniasr/modelscope_common_finetune.sh
+++ /dev/null
@@ -1,268 +0,0 @@
-#!/usr/bin/env bash
-
-. ./path.sh || exit 1;
-
-# machines configuration
-CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1"
-gpu_num=2
-count=1
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-njob=4 # the number of jobs for each gpu
-train_cmd=utils/run.pl
-
-# general configuration
-feats_dir="../DATA" #feature output dictionary, for large data
-exp_dir="."
-lang=zh
-dumpdir=dump/fbank
-feats_type=fbank
-token_type=char
-scp=feats.scp
-type=kaldi_ark
-stage=1
-stop_stage=4
-
-# feature configuration
-feats_dim=560
-sample_frequency=8000
-nj=32
-speed_perturb="1.0"
-lfr=True
-lfr_m=7
-lfr_n=6
-
-init_model_name=speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online # pre-trained model, download from modelscope during fine-tuning
-model_revision="v1.0.0" # please do not modify the model revision
-cmvn_file=init_model/${init_model_name}/am.mvn
-seg_file=init_model/${init_model_name}/seg_dict
-vocab=init_model/${init_model_name}/tokens.txt
-
-# data
-dataset= # dataset (include train/wav.scp, train/text, dev/wav.scp, dev/text, optional test/wav.scp test/text)
-
-# exp tag
-tag=""
-
-# Set bash to 'debug' mode, it will exit on :
-# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
-set -e
-set -u
-set -o pipefail
-
-train_set=train
-valid_set=dev
-test_sets="dev test"
-
-asr_config=conf/train_asr_uniasr_40e1_12d1_20e2_12d2_1280_320_lfr6.yaml
-init_param="init_model/${init_model_name}/model.pb"
-
-inference_config=conf/decode_asr_uniasr.yaml
-inference_asr_model=20epoch.pth
-
-. utils/parse_options.sh || exit 1;
-
-# download model from modelscope
-python modelscope_utils/download_model.py --model_name ${init_model_name} --model_revision ${model_revision}
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist"
- exit 1
-else
- if [ -d init_model/${init_model_name} ]; then
- echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first."
- else
- mkdir -p init_model/${init_model_name}
- cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name}
- fi
-fi
-
-model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}"
-
-# you can set gpu num for decoding here
-gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
- _ngpu=1
-else
- inference_nj=$njob
- _ngpu=0
-fi
-
-[ ! -d ${dataset} ] && echo "$0: Training data is required" && exit 1;
-[ ! -f ${dataset}/train/wav.scp ] && [ ! -f ${dataset}/train/text ] && echo "$0: Training data wav.scp or text is not found" && exit 1;
-
-if [ ! -d "${dataset}/dev" ]; then
- utils/fix_data.sh ${dataset}/train
- utils/subset_data_dir_tr_cv.sh --dev-num-utt 1000 ${dataset}/train ${dataset}
-fi
-if [ ! -d "${dataset}/test" ]; then
- test_sets="dev"
-fi
-
-feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir}
-feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir}
-feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir}
-
-if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
- echo "stage 1: Feature Generation"
- # compute fbank features
- fbankdir=${feats_dir}/fbank
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} --speed_perturb ${speed_perturb} \
- ${dataset}/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train
- utils/fix_data_feat.sh ${fbankdir}/train
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} \
- ${dataset}/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev
- utils/fix_data_feat.sh ${fbankdir}/dev
- if [ -d "${dataset}/test" ]; then
- utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --sample_frequency ${sample_frequency} \
- ${dataset}/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test
- utils/fix_data_feat.sh ${fbankdir}/test
- fi
-
- echo "apply low_frame_rate and cmvn"
- [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1;
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/train ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir}
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/dev ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir}
- if [ -d "${dataset}/test" ]; then
- utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \
- --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \
- ${fbankdir}/test ${cmvn_file} ${exp_dir}/exp/make_fbank/test ${feat_test_dir}
- fi
-
- echo "Text Tokenize"
- # 鎴戠埍reading->鎴� 鐖� read@@ ing
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/train ${seg_file} ${feat_train_dir}/log ${feat_train_dir}
- utils/fix_data_feat.sh ${feat_train_dir}
- utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/dev ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir}
- utils/fix_data_feat.sh ${feat_dev_dir}
- if [ -d "${dataset}/test" ]; then
- cp ${fbankdir}/test/text ${feat_test_dir}
- fi
-fi
-
-token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt
-echo "dictionary: ${token_list}"
-if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
- echo "stage 2: Dictionary Preparation"
- mkdir -p ${feats_dir}/data/${lang}_token_list/char/
- cp $vocab ${token_list}
-
- vocab_size=$(wc -l <${token_list})
- awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char
- awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train
- mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev
- cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train
- cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev
-fi
-
-# Training Stage
-world_size=$gpu_num # run on one machine
-if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
- echo "stage 3: Training"
- # update asr train config.yaml
- python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/finetune.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml
- finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml
-
- mkdir -p ${exp_dir}/exp/${model_dir}
- mkdir -p ${exp_dir}/exp/${model_dir}/log
- INIT_FILE=${exp_dir}/exp/${model_dir}/ddp_init
- if [ -f $INIT_FILE ];then
- rm -f $INIT_FILE
- fi
- init_method=file://$(readlink -f $INIT_FILE)
- echo "$0: init method is $init_method"
- for ((i = 0; i < $gpu_num; ++i)); do
- {
- rank=$i
- local_rank=$i
- gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1])
- asr_train_uniasr.py \
- --gpu_id $gpu_id \
- --use_preprocessor true \
- --token_type $token_type \
- --token_list $token_list \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \
- --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \
- --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \
- --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \
- --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \
- --resume true \
- --output_dir ${exp_dir}/exp/${model_dir} \
- --init_param $init_param \
- --config $finetune_config \
- --input_size $feats_dim \
- --ngpu $gpu_num \
- --num_worker_count $count \
- --multiprocessing_distributed true \
- --dist_init_method $init_method \
- --dist_world_size $world_size \
- --dist_rank $rank \
- --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1
- } &
- done
- wait
-fi
-
-# Testing Stage
-if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
- echo "stage 4: Inference"
- for dset in ${test_sets}; do
- asr_exp=${exp_dir}/exp/${model_dir}
- inference_tag="$(basename "${inference_config}" .yaml)"
- _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}"
- _logdir="${_dir}/logdir"
-
- mkdir -p "${_logdir}"
- _data="${feats_dir}/${dumpdir}/${dset}"
- key_file=${_data}/${scp}
- num_scp_file="$(<${key_file} wc -l)"
- _nj=$([ $inference_nj -le $num_scp_file ] && echo "$inference_nj" || echo "$num_scp_file")
- split_scps=
- for n in $(seq "${_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
- done
- # shellcheck disable=SC2086
- utils/split_scp.pl "${key_file}" ${split_scps}
- _opts=
- if [ -n "${inference_config}" ]; then
- _opts+="--config ${inference_config} "
- fi
- ${train_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.asr_inference_launch \
- --batch_size 1 \
- --ngpu "${_ngpu}" \
- --njob ${njob} \
- --gpuid_list ${gpuid_list} \
- --data_path_and_name_and_type "${_data}/${scp},speech,${type}" \
- --key_file "${_logdir}"/keys.JOB.scp \
- --asr_train_config "${asr_exp}"/config.yaml \
- --asr_model_file "${asr_exp}"/"${inference_asr_model}" \
- --output_dir "${_logdir}"/output.JOB \
- --mode uniasr \
- ${_opts}
-
- for f in token token_int score text; do
- if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then
- for i in $(seq "${_nj}"); do
- cat "${_logdir}/output.${i}/1best_recog/${f}"
- done | sort -k1 >"${_dir}/${f}"
- fi
- done
- python utils/proce_text.py ${_dir}/text ${_dir}/text.proc
- python utils/proce_text.py ${_data}/text ${_data}/text.proc
- python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer
- tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt
- cat ${_dir}/text.cer.txt
- done
-fi
-
diff --git a/egs_modelscope/common_uniasr/modelscope_common_infer.sh b/egs_modelscope/common_uniasr/modelscope_common_infer.sh
deleted file mode 100755
index 4e6e124..0000000
--- a/egs_modelscope/common_uniasr/modelscope_common_infer.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-model_name=speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online # pre-trained model, download from modelscope
-model_revision="v1.0.0" # please do not modify the model revision
-data_dir= # wav list, ${data_dir}/wav.scp
-exp_dir="exp"
-gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=4
-gpu_inference=true
-decode_cmd=utils/run.pl
-
-. utils/parse_options.sh
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
- _ngpu=1
-else
- inference_nj=${njob}
- _ngpu=0
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-python modelscope_utils/download_model.py \
- --model_name ${model_name} --model_revision ${model_revision}
-
-if [ -d ${exp_dir} ]; then
- echo "${exp_dir} is already exists. if you want to decode again, please delete ${exp_dir} first."
- exit 1
-else
- mkdir -p ${exp_dir}/${model_name}
- cp ${HOME}/.cache/modelscope/hub/damo/${model_name}/* ${exp_dir}/${model_name}/. -r
- _dir=${exp_dir}/decode_asr
- _logdir=${_dir}/logdir
- mkdir -p "${_dir}"
- mkdir -p "${_logdir}"
-fi
-
-for n in $(seq "${inference_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
-done
-# shellcheck disable=SC2086
-utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps}
-
-if "${use_lm}"; then
- cp ${exp_dir}/${model_name}/decoding.yaml ${exp_dir}/${model_name}/decoding.yaml.back
- sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${exp_dir}/${model_name}/decoding.yaml
- sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${exp_dir}/${model_name}/decoding.yaml
-fi
-
-echo "Decoding started... log: '${_logdir}/asr_inference.*.log'"
-# shellcheck disable=SC2086
-${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.modelscope_infer \
- --local_model_path ${exp_dir}/${model_name} \
- --wav_list ${_logdir}/keys.JOB.scp \
- --output_file ${_logdir}/text.JOB \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --ngpu ${_ngpu} \
-
- for i in $(seq ${inference_nj}); do
- cat ${_logdir}/text.${i}
- done | sort -k1 >${_dir}/text
-
-if "${use_lm}"; then
- mv ${exp_dir}/${model_name}/decoding.yaml.back ${exp_dir}/${model_name}/decoding.yaml
-fi
diff --git a/egs_modelscope/common_uniasr/modelscope_common_infer_after_finetune.sh b/egs_modelscope/common_uniasr/modelscope_common_infer_after_finetune.sh
deleted file mode 100755
index e92e0ed..0000000
--- a/egs_modelscope/common_uniasr/modelscope_common_infer_after_finetune.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-pretrained_model_name=speech_UniASR_asr_2pass-zh-cn-8k-common-vocab3445-pytorch-online # pre-trained model, download from modelscope
-data_dir= # wav list, ${data_dir}/wav.scp
-finetune_model_name= # fine-tuning model name
-finetune_exp_dir= # fine-tuning model experiment result path
-gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=4
-gpu_inference=true
-decode_cmd=utils/run.pl
-
-. utils/parse_options.sh
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
- _ngpu=1
-else
- inference_nj=${njob}
- inference_nj=${njob}
- _ngpu=0
-fi
-
-if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} ]; then
- echo "${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} must exist."
- exit 1
-else
- exp_dir=${finetune_exp_dir}/${finetune_model_name}.modelscope
- mkdir -p $exp_dir
- cp ${finetune_exp_dir}/${finetune_model_name} ${exp_dir}/${finetune_model_name}.modelscope
- cp ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name}/* ${exp_dir}/. -r
-fi
-
-_dir=${exp_dir}/decode_asr
-_logdir=${_dir}/logdir
-if [ -d ${_dir} ]; then
- echo "${_dir} is already exists. if you want to decode again, please delete ${_dir} first."
-else
- mkdir -p "${_dir}"
- mkdir -p "${_logdir}"
-fi
-
-for n in $(seq "${inference_nj}"); do
- split_scps+=" ${_logdir}/keys.${n}.scp"
-done
-# shellcheck disable=SC2086
-utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps}
-
-echo "Decoding started... log: '${_logdir}/asr_inference.*.log'"
-# shellcheck disable=SC2086
-${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \
- python -m funasr.bin.modelscope_infer \
- --local_model_path ${exp_dir} \
- --wav_list ${_logdir}/keys.JOB.scp \
- --output_file ${_logdir}/text.JOB \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --ngpu ${_ngpu} \
-
- for i in $(seq ${inference_nj}); do
- cat ${_logdir}/text.${i}
- done | sort -k1 >${_dir}/text
\ No newline at end of file
diff --git a/egs_modelscope/common_uniasr/modelscope_utils b/egs_modelscope/common_uniasr/modelscope_utils
deleted file mode 120000
index 59a672f..0000000
--- a/egs_modelscope/common_uniasr/modelscope_utils
+++ /dev/null
@@ -1 +0,0 @@
-../common/modelscope_utils
\ No newline at end of file
diff --git a/egs_modelscope/common_uniasr/path.sh b/egs_modelscope/common_uniasr/path.sh
deleted file mode 100755
index c340218..0000000
--- a/egs_modelscope/common_uniasr/path.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-export FUNASR_DIR=$PWD/../..
-
-# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8
-export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs_modelscope/common_uniasr/utils b/egs_modelscope/common_uniasr/utils
deleted file mode 120000
index 14bbd23..0000000
--- a/egs_modelscope/common_uniasr/utils
+++ /dev/null
@@ -1 +0,0 @@
-../../egs/aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/README.md b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/README.md
new file mode 100644
index 0000000..b125d48
--- /dev/null
+++ b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/README.md
@@ -0,0 +1,19 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained ModelScope Model
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+task=Tasks.punctuation,
+ model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
+
+- Setting parameters in `modelscope_common_infer.sh`
+ - <strong>model:</strong> damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch # pre-trained model, download from modelscope
+ - <strong>text_in:</strong> input path, text or url
+ - <strong>output_dir:</strong> the result dir
+- Then you can run the pipeline to infer with:
+```sh
+ python ./infer.py
+```
diff --git a/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/data/punc_example.txt b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/data/punc_example.txt
new file mode 100644
index 0000000..367be79
--- /dev/null
+++ b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/data/punc_example.txt
@@ -0,0 +1,3 @@
+1 璺ㄥ娌虫祦鏄吇鑲叉部宀镐汉姘戠殑鐢熷懡涔嬫簮闀挎湡浠ユ潵涓哄府鍔╀笅娓稿湴鍖洪槻鐏惧噺鐏句腑鏂规妧鏈汉鍛樺湪涓婃父鍦板尯鏋佷负鎭跺姡鐨勮嚜鐒舵潯浠朵笅鍏嬫湇宸ㄥぇ鍥伴毦鐢氳嚦鍐掔潃鐢熷懡鍗遍櫓鍚戝嵃鏂规彁渚涙睕鏈熸按鏂囪祫鏂欏鐞嗙揣鎬ヤ簨浠朵腑鏂归噸瑙嗗嵃鏂瑰湪璺ㄥ娌虫祦闂涓婄殑鍏冲垏鎰挎剰杩涗竴姝ュ畬鍠勫弻鏂硅仈鍚堝伐浣滄満鍒跺嚒鏄腑鏂硅兘鍋氱殑鎴戜滑閮戒細鍘诲仛鑰屼笖浼氬仛寰楁洿濂芥垜璇峰嵃搴︽湅鍙嬩滑鏀惧績涓浗鍦ㄤ笂娓哥殑浠讳綍寮�鍙戝埄鐢ㄩ兘浼氱粡杩囩瀛﹁鍒掑拰璁鸿瘉鍏奸【涓婁笅娓哥殑鍒╃泭
+2 浠庡瓨鍌ㄤ笂鏉ヨ浠呬粎鏄叏鏅浘鐗囧畠灏变細鏄浘鐗囩殑鍥涘�嶇殑瀹归噺鐒跺悗鍏ㄦ櫙鐨勮棰戜細鏄櫘閫氳棰戝叓鍊嶇殑杩欎釜瀛樺偍鐨勫瑕佹眰鑰屼笁d鐨勬ā鍨嬩細鏄浘鐗囩殑鍗佸�嶈繖閮藉鎴戜滑浠婂ぉ杩愯鍦ㄧ殑浜戣绠楃殑骞冲彴瀛樺偍鐨勫钩鍙版彁鍑轰簡鏇撮珮鐨勮姹�
+3 閭ｄ粖澶╃殑浼氬氨鍒拌繖閲屽惂 happy new year 鏄庡勾瑙�
diff --git a/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/infer.py b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/infer.py
new file mode 100644
index 0000000..3dfa416
--- /dev/null
+++ b/egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/infer.py
@@ -0,0 +1,23 @@
+
+##################text.scp鏂囦欢璺緞###################
+inputs = "./egs_modelscope/punctuation/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/data/punc_example.txt"
+
+##################text浜岃繘鍒舵暟鎹�#####################
+#inputs = "鎴戜滑閮芥槸鏈ㄥご浜轰笉浼氳璇濅笉浼氬姩"
+
+##################text鏂囦欢url#######################
+#inputs = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt"
+
+
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+inference_pipline = pipeline(
+ task=Tasks.punctuation,
+ model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
+ model_revision=None,
+ output_dir="./tmp/"
+)
+
+rec_result = inference_pipline(text_in=inputs)
+print(rec_result)
diff --git a/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/eval_eer.py b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/eval_eer.py
new file mode 100644
index 0000000..717d36a
--- /dev/null
+++ b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/eval_eer.py
@@ -0,0 +1,69 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+import os
+import argparse
+import numpy as np
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--eval_dir", type=str, default="eval")
+ parser.add_argument("--trials", type=str, default="eval/lists/trials.lst.speech")
+ parser.add_argument("--out_dir", type=str, default="./")
+ parser.add_argument("--verbose", action="store_true", default=False)
+ args = parser.parse_args()
+
+ if not os.path.exists(args.out_dir):
+ os.makedirs(args.out_dir)
+
+ inference_sv_pipline = pipeline(
+ task=Tasks.speaker_verification,
+ model='damo/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch'
+ )
+
+ trials_list = [x.strip() for x in open(args.trials, "r").readlines()]
+ enroll_list = set([x.split(" ")[0] for x in trials_list])
+ test_list = set([x.split(" ")[1] for x in trials_list])
+
+ print("extract embeddings for {} enrollments".format(len(enroll_list)))
+ enroll_embedding = {}
+ for enroll in enroll_list:
+ spk_embedding = inference_sv_pipline(
+ audio_in=os.path.join(args.eval_dir, "enroll", enroll+".wav")
+ )["spk_embedding"]
+ enroll_embedding[enroll] = spk_embedding
+
+ test_embedding = {}
+ print("extract embeddings for {} tests".format(len(test_list)))
+ for test in test_list:
+ spk_embedding = inference_sv_pipline(
+ audio_in=os.path.join(args.eval_dir, "test", test+".wav")
+ )["spk_embedding"]
+ test_embedding[test] = spk_embedding
+
+ print("calculate scores for {} trials".format(len(trials_list)))
+ fd = open(os.path.join(args.out_dir, "scores"), "w")
+ for trial in trials_list:
+ spk, utt, _ = trial.split(" ")
+ spk_emb = enroll_embedding[spk]
+ utt_emb = test_embedding[utt]
+ score = np.sum(spk_emb * utt_emb) / (np.linalg.norm(spk_emb) * np.linalg.norm(utt_emb))
+ fd.write("{} {} {:.5f}\n".format(spk, utt, score))
+ fd.close()
+
+ from funasr.utils.compute_eer import compute_eer
+ from funasr.utils.compute_min_dcf import compute_min_dcf
+ eer, threshold = compute_eer(args.trials, os.path.join(args.out_dir, "scores"))
+ print("EER is {:.4f} at threshold {:.4f}".format(eer * 100.0, threshold))
+
+ mindcf, threshold = compute_min_dcf(
+ os.path.join(args.out_dir, "scores"), args.trials,
+ c_miss=10, p_target=0.01
+ )
+ print("minDCF is {0:.4f} at threshold {1:.4f} (p-target={2}, c-miss={3}, c-fa={4})\n".format(
+ mindcf, threshold, 0.01, 10, 1
+ ))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer.py b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer.py
new file mode 100644
index 0000000..e81297a
--- /dev/null
+++ b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer.py
@@ -0,0 +1,33 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+import numpy as np
+
+if __name__ == '__main__':
+ inference_sv_pipline = pipeline(
+ task=Tasks.speaker_verification,
+ model='damo/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch'
+ )
+
+ # 鎻愬彇涓嶅悓鍙ュ瓙鐨勮璇濅汉宓屽叆鐮�
+ rec_result = inference_sv_pipline(
+ audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_enroll.wav')
+ enroll = rec_result["spk_embedding"]
+
+ rec_result = inference_sv_pipline(
+ audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_same.wav')
+ same = rec_result["spk_embedding"]
+
+ rec_result = inference_sv_pipline(
+ audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_different.wav')
+ different = rec_result["spk_embedding"]
+
+ # 瀵圭浉鍚岀殑璇磋瘽浜鸿绠椾綑寮︾浉浼煎害
+ sv_threshold = 0.9465
+ same_cos = np.sum(enroll * same) / (np.linalg.norm(enroll) * np.linalg.norm(same))
+ same_cos = max(same_cos - sv_threshold, 0.0) / (1.0 - sv_threshold) * 100.0
+ print("Similarity:", same_cos)
+
+ # 瀵逛笉鍚岀殑璇磋瘽浜鸿绠椾綑寮︾浉浼煎害
+ diff_cos = np.sum(enroll * different) / (np.linalg.norm(enroll) * np.linalg.norm(different))
+ diff_cos = max(diff_cos - sv_threshold, 0.0) / (1.0 - sv_threshold) * 100.0
+ print("Similarity:", diff_cos)
diff --git a/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer_sv.py b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer_sv.py
new file mode 100644
index 0000000..c51313d
--- /dev/null
+++ b/egs_modelscope/speaker_verification/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch/infer_sv.py
@@ -0,0 +1,21 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ inference_sv_pipline = pipeline(
+ task=Tasks.speaker_verification,
+ model='damo/speech_xvector_sv-zh-cn-cnceleb-16k-spk3465-pytorch'
+ )
+
+ # 涓や釜璇煶涓虹浉鍚岃璇濅汉
+ rec_result = inference_sv_pipline(audio_in=(
+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_enroll.wav',
+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_same.wav'))
+ print("Similarity", rec_result["scores"])
+
+ # 涓や釜璇煶涓轰笉鍚岃璇濅汉
+ rec_result = inference_sv_pipline(audio_in=(
+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_enroll.wav',
+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/sv_example_different.wav'))
+
+ print("Similarity", rec_result["scores"])
diff --git a/egs_modelscope/speechio/paraformer/README.md b/egs_modelscope/speechio/paraformer/README.md
deleted file mode 100644
index 669185f..0000000
--- a/egs_modelscope/speechio/paraformer/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# ModelScope: Paraformer-large Model
-
-## Highlight
-
-### ModelScope: Paraformer-Large Model
-- <strong>Fast</strong>: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup.
-- <strong>Accurate</strong>: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation.
-- <strong>Convenient</strong>: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference.
- - Support finetuning and inference on AISHELL-1 and AISHELL-2.
- - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio.
-
-## How to infer using a pretrained ModelScope Paraformer-large Model
-
-### Inference
-- Setting parameters in `paraformer_large_infer.sh`
- - <strong>ori_data:</strong> please set the speechio raw data path
- - <strong>data_dir:</strong> data output dictionary
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # base model, download from modelscope
- - <strong>test_sets:</strong> please set the testsets name
-- Then you can run the pipeline to infer with:
-```sh
- sh ./paraformer_large_infer.sh
-```
diff --git a/egs_modelscope/speechio/paraformer/RESULTS.md b/egs_modelscope/speechio/paraformer/RESULTS.md
deleted file mode 100644
index 9938e74..0000000
--- a/egs_modelscope/speechio/paraformer/RESULTS.md
+++ /dev/null
@@ -1,42 +0,0 @@
-# Paraformer-Large
-- Model link: <https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary>
-- Model size: 220M
-- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-
-# Environments
-- date: `Tue Nov 22 18:48:39 CST 2022`
-- python version: `3.7.12`
-- FunASR version: `0.1.0`
-- pytorch version: `pytorch 1.7.0`
-- Git hash: ``
-- Commit date: ``
-
-# Beachmark Results
-
-
-## SpeechIO TIOBE
-- Decode config 1: conf/decode_asr_transformer_noctc_1best.yaml
- - Decode without CTC
- - Decode without LM
-- Decode config 2: conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml
- - Decode without CTC
- - Decode with Transformer-LM
- - LM weight: 0.15
-
-| testset | w/o LM | w/ LM |
-|:------------------:|:----:|:----:|
-|SPEECHIO_ASR_ZH00001| 0.49 | 0.35 |
-|SPEECHIO_ASR_ZH00002| 3.23 | 2.86 |
-|SPEECHIO_ASR_ZH00003| 1.13 | 0.80 |
-|SPEECHIO_ASR_ZH00004| 1.33 | 1.10 |
-|SPEECHIO_ASR_ZH00005| 1.41 | 1.18 |
-|SPEECHIO_ASR_ZH00006| 5.25 | 4.85 |
-|SPEECHIO_ASR_ZH00007| 5.51 | 4.97 |
-|SPEECHIO_ASR_ZH00008| 3.69 | 3.18 |
-|SPEECHIO_ASR_ZH00009| 3.02 | 2.78 |
-|SPEECHIO_ASR_ZH000010| 3.35 | 2.99 |
-|SPEECHIO_ASR_ZH000011| 1.54 | 1.25 |
-|SPEECHIO_ASR_ZH000012| 2.06 | 1.68 |
-|SPEECHIO_ASR_ZH000013| 2.57 | 2.25 |
-|SPEECHIO_ASR_ZH000014| 3.86 | 3.08 |
-|SPEECHIO_ASR_ZH000015| 3.34 | 2.67 |
diff --git a/egs_modelscope/speechio/paraformer/modelscope_utils b/egs_modelscope/speechio/paraformer/modelscope_utils
deleted file mode 120000
index fc97768..0000000
--- a/egs_modelscope/speechio/paraformer/modelscope_utils
+++ /dev/null
@@ -1 +0,0 @@
-../../common/modelscope_utils
\ No newline at end of file
diff --git a/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh b/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh
deleted file mode 100755
index d0dc32b..0000000
--- a/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-ori_data=
-data_dir=
-exp_dir=
-model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch
-model_revision="v1.0.4" # please do not modify the model revision
-inference_nj=32
-gpuid_list="0" # set gpus, e.g., gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1 # the number of jobs for each gpu
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
-else
- inference_nj=$njob
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-test_sets="SPEECHIO_ASR_ZH00001 SPEECHIO_ASR_ZH00002 SPEECHIO_ASR_ZH00003 SPEECHIO_ASR_ZH00004 SPEECHIO_ASR_ZH00005 SPEECHIO_ASR_ZH00006 SPEECHIO_ASR_ZH00007 SPEECHIO_ASR_ZH00008 SPEECHIO_ASR_ZH00009 SPEECHIO_ASR_ZH00010 SPEECHIO_ASR_ZH00011 SPEECHIO_ASR_ZH00012 SPEECHIO_ASR_ZH00013 SPEECHIO_ASR_ZH00014 SPEECHIO_ASR_ZH00015"
-
-. utils/parse_options.sh
-
-for tset_name in ${test_sets}; do
- test_dir=${data_dir}/speechio/${tset_name}
- mkdir -p ${test_dir}
- find ${ori_data}/${tset_name} -iname "*.wav" > ${test_dir}/wav.flist
- sed -e 's/\.wav//' ${test_dir}/wav.flist | awk -F '/' '{print $NF}' > ${test_dir}/utt.list
- paste -d' ' ${test_dir}/utt.list ${test_dir}/wav.flist > ${test_dir}/wav.scp
- cp ${ori_data}/${tset_name}/trans.txt ${test_dir}/text
- sed -i "s/\t/ /g" ${test_dir}/text
-done
-
-mkdir -p ${exp_dir}/speechio
-
-modelscope_utils/modelscope_infer.sh \
- --data_dir ${data_dir}/speechio \
- --exp_dir ${exp_dir}/speechio \
- --test_sets "${test_sets}" \
- --model_name ${model_name} \
- --model_revision ${model_revision} \
- --inference_nj ${inference_nj} \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --gpu_inference ${gpu_inference} \
- --use_lm ${use_lm} \
- --beam_size ${beam_size} \
- --lm_weight ${lm_weight}
-
-# SpeechIO TIOBE textnorm
-for tset_name in ${test_sets}; do
- echo "$0 --> Normalizing REF text ..."
- ./utils/textnorm_zh.py \
- --has_key --to_upper \
- ${ori_data}/${tset_name}/trans.txt \
- ${data_dir}/speechio/${tset_name}/ref.txt
-
- cp ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/text.proc ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt
- sed -i "s#</s>##g" ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt
- echo "$0 --> Normalizing HYP text ..."
- ./utils/textnorm_zh.py \
- --has_key --to_upper \
- ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt \
- ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec.txt
- grep -v $'\t$' ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec.txt > ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec_non_empty.txt
-
- echo "$0 --> computing WER/CER and alignment ..."
- ./utils/error_rate_zh \
- --tokenizer char \
- --ref ${data_dir}/speechio/${tset_name}/ref.txt \
- --hyp ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec_non_empty.txt \
- ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/DETAILS.txt | tee ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/RESULTS.txt
-done
-
diff --git a/egs_modelscope/speechio/paraformer/path.sh b/egs_modelscope/speechio/paraformer/path.sh
deleted file mode 100755
index 7972642..0000000
--- a/egs_modelscope/speechio/paraformer/path.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-export FUNASR_DIR=$PWD/../../..
-
-# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8
-export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs_modelscope/speechio/paraformer/utils b/egs_modelscope/speechio/paraformer/utils
deleted file mode 120000
index 4e7a6ce..0000000
--- a/egs_modelscope/speechio/paraformer/utils
+++ /dev/null
@@ -1 +0,0 @@
-../../../egs/aishell/tranformer/utils
\ No newline at end of file
diff --git a/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/README.md b/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/README.md
new file mode 100644
index 0000000..6d9cd30
--- /dev/null
+++ b/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/README.md
@@ -0,0 +1,24 @@
+# ModelScope Model
+
+## How to finetune and infer using a pretrained ModelScope Model
+
+### Inference
+
+Or you can use the finetuned model for inference directly.
+
+- Setting parameters in `infer.py`
+ - <strong>audio_in:</strong> # support wav, url, bytes, and parsed audio format.
+ - <strong>output_dir:</strong> # If the input format is wav.scp, it needs to be set.
+
+- Then you can run the pipeline to infer with:
+```python
+ python infer.py
+```
+
+
+Modify inference related parameters in vad.yaml.
+
+- max_end_silence_time: The end-point silence duration to judge the end of sentence, the parameter range is 500ms~6000ms, and the default value is 800ms
+- speech_noise_thres: The balance of speech and silence scores, the parameter range is (-1,1)
+ - The value tends to -1, the greater probability of noise being judged as speech
+ - The value tends to 1, the greater probability of speech being judged as noise
diff --git a/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/infer.py b/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/infer.py
new file mode 100755
index 0000000..e11d5d2
--- /dev/null
+++ b/egs_modelscope/vad/speech_fsmn_vad_zh-cn-16k-common-pytorch/infer.py
@@ -0,0 +1,15 @@
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+
+if __name__ == '__main__':
+ audio_in = 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav'
+ output_dir = None
+ inference_pipline = pipeline(
+ task=Tasks.auto_speech_recognition,
+ model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ model_revision=None,
+ output_dir=output_dir,
+ batch_size=1,
+ )
+ segments_result = inference_pipline(audio_in=audio_in)
+ print(segments_result)
diff --git a/egs_modelscope/wenetspeech/paraformer/README.md b/egs_modelscope/wenetspeech/paraformer/README.md
deleted file mode 100644
index 9dc5f3f..0000000
--- a/egs_modelscope/wenetspeech/paraformer/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# ModelScope: Paraformer-large Model
-
-## Highlight
-
-### ModelScope: Paraformer-Large Model
-- <strong>Fast</strong>: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup.
-- <strong>Accurate</strong>: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation.
-- <strong>Convenient</strong>: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference.
- - Support finetuning and inference on AISHELL-1 and AISHELL-2.
- - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio.
-
-## How to infer using a pretrained ModelScope Paraformer-large Model
-
-### Inference
-- Setting parameters in `paraformer_large_infer.sh`
- - <strong>ori_data:</strong> please set the wenetspeech raw data path
- - <strong>data_dir:</strong> data output dictionary
- - <strong>exp_dir:</strong> the result path
- - <strong>model_name:</strong> speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # base model, download from modelscope
- - <strong>test_sets:</strong> please set the testsets name
-- Then you can run the pipeline to infer with:
-```sh
- sh ./paraformer_large_infer.sh
-```
diff --git a/egs_modelscope/wenetspeech/paraformer/RESULTS.md b/egs_modelscope/wenetspeech/paraformer/RESULTS.md
deleted file mode 100644
index a912c92..0000000
--- a/egs_modelscope/wenetspeech/paraformer/RESULTS.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# Paraformer-Large
-- Model link: <https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary>
-- Model size: 220M
-- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml
-
-# Environments
-- date: `Tue Nov 22 18:48:39 CST 2022`
-- python version: `3.7.12`
-- FunASR version: `0.1.0`
-- pytorch version: `pytorch 1.7.0`
-- Git hash: ``
-- Commit date: ``
-
-# Beachmark Results
-
-## Wenetspeech
-- Decode config: conf/decode_asr_transformer_noctc_1best.yaml
- - Decode without CTC
- - Decode without LM
-
-| testset | CER(%)|
-|:---------:|:-----:|
-| dev | 3.57 |
-| test | 6.97 |
-| test_net | 6.74 |
diff --git a/egs_modelscope/wenetspeech/paraformer/modelscope_utils b/egs_modelscope/wenetspeech/paraformer/modelscope_utils
deleted file mode 120000
index fc97768..0000000
--- a/egs_modelscope/wenetspeech/paraformer/modelscope_utils
+++ /dev/null
@@ -1 +0,0 @@
-../../common/modelscope_utils
\ No newline at end of file
diff --git a/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh b/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh
deleted file mode 100755
index b3dbbe4..0000000
--- a/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-ori_data=
-data_dir=
-exp_dir=
-model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch
-model_revision="v1.0.4" # please do not modify the model revision
-inference_nj=32
-gpuid_list="0" # set gpus, e.g., gpuid_list="0,1"
-ngpu=$(echo $gpuid_list | awk -F "," '{print NF}')
-njob=1 # the number of jobs for each gpu
-gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding
-
-if ${gpu_inference}; then
- inference_nj=$[${ngpu}*${njob}]
-else
- inference_nj=$njob
-fi
-
-# LM configs
-use_lm=false
-beam_size=1
-lm_weight=0.0
-
-test_sets="dev test_meeting test_net"
-
-. utils/parse_options.sh
-
-for tset_name in ${test_sets}; do
- test_dir=${data_dir}/wenetspeech/${tset_name}
- mkdir -p ${test_dir}
- find ${ori_data}/${tset_name} -iname "*.wav" > ${test_dir}/wav.flist
- sed -e 's/\.wav//' ${test_dir}/wav.flist | awk -F '/' '{print $NF}' > ${test_dir}/utt.list
- paste -d' ' ${test_dir}/utt.list ${test_dir}/wav.flist > ${test_dir}/wav.scp
- cp ${ori_data}/${tset_name}/trans.txt ${test_dir}/text
- sed -i "s/\t/ /g" ${test_dir}/text
-done
-
-mkdir -p ${exp_dir}/wenetspeech
-
-modelscope_utils/modelscope_infer.sh \
- --data_dir ${data_dir}/wenetspeech \
- --exp_dir ${exp_dir}/wenetspeech \
- --test_sets "${test_sets}" \
- --model_name ${model_name} \
- --model_revision ${model_revision} \
- --inference_nj ${inference_nj} \
- --gpuid_list ${gpuid_list} \
- --njob ${njob} \
- --gpu_inference ${gpu_inference} \
- --use_lm ${use_lm} \
- --beam_size ${beam_size} \
- --lm_weight ${lm_weight}
-
diff --git a/egs_modelscope/wenetspeech/paraformer/path.sh b/egs_modelscope/wenetspeech/paraformer/path.sh
deleted file mode 100755
index 7972642..0000000
--- a/egs_modelscope/wenetspeech/paraformer/path.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-export FUNASR_DIR=$PWD/../../..
-
-# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8
-export PATH=$FUNASR_DIR/funasr/bin:$PATH
diff --git a/egs_modelscope/wenetspeech/paraformer/utils b/egs_modelscope/wenetspeech/paraformer/utils
deleted file mode 120000
index 4e7a6ce..0000000
--- a/egs_modelscope/wenetspeech/paraformer/utils
+++ /dev/null
@@ -1 +0,0 @@
-../../../egs/aishell/tranformer/utils
\ No newline at end of file
diff --git a/fun_text_processing/README.md b/fun_text_processing/README.md
index 9778493..9947fa4 100644
--- a/fun_text_processing/README.md
+++ b/fun_text_processing/README.md
@@ -3,51 +3,30 @@
### Introduction
-FunTextProcessing is a Python toolkit for fundamental text processing in ASR including text processing , inverse text processing, which is included in the `FunASR`.
+FunTextProcessing is a Python toolkit for fundamental text processing in ASR including text processing , inverse text processing, num2words, which is included in the `FunASR`.
### Highlights
-- FunTextProcessing supports inverse text processing (ITN), text processing (TN).
-- FunTextProcessing supports multilingual, 10+ languages for ITN, 5 languages for TN.
+- FunTextProcessing supports inverse text processing (ITN), text processing (TN), number to words (num2words).
+- FunTextProcessing supports multilingual, 10+ languages for ITN, 5 languages for TN, 50+ languages for num2words.
-### Installation
-
-Fun Text Processing, specifically (Inverse) Text Normalization, requires Pynini to be installed.
-```
-bash fun_text_processing/install_pynini.sh
-```
### Example
#### Inverse Text Processing (ITN)
Given text inputs, such as speech recognition results, use `fun_text_processing/inverse_text_normalization/inverse_normalize.py` to output ITN results. You may refer to the following example scripts.
-- ITN example for English
```
-python fun_text_processing/inverse_text_normalization/inverse_normalize.py --text="one hundred twenty three" --language=en
-```
+test_file=fun_text_processing/inverse_text_normalization/id/id_itn_test_input.txt
-- ITN example for Indonesian
+python fun_text_processing/inverse_text_normalization/inverse_normalize.py --input_file $test_file --cache_dir ./itn_model/ --output_file output.txt --language=id
```
-python fun_text_processing/inverse_text_normalization/inverse_normalize.py --text="ratus dua puluh tiga" --language=id --cache_dir ./model/ --output_file output.txt
-cat output.txt
-```
-
-Arguments:
-- text - Input text. Should not exceed 500 words.
-- input_file - Input file with lines of input text. Only one of text or input_file is accepted.
-- output_file - Output file to save normalizations. Needed if input_file is specified.
-- language - language id.
-- input_case - Only for text normalization. lower_cased or cased.
-- verbose - Outputs intermediate information.
-- cache_dir - Specifies a cache directory for compiled grammars. If grammars exist, this significantly improves speed.
-- overwrite_cache - Updates grammars in cache.
-- whitelist - TSV file with custom mappings of written text to spoken form.
### Acknowledge
1. We borrowed a lot of codes from [NeMo](https://github.com/NVIDIA/NeMo).
2. We refered the codes from [WeTextProcessing](https://github.com/wenet-e2e/WeTextProcessing) for Chinese inverse text normalization.
+3. We borrowed a lot of codes from [num2words](https://pypi.org/project/num2words/) library for convert the number to words function in some languages.
### License
-This project is licensed under the Apache-2.0 license. FunTextProcessing also contains various third-party components and some code modified from other repos under other open source licenses.
+This project is licensed under the [The MIT License](https://opensource.org/licenses/MIT). FunTextProcessing also contains various third-party components and some code modified from other repos under other open source licenses.
diff --git a/fun_text_processing/__init__.py b/fun_text_processing/__init__.py
index e69de29..8b13789 100644
--- a/fun_text_processing/__init__.py
+++ b/fun_text_processing/__init__.py
@@ -0,0 +1 @@
+
diff --git a/fun_text_processing/inverse_text_normalization/de/__init__.py b/fun_text_processing/inverse_text_normalization/de/__init__.py
index 712514d..68b1048 100644
--- a/fun_text_processing/inverse_text_normalization/de/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/de/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/de/taggers/cardinal.py
index 09f273d..61058df 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/date.py b/fun_text_processing/inverse_text_normalization/de/taggers/date.py
index 9103f7d..e36a13a 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/de/taggers/decimal.py
index 68e7784..a037a97 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.taggers.decimal import get_quantity, quantities
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/de/taggers/electronic.py
index eefdb6a..185a524 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/de/taggers/fraction.py
index 09686f2..94fff82 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/measure.py b/fun_text_processing/inverse_text_normalization/de/taggers/measure.py
index 170f78b..681f157 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.taggers.measure import singular_to_plural, unit_singular
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/money.py b/fun_text_processing/inverse_text_normalization/de/taggers/money.py
index 790c1cd..091bde0 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.taggers.money import maj_singular, min_plural, min_singular
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/de/taggers/ordinal.py
index d770e98..bc4dd84 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/de/taggers/telephone.py
index 05c4c09..a0bfa20 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/time.py b/fun_text_processing/inverse_text_normalization/de/taggers/time.py
index 45fa8e0..9a30a9a 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py
index 1986418..c8c0296 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/de/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/de/taggers/whitelist.py
index 337e306..ab853d0 100644
--- a/fun_text_processing/inverse_text_normalization/de/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/de/taggers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py
index ea8d65a..b98fd48 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py
@@ -1,17 +1,5 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/decimal.py
index 348a174..3f65387 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_preserve_order
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/measure.py
index 98c9b44..2ba8f94 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/money.py
index 094c3ab..ce4a2b9 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/time.py
index d591b21..306e896 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_ALPHA, DAMO_DIGIT, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py
index 47b89c0..9d8a88e 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.de.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.de.verbalizers.decimal import DecimalFst
diff --git a/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py
index 934c0e6..78f2938 100644
--- a/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.de.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/en/__init__.py b/fun_text_processing/inverse_text_normalization/en/__init__.py
index 712514d..68b1048 100644
--- a/fun_text_processing/inverse_text_normalization/en/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/en/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/en/clean_eval_data.py b/fun_text_processing/inverse_text_normalization/en/clean_eval_data.py
index 60f57d2..0ea70c7 100644
--- a/fun_text_processing/inverse_text_normalization/en/clean_eval_data.py
+++ b/fun_text_processing/inverse_text_normalization/en/clean_eval_data.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from argparse import ArgumentParser
from typing import List
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/en/taggers/cardinal.py
index 5cec0d8..d2e138c 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path, num_to_word
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/date.py b/fun_text_processing/inverse_text_normalization/en/taggers/date.py
index 75de10c..05178f0 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/en/taggers/decimal.py
index 2809d4e..196f3fc 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/en/taggers/electronic.py
index 987ec04..d119176 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/en/taggers/fraction.py
index 470a163..6d4d4b9 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/measure.py b/fun_text_processing/inverse_text_normalization/en/taggers/measure.py
index ec5daa4..8aa9aca 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/money.py b/fun_text_processing/inverse_text_normalization/en/taggers/money.py
index 31e0955..7fdfb96 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/en/taggers/ordinal.py
index 296f324..e57aaae 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/en/taggers/punctuation.py
index 0b19860..73eb283 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/punctuation.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/en/taggers/telephone.py
index 97d92ef..2865c71 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/time.py b/fun_text_processing/inverse_text_normalization/en/taggers/time.py
index 0ae55a1..07a0773 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py
index 32b90b8..d2bddec 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/en/taggers/whitelist.py
index bddfc77..f333098 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/en/taggers/word.py b/fun_text_processing/inverse_text_normalization/en/taggers/word.py
index 6b3d70e..37775e4 100644
--- a/fun_text_processing/inverse_text_normalization/en/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/en/taggers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/en/utils.py b/fun_text_processing/inverse_text_normalization/en/utils.py
index 782688c..5c0c8b5 100644
--- a/fun_text_processing/inverse_text_normalization/en/utils.py
+++ b/fun_text_processing/inverse_text_normalization/en/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
from typing import Union
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py
index c50572c..4c3f0eb 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/date.py
index bb462c3..0e5d461 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/decimal.py
index 39b61b2..9a5041a 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/electronic.py
index 97ff436..95bffb4 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/fraction.py
index 6bbf7ef..fc2f13c 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/measure.py
index 990df91..b3fe932 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/money.py
index 9eaf0b3..89beb93 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py
index d0264b7..f2c709f 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/telephone.py
index 6e62c29..a154aac 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/time.py
index 1c9b3c7..49175f6 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize.py
index 7208619..2f7eca5 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.en.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.en.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py
index b0bc9ad..02ed2f9 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py
index 3d71cbe..f5a5f12 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/en/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/en/verbalizers/word.py
index 05b67c6..99861dc 100644
--- a/fun_text_processing/inverse_text_normalization/en/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/en/verbalizers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/__init__.py b/fun_text_processing/inverse_text_normalization/es/__init__.py
index 4325904..b95c71e 100644
--- a/fun_text_processing/inverse_text_normalization/es/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/es/__init__.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.es.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.es.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/es/taggers/cardinal.py
index ba548b2..425ec8f 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/date.py b/fun_text_processing/inverse_text_normalization/es/taggers/date.py
index e25c470..940fe41 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/es/taggers/decimal.py
index 2a63b78..a2c4cc0 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/es/taggers/electronic.py
index 06055fe..d4148f9 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/measure.py b/fun_text_processing/inverse_text_normalization/es/taggers/measure.py
index cca0e78..cf9af0c 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/money.py b/fun_text_processing/inverse_text_normalization/es/taggers/money.py
index dae53ba..3b32959 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/es/taggers/ordinal.py
index da9f47f..eb1a0ed 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/es/taggers/punctuation.py
index 0b19860..73eb283 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/punctuation.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/es/taggers/telephone.py
index c1d5307..fa68336 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/time.py b/fun_text_processing/inverse_text_normalization/es/taggers/time.py
index ee4ade1..99f8d3d 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py
index dc5a3ac..068c6f5 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/es/taggers/whitelist.py
index d44827d..947ecd6 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/es/taggers/word.py b/fun_text_processing/inverse_text_normalization/es/taggers/word.py
index 6b3d70e..37775e4 100644
--- a/fun_text_processing/inverse_text_normalization/es/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/es/taggers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/es/utils.py b/fun_text_processing/inverse_text_normalization/es/utils.py
index fdedabf..9cbd175 100644
--- a/fun_text_processing/inverse_text_normalization/es/utils.py
+++ b/fun_text_processing/inverse_text_normalization/es/utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py
index 95cfe3d..143651a 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/date.py
index c38da37..02148d8 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/decimal.py
index e110de0..7212752 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/electronic.py
index 47cdf96..53fd9d9 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/measure.py
index 30f817d..f8512b0 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/money.py
index eadb9c9..e35a2fb 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py
index e437dc3..3e0b6b1 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/telephone.py
index 144c014..09bc2e6 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/time.py
index 1b74c67..4b8b6fc 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize.py
index 873cf47..b1f0487 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.es.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.es.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py
index 37f6e01..65182b9 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.es.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py
index 30abce3..5672490 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/es/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/es/verbalizers/word.py
index 05b67c6..99861dc 100644
--- a/fun_text_processing/inverse_text_normalization/es/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/es/verbalizers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/export_models.py b/fun_text_processing/inverse_text_normalization/export_models.py
index 49149c2..9460ad9 100644
--- a/fun_text_processing/inverse_text_normalization/export_models.py
+++ b/fun_text_processing/inverse_text_normalization/export_models.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
from time import perf_counter
from argparse import ArgumentParser
@@ -21,12 +9,6 @@
parser.add_argument(
"--language", help="language", choices=['de', 'en', 'es', 'fr', 'id', 'ja', 'ko', 'pt', 'ru', 'vi', 'zh'], default="en", type=str
)
-
- parser.add_argument(
- "--token_and_classify_and_verbalize", help="export the single token&classify and verbalize or combined", choices=['single', 'combine'],
- default="combine", type=str
- )
-
parser.add_argument(
"--export_dir",
help="path to export directory. Default to current directory.",
@@ -83,17 +65,10 @@
os.makedirs(export_dir, exist_ok=True)
tagger_far_file = os.path.join(export_dir, args.language + "_itn_tagger.far")
verbalizer_far_file = os.path.join(export_dir, args.language + "_itn_verbalizer.far")
- tager_and_verbalizer_far_file = os.path.join(export_dir, args.language, "model.far" )
start_time = perf_counter()
tagger_fst, verbalizer_fst = get_grammars(args.language)
-
- if args.token_and_classify_and_verbalize == 'single':
- generator_main(tagger_far_file, {"tokenize_and_classify": tagger_fst})
- generator_main(verbalizer_far_file, {"verbalize": verbalizer_fst})
- elif args.token_and_classify_and_verbalize == 'combine':
- if not os.path.exists(os.path.join(export_dir, args.language)):
- os.makedirs(os.path.join(export_dir, args.language))
- generator_main(tager_and_verbalizer_far_file, {"tokenize_and_classify": tagger_fst, "verbalize": verbalizer_fst})
+ generator_main(tagger_far_file, {"tokenize_and_classify": tagger_fst})
+ generator_main(verbalizer_far_file, {"verbalize": verbalizer_fst})
print(f'Time to generate graph: {round(perf_counter() - start_time, 2)} sec')
diff --git a/fun_text_processing/inverse_text_normalization/fr/__init__.py b/fun_text_processing/inverse_text_normalization/fr/__init__.py
index 431a014..d30c1f8 100644
--- a/fun_text_processing/inverse_text_normalization/fr/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/fr/__init__.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/fr/graph_utils.py b/fun_text_processing/inverse_text_normalization/fr/graph_utils.py
index f023be1..f8b5086 100644
--- a/fun_text_processing/inverse_text_normalization/fr/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/fr/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/fr/taggers/cardinal.py
index 120ec14..e78f867 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/date.py b/fun_text_processing/inverse_text_normalization/fr/taggers/date.py
index ce4d19b..9e513c0 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_extra_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/fr/taggers/decimal.py
index 696f2af..2199bc7 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/fr/taggers/electronic.py
index a67498b..04dfcbd 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_ALPHA, GraphFst, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/fr/taggers/fraction.py
index 5ed8bfa..324f77e 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/measure.py b/fun_text_processing/inverse_text_normalization/fr/taggers/measure.py
index 75878ba..1cface9 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/money.py b/fun_text_processing/inverse_text_normalization/fr/taggers/money.py
index cee426a..b2e8b3a 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/fr/taggers/ordinal.py
index 5a26217..633f849 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/fr/taggers/punctuation.py
index ebfef0a..5e7f706 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/punctuation.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/fr/taggers/telephone.py
index c4edca9..6705ade 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/time.py b/fun_text_processing/inverse_text_normalization/fr/taggers/time.py
index 771fc00..ff534f7 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py
index 20aac42..3f722f6 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/fr/taggers/whitelist.py
index 4739d2a..9db2738 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/taggers/word.py b/fun_text_processing/inverse_text_normalization/fr/taggers/word.py
index 4084db1..b17a7be 100644
--- a/fun_text_processing/inverse_text_normalization/fr/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/fr/taggers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/fr/utils.py b/fun_text_processing/inverse_text_normalization/fr/utils.py
index fdedabf..9cbd175 100644
--- a/fun_text_processing/inverse_text_normalization/fr/utils.py
+++ b/fun_text_processing/inverse_text_normalization/fr/utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py
index be44f73..c901e3e 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/date.py
index 363e6fe..da2071b 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py
index ef2c9ec..daa532e 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py
index 7308fd3..06139e0 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py
index 85f009a..cecd3e2 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/measure.py
index ffb3b27..6f0eb0c 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/money.py
index dfeb75d..fdeb35e 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py
index 5cc2eac..92d715b 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py
index e5c5322..2985f50 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/time.py
index 5e0b2d2..c75c69c 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize.py
index 1c42991..40271aa 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py
index 418a702..f284aac 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_extra_space, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py
index aa355fa..c94eebf 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/fr/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/fr/verbalizers/word.py
index 151a86c..1339dac 100644
--- a/fun_text_processing/inverse_text_normalization/fr/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/fr/verbalizers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/clean_eval_data.py b/fun_text_processing/inverse_text_normalization/id/clean_eval_data.py
index e8fd1ea..97abe13 100644
--- a/fun_text_processing/inverse_text_normalization/id/clean_eval_data.py
+++ b/fun_text_processing/inverse_text_normalization/id/clean_eval_data.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from argparse import ArgumentParser
from typing import List
diff --git a/fun_text_processing/inverse_text_normalization/id/graph_utils.py b/fun_text_processing/inverse_text_normalization/id/graph_utils.py
index 8192982..ccc9fa9 100644
--- a/fun_text_processing/inverse_text_normalization/id/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/id/graph_utils.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
import string
from pathlib import Path
diff --git a/fun_text_processing/inverse_text_normalization/id/id_itn_test_expected.txt b/fun_text_processing/inverse_text_normalization/id/id_itn_test_expected.txt
new file mode 100644
index 0000000..dfee6bd
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/id/id_itn_test_expected.txt
@@ -0,0 +1,29 @@
+2022
+300
+9999
+100001
+100
+1000
+10289
+1289
+01 2345-6789
+14
+15
+16
+17
+18
+19
+20
+106
+600
+100
+100
+1 miliar
+123
+123
+24 maret
+10076
+100076
+10076 rupiah
+76
++62 21 6539-0605
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/id/id_itn_test_input.txt b/fun_text_processing/inverse_text_normalization/id/id_itn_test_input.txt
new file mode 100644
index 0000000..cf1b3b8
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/id/id_itn_test_input.txt
@@ -0,0 +1,29 @@
+dua ribu dua puluh dua
+tiga ribu
+sembilan ribu sembilan ratus sembilan puluh sembilan
+seribu satu
+ribu
+seribu
+seribu dua ratus delapan puluh sembilan
+ribu dua ratus delapan puluh sembilan
+nol satu dua tiga empat lima enam tujuh delapan sembilan
+empat belas
+limabelas
+enambelas
+tujuh belas
+delapan belas
+sembilan belas
+dua puluh
+seratus enam
+enam ratus
+ratus
+seratus
+satu miliar
+seratus dua puluh tiga
+ratus dua puluh tiga
+dua puluh empat maret
+ribu tujuh puluh enam
+seribu tujuh puluh enam
+ribu tujuh puluh enam rupiah
+tujuh puluh enam
+ditambah enam dua dua satu enam lima tiga sembilan nol enam nol lima
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/id/id_unit_test.tsv b/fun_text_processing/inverse_text_normalization/id/id_unit_test.tsv
new file mode 100644
index 0000000..662bf04
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/id/id_unit_test.tsv
@@ -0,0 +1,29 @@
+dua ribu dua puluh dua 2022
+tiga ribu 300
+sembilan ribu sembilan ratus sembilan puluh sembilan 9999
+seribu satu 100001
+ribu 100
+seribu 1000
+seribu dua ratus delapan puluh sembilan 10289
+ribu dua ratus delapan puluh sembilan 1289
+nol satu dua tiga empat lima enam tujuh delapan sembilan 01 2345-6789
+empat belas 14
+limabelas 15
+enambelas 16
+tujuh belas 17
+delapan belas 18
+sembilan belas 19
+dua puluh 20
+seratus enam 106
+enam ratus 600
+ratus 100
+seratus 100
+satu miliar 1 miliar
+seratus dua puluh tiga 123
+ratus dua puluh tiga 123
+dua puluh empat maret 24 maret
+ribu tujuh puluh enam 10076
+seribu tujuh puluh enam 100076
+ribu tujuh puluh enam rupiah 10076 rupiah
+tujuh puluh enam 76
+ditambah enam dua dua satu enam lima tiga sembilan nol enam nol lima +62 21 6539-0605
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/id/taggers/cardinal.py
index f2123a6..6b2fce5 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/cardinal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path, num_to_word
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/date.py b/fun_text_processing/inverse_text_normalization/id/taggers/date.py
index e914e77..7ae0a14 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/date.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/id/taggers/decimal.py
index ad8c3c0..c517748 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/decimal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/id/taggers/electronic.py
index 26d4d75..b5aba46 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/electronic.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_ALPHA, GraphFst, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/id/taggers/fraction.py
index 9345b0d..48f3a98 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/fraction.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from fun_text_processing.inverse_text_normalization.id.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/measure.py b/fun_text_processing/inverse_text_normalization/id/taggers/measure.py
index 2effbf8..458469b 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/measure.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/money.py b/fun_text_processing/inverse_text_normalization/id/taggers/money.py
index d56d852..61d20d5 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/money.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/id/taggers/ordinal.py
index 3eac1d1..3d68d96 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/ordinal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_CHAR, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/id/taggers/punctuation.py
index 261bd34..611fdfa 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/punctuation.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/id/taggers/telephone.py
index 31185b8..8875d29 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/telephone.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/time.py b/fun_text_processing/inverse_text_normalization/id/taggers/time.py
index 2a78616..3a866f5 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/time.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.taggers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path, num_to_word
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/id/taggers/tokenize_and_classify.py
index 02df7b4..2460b34 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/tokenize_and_classify.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/id/taggers/whitelist.py
index e7d3b80..87a54c1 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/whitelist.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.id.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/id/taggers/word.py b/fun_text_processing/inverse_text_normalization/id/taggers/word.py
index 24c66f4..76098c5 100644
--- a/fun_text_processing/inverse_text_normalization/id/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/id/taggers/word.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_SPACE, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/text_cases.tsv b/fun_text_processing/inverse_text_normalization/id/text_cases.tsv
new file mode 100644
index 0000000..d8c396a
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/id/text_cases.tsv
@@ -0,0 +1,40 @@
+dua ribu dua puluh dua 2022
+nol satu dua tiga empat lima enam tujuh delapan sembilan 0123456789
+empat belas 14
+limabelas 15
+enambelas 16
+tujuh belas 17
+delapan belas 18
+sembilan belas 19
+dua puluh 20
+seratus enam 106
+enam ratus 600
+ratus 100
+seratus 100
+satu juta 1,000,000
+satu miliar 1 miliar
+seratus dua puluh tiga 123
+ratus dua puluh tiga 123
+dua puluh empat maret 24th March
+seribu dua ratus delapan puluh sembilan 1289
+lima juta tiga ribu tujuh puluh enam rupiah Rp5003076
+ribu tujuh puluh enam rupiah Rp1076
+tujuh puluh enam rupiah dollar $1076
+ditambah enam dua dua satu enam lima tiga sembilan nol enam nol lima +62 21 6539-0605
+tiga ribu 3000
+sembilan ribu sembilan ratus sembilan puluh sembilan 9999
+seribu satu 1001
+nol 0
+satu 1
+dua 2
+tiga 3
+empat 4
+lima 5
+enam 6
+tujuh 7
+delapan 8
+sembilan 9
+sepuluh 10
+sebelas 11
+dua belas 12
+tigabelas 13
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/id/utils.py b/fun_text_processing/inverse_text_normalization/id/utils.py
index 782688c..9b97867 100644
--- a/fun_text_processing/inverse_text_normalization/id/utils.py
+++ b/fun_text_processing/inverse_text_normalization/id/utils.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
from typing import Union
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/cardinal.py
index eb636f9..741a62b 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/cardinal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/date.py
index ddc0449..82ad1fc 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/date.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
DAMO_NOT_QUOTE,
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/decimal.py
index c1b2563..044780d 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/decimal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/electronic.py
index 0c6acdb..d88424b 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/electronic.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/fraction.py
index b4f3354..15557da 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/fraction.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from fun_text_processing.inverse_text_normalization.id.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/measure.py
index d0492b9..8695cc9 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/measure.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_CHAR, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/money.py
index 6a64ba4..d02faaa 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/money.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_CHAR, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/ordinal.py
index 5c6c989..0571f78 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/ordinal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/telephone.py
index 195a120..fb41d58 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/telephone.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_NOT_QUOTE, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/time.py
index 7084219..da02ae2 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/time.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import (
DAMO_CHAR,
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize.py
index 0cf4624..8271474 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from fun_text_processing.inverse_text_normalization.id.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.id.verbalizers.date import DateFst
from fun_text_processing.inverse_text_normalization.id.verbalizers.decimal import DecimalFst
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize_final.py
index edb1604..d5fcf58 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/verbalize_final.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.verbalizers.verbalize import VerbalizeFst
from fun_text_processing.inverse_text_normalization.id.verbalizers.word import WordFst
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/whitelist.py
index abc1b8a..f1ced47 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/whitelist.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/id/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/id/verbalizers/word.py
index 6ffde71..9a638f2 100644
--- a/fun_text_processing/inverse_text_normalization/id/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/id/verbalizers/word.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.id.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ja/__init__.py b/fun_text_processing/inverse_text_normalization/ja/__init__.py
index 712514d..a3740c2 100644
--- a/fun_text_processing/inverse_text_normalization/ja/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/ja/__init__.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst
diff --git a/fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py b/fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py
index b72e136..c1e1bcf 100644
--- a/fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py
+++ b/fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from argparse import ArgumentParser
from typing import List
diff --git a/fun_text_processing/inverse_text_normalization/ja/graph_utils.py b/fun_text_processing/inverse_text_normalization/ja/graph_utils.py
index 3a20bfd..d645118 100644
--- a/fun_text_processing/inverse_text_normalization/ja/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/ja/graph_utils.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
import string
from pathlib import Path
diff --git a/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_expected.txt b/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_expected.txt
new file mode 100644
index 0000000..f207d87
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_expected.txt
@@ -0,0 +1,20 @@
+ps3锛宲s4锛孉TM锛屽崄鏁般亱浜屽崄锛屽垢鍥涢儙锛岀櫧闆Й銇ㄤ竷浜恒伄銇撱伋銇紝涓冨锛屽崈澶滀竴澶滐紝涓夌櫨鍏崄琛岋紝鍗佷簲銇湀锛屽喎鍑嶄笁瓒筹紝涓夊崄骞存渤鏉憋紝涓夊崄骞存渤瑗匡紝鍗冨勾寰岋紝浜旀湀澶╋紝瀹夊�嶆檵涓夛紝灏忔硥绱斾竴閮庯紝灞辨湰浜斿崄鍏�
+绗�1銆佺2銆佺3銆佺4銆佺5銆佺6銆佺7銆佺8銆佺9銆佺10銆佺11銆佺12銆佺13銆佺14銆佺15銆佺16銆佺17銆佺18銆佺19銆佺20銆�20銆佺24銆�24銆佺56銆佺100銆佺101
+銇с�併倧銇熴仐銇叆銇ｃ仧鍘熺鏂欍亱銈夈伄1/5銈掋亗銇仧銇浮銇欍仺銇勩亞銇撱仺銇伄銇犮亴銆傚爡銇傘亽鎰涖伅娣便亸銆屼綋銇�1/3銆�1/4銆�1/2銇爡銇傘亽銇仾銇ｃ仸銇勩伨銇欍��
+06骞�4鏈堬綖19骞�12鏈堛伀鏃ユ湰銇ф壙瑾嶃仌銈屻仧鍖荤檪姗熷櫒銇�529浠躲�併仢銇亞銇″皬鍏愮敤銇倧銇氥亱12浠躲��
+锛戯紮锛欙紦骞淬伀瑾曠敓銇椼仧鍚屽晢鍝併伀銇°仾銇裤�佺磩锛擄紣骞村墠銆侊紥锛愭銇爟銇垢鍥涢儙銇啓鐪熴倰鍏枊銆�
+濂棙銇欍倠鑷韩銇Э銈掑弾銈併仧銉夈偔銉ャ儭銉炽偪銉兗鏄犵敾銆岋紤锛愬剟鍐嗙銇愩�嶏紙锛戯紤鏈堬紥锛愭棩鏉变含銇у叕闁嬶級銈掞及锛层仚銈嬨仧銈佹澅浜兘鍐呫仹瀛︾敓鍚戙亼銈汇儫銉娿兗銈掗枊鍌�傘偛銈广儓銇�岋即锛讥銆�铏庤垶绔溿�嶃伄楂樻銈搞儳銉笺偢銇屻�佷綔瑭炰綔鏇层仐銇熴�屻儹銉笺儔銆嶃仹鍗扮◣锛戯紪鍎勫唵銈掔銇勩仩銇ㄦ槑銇嬨仐銆佷細鍫淬伅銇┿倛銈併亜銇熴��
+鐧哄２銇洸銇紥锛掞紣涓囨灇銈掑２銈娿�侀珮姗嬨伅鍗扮◣銈掋仾銈撱仺锛掑勾銇т娇銇勫垏銇ｃ仧銇濄亞銇犮亴銆岀殕銇曘倱銇屻偒銉┿偑銈便仹锛戝洖姝屻亞銇紬鍐嗗叆銈娿伨銇欍仐銆佷粖銈傚勾闁擄紤锛掞紣锛愪竾鍐嗐亹銈夈亜銆侀粰銇ｃ仸銇︺倐鍏ャ仯銇︺亶銇俱仚銆嶃�備綍銇с倐銇亜銈堛亞銇亾銇ㄣ亴骞搞仜銇犮仯銇熴仺鎬濄亞銇ㄦ瓕銇ｃ仧鏇层伅銆併仺銈撱仹銈傘仾銇勫嵃绋庛倰鐢熴伩鍑恒仐銇︺亜銇︺�併儐銉兗銈傞檷鍙傘�俆ikToker銈嗐倞銇們浣撻噸39kg鈥�15kg娓涢噺鈥濇灄銇裤仾銇汇偄銉娿��
+15鏃ャ伀绗�64鍥炴棩鏈儸銈炽兗銉夊ぇ璩烇紙涓诲偓鏃ユ湰浣滄洸瀹跺崝浼氾級銇悇璩炪亴鐧鸿〃銇曘倢銇熴亴銆併亾銈屻伨銇�12骞撮�ｇ稓銇у劒绉�浣滃搧璩炪倰鍙楄碁銇椼仸銇勩仧AK47銇儶銈广儓銇叆銈夈仛銆佽閷层亴閫旂刀銇堛仸銇勩仧銆�
+闁嬬エ銇岀稓銇忋偄銉°儶銈伄涓枔閬告寵銇ц鏁般伄銈€儭銉偒涓昏銉°儑銈ｃ偄銇�11鏈�16鏃ャ�侀噹鍏氥兓鍏卞拰鍏氥亴瀹氭暟435銇�ｉ偊璀颁細銇笅闄€仹218璀板腑銈掔嵅寰椼仐4骞淬伓銈娿伀澶氭暟娲俱倰濂倓銇椼仧銇ㄥ牨銇樸仧銆�
+銇撱倢銇�2021鏈�1鏈堛伀璧枫亾銇ｃ仧閫ｉ偊璀颁細璀颁簨鍫傝ゲ鎾冧簨浠躲伀闁€仚銈嬩笅闄€伄鐗瑰垾瑾挎熁濮斿摗浼氥伅瑙ｆ暎銇曘仜銈夈倢銈嬨亾銇ㄣ伀銇倞銇濄亞銇犮��
+銆屾槰骞淬伅鏂拌胺銇屾湰瑾垮瓙銇樸們銇亜涓仹銆併仢銈屻仹銈傘亗銈屻仩銇戣蛋銇ｃ仸銇忋倢銇﹀姪銇嬨倞銇俱仐銇熴�嶃仺鐝惧湪銇ソ瑾裤伓銈娿亴浼濄亪銈夈倢銇︺亜銇熸柊璋烽伕鎵嬨伅11鏈�13鏃ャ伀琛屻倧銈屻仧鏉辨棩鏈コ瀛愰浼濄仹銈€兂銈兗銈掑嫏銈併��10銈儹銈�31鍒�08绉掋伄鍖洪枔璩炪仹鏉变含銇�嗚虎鍎嫕銇珛褰硅�呫仺銇倞銇俱仐銇熴��
+銉曘儷銉炪儵銈姐兂銇儥銈广儓銈裤偆銉犮伅2鏅傞枔40鍒�34绉掋��
+涔呯暀绫冲競銇с伅浠婃湞涓�鏅傞枔銇�92鐐�5mm銇寷鐑堛仾闆ㄣ倰瑕虫脯涓�鏅傞枔褰撱仧銈娿伄闆ㄩ噺銇ㄣ仐銇︺伅1977骞淬伄绲辫▓闁嬪浠ユ潵鏈�澶с仹銇�
+娌荤檪銈掑繀瑕併仺銇欍倠鍕曡剤绠￠枊瀛樼棁銇亗銈嬭丹銇°們銈撱伅銆�1500銈般儵銉犳湭婧�銇с伅绱�30%銆�1000銈般儵銉犳湭婧�銇с伅绱�50%銇ㄣ仌銈屻倠銆傝柆銇ц绠°亴闁夈仒銈嬨亾銇ㄣ倐澶氥亜銇屻�佸僵钁夈仭銈冦倱銇柆銇с伅琛�绠°亴闁夈仒銇亱銇ｃ仧銆�668km銆�
+100|150|123|111|123|0|0|1|2|3|10|11|12|13|15|19|20|50|99|1200|1234|1011|1997|01234567890|123456789|102|324|100|200|1000|5000|1涓噟50涓噟1000000|4000涓噟6鍎剕10鍎剕9鍏�
+1679
+10086
+08613794568
+銈姐兗銈枫儯銉儑銈ｃ偣0锛�0锛�1锛�2锛�3锛�120锛�324锛�100锛�200锛�1000锛�1500锛�1679锛�5000锛�10000锛�1000000锛�10000000
+銈裤兂銈广伀銈傞亰銇冲績326銇岄殸銇曘倢銇︺亜銇俱仐銇熴亴
diff --git a/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_input.txt b/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_input.txt
new file mode 100644
index 0000000..c621475
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/ja/ja_itn_test_input.txt
@@ -0,0 +1,20 @@
+ps涓夛紝ps鍥涳紝ATM锛屽崄鏁般亱浜屽崄锛屽垢鍥涢儙锛岀櫧闆Й銇ㄤ竷浜恒伄銇撱伋銇紝涓冨锛屽崈澶滀竴澶滐紝涓夌櫨鍏崄琛岋紝鍗佷簲銇湀锛屽喎鍑嶄笁瓒筹紝涓夊崄骞存渤鏉憋紝涓夊崄骞存渤瑗匡紝鍗冨勾寰岋紝浜旀湀澶╋紝瀹夊�嶆檵涓夛紝灏忔硥绱斾竴閮庯紝灞辨湰浜斿崄鍏�
+绗竴銆佺浜屻�佺涓夈�佺鍥涖�佺浜斻�佺鍏�佺涓冦�佺鍏�佺涔濄�佺鍗併�佺鍗佷竴銆佺鍗佷簩銆佺鍗佷笁銆佺鍗佸洓銆佺鍗佷簲銆佺鍗佸叚銆佺鍗佷竷銆佺鍗佸叓銆佺鍗佷節銆佺浜屽崄銆佷簩鍗併�佺浜屽崄鍥涖�佷簩鍗佸洓銆佺浜斿崄鍏�佺鐧俱�佺鐧句竴
+銇с�併倧銇熴仐銇叆銇ｃ仧鍘熺鏂欍亱銈夈伄浜斿垎銇竴銆佷簲鍒嗐伄涓夈倰銇傘仾銇熴伀娓°仚銇ㄣ亜銇嗐亾銇ㄣ仾銇仩銇屻�傚爡銇傘亽鎰涖伅娣便亸銆屼綋銇笁鍒嗐伄涓�銆佸洓鍒嗐伄涓�銆佷簩鍒嗐伄涓�銇爡銇傘亽銇仾銇ｃ仸銇勩伨銇欍��
+闆跺叚骞村洓鏈堬綖涓�涔濆勾鍗佷簩鏈堛伀鏃ユ湰銇ф壙瑾嶃仌銈屻仧鍖荤檪姗熷櫒銇簲鐧句簩鍗佷節浠躲�併仢銇亞銇″皬鍏愮敤銇倧銇氥亱鍗佷簩浠躲��
+涓�涔濅節涓夊勾銇獣鐢熴仐銇熷悓鍟嗗搧銇仭銇伩銆佺磩涓夊崄骞村墠銆佷簩鍗佹銇爟銇垢鍥涢儙銇啓鐪熴倰鍏枊銆�
+濂棙銇欍倠鑷韩銇Э銈掑弾銈併仧銉夈偔銉ャ儭銉炽偪銉兗鏄犵敾銆屽崄鍎勫唵绋笺亹銆嶏紙鍗佷竴鏈堜簩鍗佹棩鏉变含銇у叕闁嬶級銈掞及锛层仚銈嬨仧銈佹澅浜兘鍐呫仹瀛︾敓鍚戙亼銈汇儫銉娿兗銈掗枊鍌�傘偛銈广儓銇�岋即锛讥銆�铏庤垶绔溿�嶃伄楂樻銈搞儳銉笺偢銇屻�佷綔瑭炰綔鏇层仐銇熴�屻儹銉笺儔銆嶃仹鍗扮◣鍗佸叚鍎勫唵銈掔銇勩仩銇ㄦ槑銇嬨仐銆佷細鍫淬伅銇┿倛銈併亜銇熴��
+鐧哄２銇洸銇簩鐧句簩鍗佷竾鏋氥倰澹层倞銆侀珮姗嬨伅鍗扮◣銈掋仾銈撱仺浜屽勾銇т娇銇勫垏銇ｃ仧銇濄亞銇犮亴銆岀殕銇曘倱銇屻偒銉┿偑銈便仹涓�鍥炴瓕銇嗐仺涓冨唵鍏ャ倞銇俱仚銇椼�佷粖銈傚勾闁撲竴鍗冧簩鐧句竾鍐嗐亹銈夈亜銆侀粰銇ｃ仸銇︺倐鍏ャ仯銇︺亶銇俱仚銆嶃�備綍銇с倐銇亜銈堛亞銇亾銇ㄣ亴骞搞仜銇犮仯銇熴仺鎬濄亞銇ㄦ瓕銇ｃ仧鏇层伅銆併仺銈撱仹銈傘仾銇勫嵃绋庛倰鐢熴伩鍑恒仐銇︺亜銇︺�併儐銉兗銈傞檷鍙傘�俆ikToker銈嗐倞銇們浣撻噸涓夊崄涔濄偔銉�滃崄浜斻偔銉笡閲忊�濇灄銇裤仾銇汇偄銉娿��
+鍗佷簲鏃ャ伀绗叚鍗佸洓鍥炴棩鏈儸銈炽兗銉夊ぇ璩烇紙涓诲偓鏃ユ湰浣滄洸瀹跺崝浼氾級銇悇璩炪亴鐧鸿〃銇曘倢銇熴亴銆併亾銈屻伨銇у崄浜屽勾閫ｇ稓銇у劒绉�浣滃搧璩炪倰鍙楄碁銇椼仸銇勩仧AK鍥涘崄涓冦伅銉偣銉堛伀鍏ャ倝銇氥�佽閷层亴閫旂刀銇堛仸銇勩仧銆�
+闁嬬エ銇岀稓銇忋偄銉°儶銈伄涓枔閬告寵銇ц鏁般伄銈€儭銉偒涓昏銉°儑銈ｃ偄銇崄涓�鏈堝崄鍏棩銆侀噹鍏氥兓鍏卞拰鍏氥亴瀹氭暟鍥涚櫨涓夊崄浜斻伄閫ｉ偊璀颁細銇笅闄€仹浜岀櫨鍗佸叓璀板腑銈掔嵅寰椼仐鍥涘勾銇躲倞銇鏁版淳銈掑オ閭勩仐銇熴仺鍫便仒銇熴��
+銇撱倢銇т簩銆囦簩涓�鏈堜竴鏈堛伀璧枫亾銇ｃ仧閫ｉ偊璀颁細璀颁簨鍫傝ゲ鎾冧簨浠躲伀闁€仚銈嬩笅闄€伄鐗瑰垾瑾挎熁濮斿摗浼氥伅瑙ｆ暎銇曘仜銈夈倢銈嬨亾銇ㄣ伀銇倞銇濄亞銇犮��
+銆屾槰骞淬伅鏂拌胺銇屾湰瑾垮瓙銇樸們銇亜涓仹銆併仢銈屻仹銈傘亗銈屻仩銇戣蛋銇ｃ仸銇忋倢銇﹀姪銇嬨倞銇俱仐銇熴�嶃仺鐝惧湪銇ソ瑾裤伓銈娿亴浼濄亪銈夈倢銇︺亜銇熸柊璋烽伕鎵嬨伅鍗佷竴鏈堝崄涓夋棩銇銈忋倢銇熸澅鏃ユ湰濂冲瓙椐呬紳銇с偄銉炽偒銉笺倰鍕欍倎銆佸崄銈儹銈掍笁鍗佷竴鍒嗐�囧叓绉掋伄鍖洪枔璩炪仹鏉变含銇�嗚虎鍎嫕銇珛褰硅�呫仺銇倞銇俱仐銇熴��
+銉曘儷銉炪儵銈姐兂銇儥銈广儓銈裤偆銉犮伅浜屾檪闁撳洓鍗佸垎涓夊崄鍥涚銆�
+涔呯暀绫冲競銇с伅浠婃湞涓�鏅傞枔銇節鍗佷簩鐐逛簲銉熴儶銇寷鐑堛仾闆ㄣ倰瑕虫脯涓�鏅傞枔褰撱仧銈娿伄闆ㄩ噺銇ㄣ仐銇︺伅鍗冧節鐧句竷鍗佷竷骞淬伄绲辫▓闁嬪浠ユ潵鏈�澶с仹銇�
+娌荤檪銈掑繀瑕併仺銇欍倠鍕曡剤绠￠枊瀛樼棁銇亗銈嬭丹銇°們銈撱伅銆佷竴鍗冧簲銈般儵銉犳湭婧�銇с伅绱勪笁鍗併儜銉笺偦銉炽儓銆佷竴鍗冦偘銉┿儬鏈簚銇с伅绱勪簲鍗併儜銉笺偦銉炽儓銇ㄣ仌銈屻倠銆傝柆銇ц绠°亴闁夈仒銈嬨亾銇ㄣ倐澶氥亜銇屻�佸僵钁夈仭銈冦倱銇柆銇с伅琛�绠°亴闁夈仒銇亱銇ｃ仧銆傚叚鐧惧叚鍗佸叓銈儹銉°兗銉堛儷銆�
+鐧緗鐧句簲鍗亅鐧句簩鍗佷笁|鐧惧崄涓�|涓�鐧句簩鍗佷笁|銆噟闆秥涓�|浜寍涓墊鍗亅鍗佷竴|鍗佷簩|鍗佷笁|鍗佷簲|鍗佷節|浜屽崄|浜斿崄|涔濆崄涔潀涓�鍗冧簩|涓�鍗冧簩鐧句笁鍗佸洓|鍗冨崄涓�|鍗冧節鐧句節鍗佷竷|銆囦竴浜屼笁鍥涗簲鍏竷鍏節闆秥涓�浜屼笁鍥涗簲鍏竷鍏節|鐧句簩|涓夌櫨浜屽崄鍥泑涓�鐧緗浜岀櫨|涓�鍗億浜斿崈|涓�涓噟浜斿崄涓噟涓�鐧句竾|鍥涘崈涓噟鍏剟|鍗佸剟|涔濆厗
+涓�鍗冨叚鐧句竷鍗佷節
+涓�銆囥�囧叓鍏�
+銆囧叓鍏竴涓変竷涔濆洓浜斿叚鍏�
+銈姐兗銈枫儯銉儑銈ｃ偣銆囷紝闆讹紝涓�锛屼簩锛屼笁锛岀櫨浜岋紝涓夌櫨浜屽崄鍥涳紝涓�鐧撅紝浜岀櫨锛屼竴鍗冿紝涓�鍗冧簲鐧撅紝涓�鍗冨叚鐧句竷鍗佷節锛屼簲鍗冿紝涓�涓囷紝涓�鐧句竾锛屼竴鍗冧竾
+銈裤兂銈广伀銈傞亰銇冲績涓夌櫨浜屽崄鍏亴闅犮仌銈屻仸銇勩伨銇椼仧銇�
diff --git a/fun_text_processing/inverse_text_normalization/ja/ja_unit_test.tsv b/fun_text_processing/inverse_text_normalization/ja/ja_unit_test.tsv
new file mode 100644
index 0000000..7af970f
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/ja/ja_unit_test.tsv
@@ -0,0 +1,20 @@
+ps涓夛紝ps鍥涳紝ATM锛屽崄鏁般亱浜屽崄锛屽垢鍥涢儙锛岀櫧闆Й銇ㄤ竷浜恒伄銇撱伋銇紝涓冨锛屽崈澶滀竴澶滐紝涓夌櫨鍏崄琛岋紝鍗佷簲銇湀锛屽喎鍑嶄笁瓒筹紝涓夊崄骞存渤鏉憋紝涓夊崄骞存渤瑗匡紝鍗冨勾寰岋紝浜旀湀澶╋紝瀹夊�嶆檵涓夛紝灏忔硥绱斾竴閮庯紝灞辨湰浜斿崄鍏� ps3锛宲s4锛孉TM锛屽崄鏁般亱浜屽崄锛屽垢鍥涢儙锛岀櫧闆Й銇ㄤ竷浜恒伄銇撱伋銇紝涓冨锛屽崈澶滀竴澶滐紝涓夌櫨鍏崄琛岋紝鍗佷簲銇湀锛屽喎鍑嶄笁瓒筹紝涓夊崄骞存渤鏉憋紝涓夊崄骞存渤瑗匡紝鍗冨勾寰岋紝浜旀湀澶╋紝瀹夊�嶆檵涓夛紝灏忔硥绱斾竴閮庯紝灞辨湰浜斿崄鍏�
+绗竴銆佺浜屻�佺涓夈�佺鍥涖�佺浜斻�佺鍏�佺涓冦�佺鍏�佺涔濄�佺鍗併�佺鍗佷竴銆佺鍗佷簩銆佺鍗佷笁銆佺鍗佸洓銆佺鍗佷簲銆佺鍗佸叚銆佺鍗佷竷銆佺鍗佸叓銆佺鍗佷節銆佺浜屽崄銆佷簩鍗併�佺浜屽崄鍥涖�佷簩鍗佸洓銆佺浜斿崄鍏�佺鐧俱�佺鐧句竴 绗�1銆佺2銆佺3銆佺4銆佺5銆佺6銆佺7銆佺8銆佺9銆佺10銆佺11銆佺12銆佺13銆佺14銆佺15銆佺16銆佺17銆佺18銆佺19銆佺20銆�20銆佺24銆�24銆佺56銆佺100銆佺101
+銇с�併倧銇熴仐銇叆銇ｃ仧鍘熺鏂欍亱銈夈伄浜斿垎銇竴銆佷簲鍒嗐伄涓夈倰銇傘仾銇熴伀娓°仚銇ㄣ亜銇嗐亾銇ㄣ仾銇仩銇屻�傚爡銇傘亽鎰涖伅娣便亸銆屼綋銇笁鍒嗐伄涓�銆佸洓鍒嗐伄涓�銆佷簩鍒嗐伄涓�銇爡銇傘亽銇仾銇ｃ仸銇勩伨銇欍�� 銇с�併倧銇熴仐銇叆銇ｃ仧鍘熺鏂欍亱銈夈伄1/5銈掋亗銇仧銇浮銇欍仺銇勩亞銇撱仺銇伄銇犮亴銆傚爡銇傘亽鎰涖伅娣便亸銆屼綋銇�1/3銆�1/4銆�1/2銇爡銇傘亽銇仾銇ｃ仸銇勩伨銇欍��
+闆跺叚骞村洓鏈堬綖涓�涔濆勾鍗佷簩鏈堛伀鏃ユ湰銇ф壙瑾嶃仌銈屻仧鍖荤檪姗熷櫒銇簲鐧句簩鍗佷節浠躲�併仢銇亞銇″皬鍏愮敤銇倧銇氥亱鍗佷簩浠躲�� 06骞�4鏈堬綖19骞�12鏈堛伀鏃ユ湰銇ф壙瑾嶃仌銈屻仧鍖荤檪姗熷櫒銇�529浠躲�併仢銇亞銇″皬鍏愮敤銇倧銇氥亱12浠躲��
+涓�涔濅節涓夊勾銇獣鐢熴仐銇熷悓鍟嗗搧銇仭銇伩銆佺磩涓夊崄骞村墠銆佷簩鍗佹銇爟銇垢鍥涢儙銇啓鐪熴倰鍏枊銆� 锛戯紮锛欙紦骞淬伀瑾曠敓銇椼仧鍚屽晢鍝併伀銇°仾銇裤�佺磩锛擄紣骞村墠銆侊紥锛愭銇爟銇垢鍥涢儙銇啓鐪熴倰鍏枊銆�
+濂棙銇欍倠鑷韩銇Э銈掑弾銈併仧銉夈偔銉ャ儭銉炽偪銉兗鏄犵敾銆屽崄鍎勫唵绋笺亹銆嶏紙鍗佷竴鏈堜簩鍗佹棩鏉变含銇у叕闁嬶級銈掞及锛层仚銈嬨仧銈佹澅浜兘鍐呫仹瀛︾敓鍚戙亼銈汇儫銉娿兗銈掗枊鍌�傘偛銈广儓銇�岋即锛讥銆�铏庤垶绔溿�嶃伄楂樻銈搞儳銉笺偢銇屻�佷綔瑭炰綔鏇层仐銇熴�屻儹銉笺儔銆嶃仹鍗扮◣鍗佸叚鍎勫唵銈掔銇勩仩銇ㄦ槑銇嬨仐銆佷細鍫淬伅銇┿倛銈併亜銇熴�� 濂棙銇欍倠鑷韩銇Э銈掑弾銈併仧銉夈偔銉ャ儭銉炽偪銉兗鏄犵敾銆岋紤锛愬剟鍐嗙銇愩�嶏紙锛戯紤鏈堬紥锛愭棩鏉变含銇у叕闁嬶級銈掞及锛层仚銈嬨仧銈佹澅浜兘鍐呫仹瀛︾敓鍚戙亼銈汇儫銉娿兗銈掗枊鍌�傘偛銈广儓銇�岋即锛讥銆�铏庤垶绔溿�嶃伄楂樻銈搞儳銉笺偢銇屻�佷綔瑭炰綔鏇层仐銇熴�屻儹銉笺儔銆嶃仹鍗扮◣锛戯紪鍎勫唵銈掔銇勩仩銇ㄦ槑銇嬨仐銆佷細鍫淬伅銇┿倛銈併亜銇熴��
+鐧哄２銇洸銇簩鐧句簩鍗佷竾鏋氥倰澹层倞銆侀珮姗嬨伅鍗扮◣銈掋仾銈撱仺浜屽勾銇т娇銇勫垏銇ｃ仧銇濄亞銇犮亴銆岀殕銇曘倱銇屻偒銉┿偑銈便仹涓�鍥炴瓕銇嗐仺涓冨唵鍏ャ倞銇俱仚銇椼�佷粖銈傚勾闁撲竴鍗冧簩鐧句竾鍐嗐亹銈夈亜銆侀粰銇ｃ仸銇︺倐鍏ャ仯銇︺亶銇俱仚銆嶃�備綍銇с倐銇亜銈堛亞銇亾銇ㄣ亴骞搞仜銇犮仯銇熴仺鎬濄亞銇ㄦ瓕銇ｃ仧鏇层伅銆併仺銈撱仹銈傘仾銇勫嵃绋庛倰鐢熴伩鍑恒仐銇︺亜銇︺�併儐銉兗銈傞檷鍙傘�俆ikToker銈嗐倞銇們浣撻噸涓夊崄涔濄偔銉�滃崄浜斻偔銉笡閲忊�濇灄銇裤仾銇汇偄銉娿�� 鐧哄２銇洸銇紥锛掞紣涓囨灇銈掑２銈娿�侀珮姗嬨伅鍗扮◣銈掋仾銈撱仺锛掑勾銇т娇銇勫垏銇ｃ仧銇濄亞銇犮亴銆岀殕銇曘倱銇屻偒銉┿偑銈便仹锛戝洖姝屻亞銇紬鍐嗗叆銈娿伨銇欍仐銆佷粖銈傚勾闁擄紤锛掞紣锛愪竾鍐嗐亹銈夈亜銆侀粰銇ｃ仸銇︺倐鍏ャ仯銇︺亶銇俱仚銆嶃�備綍銇с倐銇亜銈堛亞銇亾銇ㄣ亴骞搞仜銇犮仯銇熴仺鎬濄亞銇ㄦ瓕銇ｃ仧鏇层伅銆併仺銈撱仹銈傘仾銇勫嵃绋庛倰鐢熴伩鍑恒仐銇︺亜銇︺�併儐銉兗銈傞檷鍙傘�俆ikToker銈嗐倞銇們浣撻噸39kg鈥�15kg娓涢噺鈥濇灄銇裤仾銇汇偄銉娿��
+鍗佷簲鏃ャ伀绗叚鍗佸洓鍥炴棩鏈儸銈炽兗銉夊ぇ璩烇紙涓诲偓鏃ユ湰浣滄洸瀹跺崝浼氾級銇悇璩炪亴鐧鸿〃銇曘倢銇熴亴銆併亾銈屻伨銇у崄浜屽勾閫ｇ稓銇у劒绉�浣滃搧璩炪倰鍙楄碁銇椼仸銇勩仧AK鍥涘崄涓冦伅銉偣銉堛伀鍏ャ倝銇氥�佽閷层亴閫旂刀銇堛仸銇勩仧銆� 15鏃ャ伀绗�64鍥炴棩鏈儸銈炽兗銉夊ぇ璩烇紙涓诲偓鏃ユ湰浣滄洸瀹跺崝浼氾級銇悇璩炪亴鐧鸿〃銇曘倢銇熴亴銆併亾銈屻伨銇�12骞撮�ｇ稓銇у劒绉�浣滃搧璩炪倰鍙楄碁銇椼仸銇勩仧AK47銇儶銈广儓銇叆銈夈仛銆佽閷层亴閫旂刀銇堛仸銇勩仧銆�
+闁嬬エ銇岀稓銇忋偄銉°儶銈伄涓枔閬告寵銇ц鏁般伄銈€儭銉偒涓昏銉°儑銈ｃ偄銇崄涓�鏈堝崄鍏棩銆侀噹鍏氥兓鍏卞拰鍏氥亴瀹氭暟鍥涚櫨涓夊崄浜斻伄閫ｉ偊璀颁細銇笅闄€仹浜岀櫨鍗佸叓璀板腑銈掔嵅寰椼仐鍥涘勾銇躲倞銇鏁版淳銈掑オ閭勩仐銇熴仺鍫便仒銇熴�� 闁嬬エ銇岀稓銇忋偄銉°儶銈伄涓枔閬告寵銇ц鏁般伄銈€儭銉偒涓昏銉°儑銈ｃ偄銇�11鏈�16鏃ャ�侀噹鍏氥兓鍏卞拰鍏氥亴瀹氭暟435銇�ｉ偊璀颁細銇笅闄€仹218璀板腑銈掔嵅寰椼仐4骞淬伓銈娿伀澶氭暟娲俱倰濂倓銇椼仧銇ㄥ牨銇樸仧銆�
+銇撱倢銇т簩銆囦簩涓�鏈堜竴鏈堛伀璧枫亾銇ｃ仧閫ｉ偊璀颁細璀颁簨鍫傝ゲ鎾冧簨浠躲伀闁€仚銈嬩笅闄€伄鐗瑰垾瑾挎熁濮斿摗浼氥伅瑙ｆ暎銇曘仜銈夈倢銈嬨亾銇ㄣ伀銇倞銇濄亞銇犮�� 銇撱倢銇�2021鏈�1鏈堛伀璧枫亾銇ｃ仧閫ｉ偊璀颁細璀颁簨鍫傝ゲ鎾冧簨浠躲伀闁€仚銈嬩笅闄€伄鐗瑰垾瑾挎熁濮斿摗浼氥伅瑙ｆ暎銇曘仜銈夈倢銈嬨亾銇ㄣ伀銇倞銇濄亞銇犮��
+銆屾槰骞淬伅鏂拌胺銇屾湰瑾垮瓙銇樸們銇亜涓仹銆併仢銈屻仹銈傘亗銈屻仩銇戣蛋銇ｃ仸銇忋倢銇﹀姪銇嬨倞銇俱仐銇熴�嶃仺鐝惧湪銇ソ瑾裤伓銈娿亴浼濄亪銈夈倢銇︺亜銇熸柊璋烽伕鎵嬨伅鍗佷竴鏈堝崄涓夋棩銇銈忋倢銇熸澅鏃ユ湰濂冲瓙椐呬紳銇с偄銉炽偒銉笺倰鍕欍倎銆佸崄銈儹銈掍笁鍗佷竴鍒嗐�囧叓绉掋伄鍖洪枔璩炪仹鏉变含銇�嗚虎鍎嫕銇珛褰硅�呫仺銇倞銇俱仐銇熴�� 銆屾槰骞淬伅鏂拌胺銇屾湰瑾垮瓙銇樸們銇亜涓仹銆併仢銈屻仹銈傘亗銈屻仩銇戣蛋銇ｃ仸銇忋倢銇﹀姪銇嬨倞銇俱仐銇熴�嶃仺鐝惧湪銇ソ瑾裤伓銈娿亴浼濄亪銈夈倢銇︺亜銇熸柊璋烽伕鎵嬨伅11鏈�13鏃ャ伀琛屻倧銈屻仧鏉辨棩鏈コ瀛愰浼濄仹銈€兂銈兗銈掑嫏銈併��10銈儹銈�31鍒�08绉掋伄鍖洪枔璩炪仹鏉变含銇�嗚虎鍎嫕銇珛褰硅�呫仺銇倞銇俱仐銇熴��
+銉曘儷銉炪儵銈姐兂銇儥銈广儓銈裤偆銉犮伅浜屾檪闁撳洓鍗佸垎涓夊崄鍥涚銆� 銉曘儷銉炪儵銈姐兂銇儥銈广儓銈裤偆銉犮伅2鏅傞枔40鍒�34绉掋��
+涔呯暀绫冲競銇с伅浠婃湞涓�鏅傞枔銇節鍗佷簩鐐逛簲銉熴儶銇寷鐑堛仾闆ㄣ倰瑕虫脯涓�鏅傞枔褰撱仧銈娿伄闆ㄩ噺銇ㄣ仐銇︺伅鍗冧節鐧句竷鍗佷竷骞淬伄绲辫▓闁嬪浠ユ潵鏈�澶с仹銇� 涔呯暀绫冲競銇с伅浠婃湞涓�鏅傞枔銇�92鐐�5mm銇寷鐑堛仾闆ㄣ倰瑕虫脯涓�鏅傞枔褰撱仧銈娿伄闆ㄩ噺銇ㄣ仐銇︺伅1977骞淬伄绲辫▓闁嬪浠ユ潵鏈�澶с仹銇�
+娌荤檪銈掑繀瑕併仺銇欍倠鍕曡剤绠￠枊瀛樼棁銇亗銈嬭丹銇°們銈撱伅銆佷竴鍗冧簲銈般儵銉犳湭婧�銇с伅绱勪笁鍗併儜銉笺偦銉炽儓銆佷竴鍗冦偘銉┿儬鏈簚銇с伅绱勪簲鍗併儜銉笺偦銉炽儓銇ㄣ仌銈屻倠銆傝柆銇ц绠°亴闁夈仒銈嬨亾銇ㄣ倐澶氥亜銇屻�佸僵钁夈仭銈冦倱銇柆銇с伅琛�绠°亴闁夈仒銇亱銇ｃ仧銆傚叚鐧惧叚鍗佸叓銈儹銉°兗銉堛儷銆� 娌荤檪銈掑繀瑕併仺銇欍倠鍕曡剤绠￠枊瀛樼棁銇亗銈嬭丹銇°們銈撱伅銆�1500銈般儵銉犳湭婧�銇с伅绱�30%銆�1000銈般儵銉犳湭婧�銇с伅绱�50%銇ㄣ仌銈屻倠銆傝柆銇ц绠°亴闁夈仒銈嬨亾銇ㄣ倐澶氥亜銇屻�佸僵钁夈仭銈冦倱銇柆銇с伅琛�绠°亴闁夈仒銇亱銇ｃ仧銆�668km銆�
+鐧緗鐧句簲鍗亅鐧句簩鍗佷笁|鐧惧崄涓�|涓�鐧句簩鍗佷笁|銆噟闆秥涓�|浜寍涓墊鍗亅鍗佷竴|鍗佷簩|鍗佷笁|鍗佷簲|鍗佷節|浜屽崄|浜斿崄|涔濆崄涔潀涓�鍗冧簩|涓�鍗冧簩鐧句笁鍗佸洓|鍗冨崄涓�|鍗冧節鐧句節鍗佷竷|銆囦竴浜屼笁鍥涗簲鍏竷鍏節闆秥涓�浜屼笁鍥涗簲鍏竷鍏節|鐧句簩|涓夌櫨浜屽崄鍥泑涓�鐧緗浜岀櫨|涓�鍗億浜斿崈|涓�涓噟浜斿崄涓噟涓�鐧句竾|鍥涘崈涓噟鍏剟|鍗佸剟|涔濆厗 100|150|123|111|123|0|0|1|2|3|10|11|12|13|15|19|20|50|99|1200|1234|1011|1997|01234567890|123456789|102|324|100|200|1000|5000|1涓噟50涓噟1000000|4000涓噟6鍎剕10鍎剕9鍏�
+涓�鍗冨叚鐧句竷鍗佷節 1679
+涓�銆囥�囧叓鍏� 10086
+銆囧叓鍏竴涓変竷涔濆洓浜斿叚鍏� 08613794568
+銈姐兗銈枫儯銉儑銈ｃ偣銆囷紝闆讹紝涓�锛屼簩锛屼笁锛岀櫨浜岋紝涓夌櫨浜屽崄鍥涳紝涓�鐧撅紝浜岀櫨锛屼竴鍗冿紝涓�鍗冧簲鐧撅紝涓�鍗冨叚鐧句竷鍗佷節锛屼簲鍗冿紝涓�涓囷紝涓�鐧句竾锛屼竴鍗冧竾 銈姐兗銈枫儯銉儑銈ｃ偣0锛�0锛�1锛�2锛�3锛�120锛�324锛�100锛�200锛�1000锛�1500锛�1679锛�5000锛�10000锛�1000000锛�10000000
+銈裤兂銈广伀銈傞亰銇冲績涓夌櫨浜屽崄鍏亴闅犮仌銈屻仸銇勩伨銇椼仧銇� 銈裤兂銈广伀銈傞亰銇冲績326銇岄殸銇曘倢銇︺亜銇俱仐銇熴亴
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py
index 72ffb7d..2b2af28 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py
@@ -1,16 +1,5 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from pynini import accep, cross, string_file, union
@@ -188,4 +177,4 @@
)
self.graph_hundred_component_at_least_one_none_zero_digit = (
graph_hundred_component_at_least_one_none_zero_digit
- )
+ )
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/date.py b/fun_text_processing/inverse_text_normalization/ja/taggers/date.py
index 9362824..2c2db40 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/date.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py
index ed9dcc7..5ae9e8b 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py
index 13f59fd..88c7d9c 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_ALPHA, GraphFst, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py
index 6a42fff..2e66a96 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst, delete_extra_space, delete_space, insert_space, DAMO_CHAR
from pynini.lib import pynutil
@@ -52,4 +40,4 @@
graph = optional_graph_negative + graph
final_graph = self.add_tokens(graph)
- self.fst = final_graph.optimize()
+ self.fst = final_graph.optimize()
\ No newline at end of file
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/measure.py b/fun_text_processing/inverse_text_normalization/ja/taggers/measure.py
index 4dde427..a81204a 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/measure.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/money.py b/fun_text_processing/inverse_text_normalization/ja/taggers/money.py
index a374cfe..c235153 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/money.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py
index 3494ba6..3dd6fcb 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from pynini import cross
from pynini.lib.pynutil import delete, insert, add_weight
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py b/fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py
index d408e4f..b101500 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_SIGMA, GraphFst
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py
index 8c119a9..4a9a03d 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py
index d256891..51174bc 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/time.py b/fun_text_processing/inverse_text_normalization/ja/taggers/time.py
index f28cf9f..883a39b 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/time.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.taggers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path, num_to_word
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py
index b17f58a..68ab211 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py
index eee1c20..9ef4019 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/taggers/word.py b/fun_text_processing/inverse_text_normalization/ja/taggers/word.py
index c923a95..b5879d7 100644
--- a/fun_text_processing/inverse_text_normalization/ja/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/ja/taggers/word.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_SPACE, GraphFst, DAMO_CHAR
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ja/utils.py b/fun_text_processing/inverse_text_normalization/ja/utils.py
index 782688c..9b97867 100644
--- a/fun_text_processing/inverse_text_normalization/ja/utils.py
+++ b/fun_text_processing/inverse_text_normalization/ja/utils.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
from typing import Union
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py
index d7e97e2..03f6915 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py
@@ -1,15 +1,5 @@
+
# -*- coding: utf-8 -*-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, DAMO_CHAR
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py
index ddd628e..d017ae4 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py
index 64157e3..81ed2b4 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py
index 8f8121f..faa2ec8 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py
index 6ed6686..503f64f 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, DAMO_SIGMA, delete_extra_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py
index 05ad326..bb0dee2 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py
index 61c22c3..510e8d6 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py
index 4b14b4a..6d665c7 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py
index 458a16e..0c727bb 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py
index 8bff580..5fd56aa 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py
index bc107c7..1f2d071 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.ja.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.ja.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py
index d5d9f34..c18c6b0 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py
index 5e3416d..abe1c3a 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py
index 3a99b21..47c278f 100644
--- a/fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/__init__.py b/fun_text_processing/inverse_text_normalization/ko/__init__.py
index d9bfbf6..888da72 100755
--- a/fun_text_processing/inverse_text_normalization/ko/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/ko/__init__.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py b/fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py
index b72e136..f59f8d0 100755
--- a/fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py
+++ b/fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from argparse import ArgumentParser
from typing import List
diff --git a/fun_text_processing/inverse_text_normalization/ko/graph_utils.py b/fun_text_processing/inverse_text_normalization/ko/graph_utils.py
index 8b85e2c..c9f737d 100755
--- a/fun_text_processing/inverse_text_normalization/ko/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/ko/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py
index b788bfd..0a54905 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/date.py b/fun_text_processing/inverse_text_normalization/ko/taggers/date.py
index 24ce259..8d31fc7 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py
index b3f39b8..3c30710 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py
index e3bcbff..8ab0f78 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py
index 1dc0b75..e841f88 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
DAMO_NOT_QUOTE,
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/measure.py b/fun_text_processing/inverse_text_normalization/ko/taggers/measure.py
index e2171bc..71645d3 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/money.py b/fun_text_processing/inverse_text_normalization/ko/taggers/money.py
index e8e1582..4e41b94 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py
index e28ab72..0aac488 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py
index 0253998..17473a7 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/time.py b/fun_text_processing/inverse_text_normalization/ko/taggers/time.py
index 9ea76fe..24c82bc 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py
index fe22f1f..b9a2502 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py
index 8189059..928a7f5 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/ko/taggers/word.py b/fun_text_processing/inverse_text_normalization/ko/taggers/word.py
index 411bb2b..04aa2bc 100755
--- a/fun_text_processing/inverse_text_normalization/ko/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/ko/taggers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/utils.py b/fun_text_processing/inverse_text_normalization/ko/utils.py
index 5048869..75d66e6 100755
--- a/fun_text_processing/inverse_text_normalization/ko/utils.py
+++ b/fun_text_processing/inverse_text_normalization/ko/utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import csv
import os
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py
index de3c81b..8ffcf43 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py
index 92ec466..56358b8 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py
index 67065d7..6e089b3 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py
index b2a90a2..e217827 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py
index 48a3583..4acf7ef 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
DAMO_NOT_QUOTE,
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py
index 4779a5e..56067bd 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py
index 21d8131..5988283 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py
index a365b95..98b0e11 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py
index 1b5ca58..ac90a7f 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py
index 0e6904b..9897c7e 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py
index d62f101..5085a81 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.ko.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py
index 263025e..dfd0b39 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py
index 676a62d..60b755a 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py
index 50d3ca9..1d01e84 100755
--- a/fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/pt/__init__.py b/fun_text_processing/inverse_text_normalization/pt/__init__.py
index 24cfe3b..498c4e2 100644
--- a/fun_text_processing/inverse_text_normalization/pt/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/pt/__init__.py
@@ -1,17 +1,5 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/pt/taggers/cardinal.py
index 811f4ce..465cc44 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.pt.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/date.py b/fun_text_processing/inverse_text_normalization/pt/taggers/date.py
index b4625f3..4c9eb01 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/pt/taggers/decimal.py
index b1a8ea8..a3c2f5d 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/pt/taggers/electronic.py
index f1ad4fd..1aee1be 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/measure.py b/fun_text_processing/inverse_text_normalization/pt/taggers/measure.py
index e88a231..1859cd9 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/money.py b/fun_text_processing/inverse_text_normalization/pt/taggers/money.py
index 090cfad..a163e83 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/pt/taggers/ordinal.py
index 13835aa..a2f30b5 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/pt/taggers/punctuation.py
index b4cd2de..ef63904 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/punctuation.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/pt/taggers/telephone.py
index 1730d64..934363e 100755
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/time.py b/fun_text_processing/inverse_text_normalization/pt/taggers/time.py
index cecdcf8..826c819 100755
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py
index 3d7cb1d..7ad1322 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/pt/taggers/whitelist.py
index 23ea39b..9aa2ffc 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/taggers/word.py b/fun_text_processing/inverse_text_normalization/pt/taggers/word.py
index 763834c..c6bae25 100644
--- a/fun_text_processing/inverse_text_normalization/pt/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/pt/taggers/word.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/utils.py b/fun_text_processing/inverse_text_normalization/pt/utils.py
index caca48b..ed78496 100644
--- a/fun_text_processing/inverse_text_normalization/pt/utils.py
+++ b/fun_text_processing/inverse_text_normalization/pt/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py
index a11b121..1befda1 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/date.py
index 8f25753..759c434 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py
index 5cfb561..f8366f1 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py
index 29f5c65..e4cb1a7 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/measure.py
index e4cf7ce..c0b83dd 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/money.py
index f99e216..9cde5f4 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py
index d532cf9..0db1541 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py
index b7f621c..ef2ba51 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/time.py
index f5a8de0..9e3597e 100755
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py
index 44e6e43..fbc0873 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.pt.verbalizers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py
index 024c182..34f0d69 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py
index 0eb358f..4a25f67 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/pt/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/pt/verbalizers/word.py
index 05b67c6..590dc65 100644
--- a/fun_text_processing/inverse_text_normalization/pt/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/pt/verbalizers/word.py
@@ -1,16 +1,5 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/ru/taggers/cardinal.py
index 97a1929..029997b 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/date.py b/fun_text_processing/inverse_text_normalization/ru/taggers/date.py
index bbbd2c2..57b2d17 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/decimals.py b/fun_text_processing/inverse_text_normalization/ru/taggers/decimals.py
index 72504c1..22a2b89 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/decimals.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/decimals.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SPACE, GraphFst, delete_extra_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/ru/taggers/electronic.py
index 9fde529..c59f225 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/measure.py b/fun_text_processing/inverse_text_normalization/ru/taggers/measure.py
index 154f458..920316b 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/money.py b/fun_text_processing/inverse_text_normalization/ru/taggers/money.py
index 5ab47f6..89a0b96 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/ru/taggers/ordinal.py
index 74398db..b3b1a78 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/ru/taggers/telephone.py
index 1c6ee85..aa7370a 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/time.py b/fun_text_processing/inverse_text_normalization/ru/taggers/time.py
index 256d0f5..b725a6e 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py
index 6f3aba9..ee5128c 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/ru/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/ru/taggers/whitelist.py
index 67d0f6a..601e929 100644
--- a/fun_text_processing/inverse_text_normalization/ru/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/ru/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py
index 6afdce9..ab5afee 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/date.py
index 0e1b755..aa74b77 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py
index e49a38f..da1928d 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, DAMO_SPACE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py
index d637727..378212b 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/measure.py
index c2d98d5..0a119bb 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/money.py
index eb68b87..531dac7 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py
index 3b7ace6..7e175d3 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py
index 244908a..5a0f3c7 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/time.py
index 2565d7b..abc6ccc 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize.py
index f897ffd..6683fbf 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.en.verbalizers.whitelist import WhiteListFst
from fun_text_processing.inverse_text_normalization.ru.verbalizers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py
index d8f6679..5ea1ae8 100644
--- a/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst
diff --git a/fun_text_processing/inverse_text_normalization/run_evaluate.py b/fun_text_processing/inverse_text_normalization/run_evaluate.py
index 9723c9e..7fb4a55 100644
--- a/fun_text_processing/inverse_text_normalization/run_evaluate.py
+++ b/fun_text_processing/inverse_text_normalization/run_evaluate.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
from argparse import ArgumentParser
from fun_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
diff --git a/fun_text_processing/inverse_text_normalization/tl/__init__.py b/fun_text_processing/inverse_text_normalization/tl/__init__.py
index 0edfc8a..3622fb2 100755
--- a/fun_text_processing/inverse_text_normalization/tl/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/tl/__init__.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.tl.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.tl.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/clean_eval_data.py b/fun_text_processing/inverse_text_normalization/tl/clean_eval_data.py
index b72e136..f59f8d0 100755
--- a/fun_text_processing/inverse_text_normalization/tl/clean_eval_data.py
+++ b/fun_text_processing/inverse_text_normalization/tl/clean_eval_data.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from argparse import ArgumentParser
from typing import List
diff --git a/fun_text_processing/inverse_text_normalization/tl/graph_utils.py b/fun_text_processing/inverse_text_normalization/tl/graph_utils.py
index e3ccd31..833776d 100755
--- a/fun_text_processing/inverse_text_normalization/tl/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/tl/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/tl/taggers/cardinal.py
index 491b221..1bbeb9f 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/cardinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path, num_to_word
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/date.py b/fun_text_processing/inverse_text_normalization/tl/taggers/date.py
index cb07c06..4f01bc3 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/tl/taggers/decimal.py
index 4a5f791..a0771fa 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/decimal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/tl/taggers/electronic.py
index a137113..5c9d804 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/tl/taggers/fraction.py
index ba5f18c..3330c87 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.tl.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/measure.py b/fun_text_processing/inverse_text_normalization/tl/taggers/measure.py
index 6d29d6b..ad5572b 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/money.py b/fun_text_processing/inverse_text_normalization/tl/taggers/money.py
index 10b6574..aca058c 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/tl/taggers/ordinal.py
index f7620f7..599a023 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/ordinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/tl/taggers/punctuation.py
index 77ca5d9..fc48a66 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/punctuation.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/tl/taggers/telephone.py
index f50a4ce..790e396 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/time.py b/fun_text_processing/inverse_text_normalization/tl/taggers/time.py
index 81c4815..232c138 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.taggers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/tl/taggers/tokenize_and_classify.py
index a2a3557..a5c67dd 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/tokenize_and_classify.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/tl/taggers/whitelist.py
index ca54fad..0c725d6 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/whitelist.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/tl/taggers/word.py b/fun_text_processing/inverse_text_normalization/tl/taggers/word.py
index 23bf20f..12d2cba 100755
--- a/fun_text_processing/inverse_text_normalization/tl/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/tl/taggers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/utils.py b/fun_text_processing/inverse_text_normalization/tl/utils.py
index 782688c..042d6e0 100755
--- a/fun_text_processing/inverse_text_normalization/tl/utils.py
+++ b/fun_text_processing/inverse_text_normalization/tl/utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
from typing import Union
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/cardinal.py
index 446f571..2643007 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/cardinal.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/date.py
index 3999b07..4869637 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/decimal.py
index abcb1f4..7d0ffa7 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/decimal.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/electronic.py
index e03d547..63bd0e1 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/fraction.py
index 2f92e5a..613de0f 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.tl.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/measure.py
index 3ed5fd9..10e688c 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/money.py
index 4f870c2..cc76b83 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/ordinal.py
index 32747a1..5649d05 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/ordinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/telephone.py
index f474aad..c59f916 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/time.py
index 651d349..10907de 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize.py
index de3feae..17ff479 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.tl.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.tl.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize_final.py
index 6fd8042..bb3eaf9 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/verbalize_final.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/whitelist.py
index a2482e5..137529a 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/whitelist.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/tl/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/tl/verbalizers/word.py
index 54d8a15..2f43e38 100755
--- a/fun_text_processing/inverse_text_normalization/tl/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/tl/verbalizers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.tl.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/__init__.py b/fun_text_processing/inverse_text_normalization/vi/__init__.py
index f2f04de..f84f840 100644
--- a/fun_text_processing/inverse_text_normalization/vi/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/vi/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/vi/graph_utils.py b/fun_text_processing/inverse_text_normalization/vi/graph_utils.py
index 3a20bfd..aac8289 100644
--- a/fun_text_processing/inverse_text_normalization/vi/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/vi/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/vi/taggers/cardinal.py
index 1d887bf..e168f3f 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/date.py b/fun_text_processing/inverse_text_normalization/vi/taggers/date.py
index de51ab7..e0783c8 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/vi/taggers/decimal.py
index f47f55a..1b2098d 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/vi/taggers/electronic.py
index d29cccd..00f0e18 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_ALPHA, GraphFst, insert_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/vi/taggers/fraction.py
index 757d56b..60c4737 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/measure.py b/fun_text_processing/inverse_text_normalization/vi/taggers/measure.py
index d61670b..b711b5f 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/money.py b/fun_text_processing/inverse_text_normalization/vi/taggers/money.py
index 95976d9..5f25f54 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/ordinal.py b/fun_text_processing/inverse_text_normalization/vi/taggers/ordinal.py
index a065b15..e1e27ec 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/vi/taggers/punctuation.py
index 33ce1a0..b05da58 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/punctuation.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/vi/taggers/telephone.py
index 21ea392..7c738a4 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/time.py b/fun_text_processing/inverse_text_normalization/vi/taggers/time.py
index 343539e..6bf7579 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py
index 8a8795b..82beedd 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/vi/taggers/whitelist.py
index 8dd70b2..2e39612 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/taggers/word.py b/fun_text_processing/inverse_text_normalization/vi/taggers/word.py
index b8130aa..5762cfc 100644
--- a/fun_text_processing/inverse_text_normalization/vi/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/vi/taggers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/vi/utils.py b/fun_text_processing/inverse_text_normalization/vi/utils.py
index 21320b7..64bf5c6 100644
--- a/fun_text_processing/inverse_text_normalization/vi/utils.py
+++ b/fun_text_processing/inverse_text_normalization/vi/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py
index dc50785..f346944 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/date.py
index cd74e52..767df49 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py
index 3b5e18f..330e6d8 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py
index 18a0251..38a82a3 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py
index 6b8d0b6..6bd2d75 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/measure.py
index 3011e81..535a143 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/money.py
index b1991f8..4e3175d 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py
index c84385a..79a4325 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py
index 194bc1c..43fc3a8 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/time.py
index 277273d..0f91873 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize.py
index 4c20808..9b592ac 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst
from fun_text_processing.inverse_text_normalization.vi.verbalizers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py
index 288eae9..00928cf 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py
index 76214b9..0a08a7b 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/vi/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/vi/verbalizers/word.py
index 03d0ce4..095d4af 100644
--- a/fun_text_processing/inverse_text_normalization/vi/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/vi/verbalizers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/zh/__init__.py b/fun_text_processing/inverse_text_normalization/zh/__init__.py
index 59ca10b..6b2a555 100755
--- a/fun_text_processing/inverse_text_normalization/zh/__init__.py
+++ b/fun_text_processing/inverse_text_normalization/zh/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/inverse_text_normalization/zh/graph_utils.py b/fun_text_processing/inverse_text_normalization/zh/graph_utils.py
index 9fabd02..5349863 100755
--- a/fun_text_processing/inverse_text_normalization/zh/graph_utils.py
+++ b/fun_text_processing/inverse_text_normalization/zh/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/.cardinal.py.swp b/fun_text_processing/inverse_text_normalization/zh/taggers/.cardinal.py.swp
new file mode 100644
index 0000000..f9e442b
--- /dev/null
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/.cardinal.py.swp
Binary files differ
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/cardinal.py b/fun_text_processing/inverse_text_normalization/zh/taggers/cardinal.py
index e2fa14f..61db01b 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/cardinal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/date.py b/fun_text_processing/inverse_text_normalization/zh/taggers/date.py
index 1748be1..6aec084 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/date.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/decimal.py b/fun_text_processing/inverse_text_normalization/zh/taggers/decimal.py
index 00fbd9c..1016035 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/decimal.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/electronic.py b/fun_text_processing/inverse_text_normalization/zh/taggers/electronic.py
index 279b589..573194f 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/fraction.py b/fun_text_processing/inverse_text_normalization/zh/taggers/fraction.py
index ac46bed..e64f4f4 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
DAMO_NOT_QUOTE,
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/measure.py b/fun_text_processing/inverse_text_normalization/zh/taggers/measure.py
index d64ff3b..311a5d2 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/money.py b/fun_text_processing/inverse_text_normalization/zh/taggers/money.py
index 8c92d48..53dc5e2 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/money.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/punctuation.py b/fun_text_processing/inverse_text_normalization/zh/taggers/punctuation.py
index fd56019..3df5e38 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/punctuation.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/punctuation.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/telephone.py b/fun_text_processing/inverse_text_normalization/zh/taggers/telephone.py
index 193f9fe..8f15ebb 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/time.py b/fun_text_processing/inverse_text_normalization/zh/taggers/time.py
index 0cf6232..fc2a37c 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/time.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py b/fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py
index 4ceb806..b4c8629 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py
@@ -1,15 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
import pynini
from fun_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst
@@ -28,7 +16,6 @@
GraphFst,
delete_extra_space,
delete_space,
- insert_space,
generator_main,
)
from pynini.lib import pynutil
@@ -95,10 +82,10 @@
punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }")
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
token_plus_punct = (
- pynini.closure(punct + insert_space) + token + pynini.closure(insert_space + punct)
+ pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct)
)
- graph = token_plus_punct + pynini.closure(insert_space + token_plus_punct)
+ graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
graph = delete_space + graph + delete_space
self.fst = graph.optimize()
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/whitelist.py b/fun_text_processing/inverse_text_normalization/zh/taggers/whitelist.py
index 545f60b..3287360 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/whitelist.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
diff --git a/fun_text_processing/inverse_text_normalization/zh/taggers/word.py b/fun_text_processing/inverse_text_normalization/zh/taggers/word.py
index 51a83db..59d77bb 100755
--- a/fun_text_processing/inverse_text_normalization/zh/taggers/word.py
+++ b/fun_text_processing/inverse_text_normalization/zh/taggers/word.py
@@ -1,17 +1,6 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
-from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, DAMO_CHAR, GraphFst
+from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, GraphFst
from pynini.lib import pynutil
@@ -23,5 +12,5 @@
def __init__(self):
super().__init__(name="word", kind="classify")
- word = pynutil.insert("name: \"") + DAMO_NOT_SPACE + pynutil.insert("\"")
+ word = pynutil.insert("name: \"") + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert("\"")
self.fst = word.optimize()
diff --git a/fun_text_processing/inverse_text_normalization/zh/utils.py b/fun_text_processing/inverse_text_normalization/zh/utils.py
index 81fab8c..281e632 100755
--- a/fun_text_processing/inverse_text_normalization/zh/utils.py
+++ b/fun_text_processing/inverse_text_normalization/zh/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import csv
import os
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py
index ca5dde2..19a8c56 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_SPACE, DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/date.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/date.py
index 04b05e0..f95f920 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/date.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/date.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py
index 9cbb9e8..7aa063e 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/electronic.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/electronic.py
index 3414036..56881e7 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/electronic.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/electronic.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py
index fa6f411..003bb0a 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
DAMO_NOT_QUOTE,
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/measure.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/measure.py
index ee82e26..ab4188d 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/measure.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/measure.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/money.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/money.py
index fbc6a93..5aaac24 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/money.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/money.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_CHAR, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/telephone.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/telephone.py
index 8300b9c..a4e408e 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/telephone.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/telephone.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/time.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/time.py
index 566fa5b..60a0d5e 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/time.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/time.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py
index 2f3d5b4..63a57a3 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.inverse_text_normalization.zh.verbalizers.cardinal import CardinalFst
from fun_text_processing.inverse_text_normalization.zh.verbalizers.date import DateFst
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py
index 25133e9..62badfc 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst
@@ -37,5 +26,5 @@
+ delete_space
+ pynutil.delete("}")
)
- graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space
+ graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
self.fst = graph
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py
index 4b9df58..42fca61 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py
@@ -1,14 +1,4 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/inverse_text_normalization/zh/verbalizers/word.py b/fun_text_processing/inverse_text_normalization/zh/verbalizers/word.py
index 8f9b0f9..93c30f3 100755
--- a/fun_text_processing/inverse_text_normalization/zh/verbalizers/word.py
+++ b/fun_text_processing/inverse_text_normalization/zh/verbalizers/word.py
@@ -1,14 +1,3 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/num2words/.coveragerc b/fun_text_processing/num2words/.coveragerc
new file mode 100644
index 0000000..6d88b72
--- /dev/null
+++ b/fun_text_processing/num2words/.coveragerc
@@ -0,0 +1,5 @@
+[run]
+branch = true
+source =
+ num2words
+ tests
diff --git a/fun_text_processing/num2words/.gitignore b/fun_text_processing/num2words/.gitignore
new file mode 100644
index 0000000..e5a9b78
--- /dev/null
+++ b/fun_text_processing/num2words/.gitignore
@@ -0,0 +1,7 @@
+*.pyc
+build
+dist
+.idea/
+*.egg-info
+/.tox
+.eggs/
diff --git a/fun_text_processing/num2words/.travis.yml b/fun_text_processing/num2words/.travis.yml
new file mode 100644
index 0000000..aa600d0
--- /dev/null
+++ b/fun_text_processing/num2words/.travis.yml
@@ -0,0 +1,20 @@
+sudo: false
+language: python
+python:
+ - "2.7"
+ - "3.4"
+ - "3.5"
+ - "3.6"
+matrix:
+ include:
+ - { python: 3.6, env: TOXENV=flake8 }
+ - { python: 3.6, env: TOXENV=isort }
+ # Py37 requires xenial distrubution and sudo
+ # See travis-ci/travis-ci#9069
+ - { python: 3.7, dist: xenial, sudo: true }
+
+install:
+ - pip install tox-travis
+ - pip install coveralls
+script: tox
+after_success: if [ -e .coverage ]; then coveralls; fi
diff --git a/fun_text_processing/num2words/MANIFEST.in b/fun_text_processing/num2words/MANIFEST.in
new file mode 100644
index 0000000..2d05b8c
--- /dev/null
+++ b/fun_text_processing/num2words/MANIFEST.in
@@ -0,0 +1 @@
+include bin/num2words
diff --git a/fun_text_processing/num2words/docker-compose.yml b/fun_text_processing/num2words/docker-compose.yml
new file mode 100644
index 0000000..d2b033f
--- /dev/null
+++ b/fun_text_processing/num2words/docker-compose.yml
@@ -0,0 +1,7 @@
+version: '3.0'
+services:
+ web:
+ image: python:3-alpine
+ command: python3 -m http.server 8080
+ volumes:
+ - .:/num2words
diff --git a/fun_text_processing/num2words/num2words/__init__.py b/fun_text_processing/num2words/num2words/__init__.py
new file mode 100644
index 0000000..3d0192b
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/__init__.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from . import (lang_AR, lang_AZ, lang_BN, lang_BG, lang_CA, lang_CZ, lang_DE, lang_DK,
+ lang_EN, lang_EL, lang_EN_IN,
+ lang_ES, lang_ES_CO, lang_ES_NI, lang_ES_VE, lang_FI, lang_FR,
+ lang_FR_BE, lang_FR_CH, lang_FR_DZ, lang_HE, lang_HU, lang_ID,
+ lang_IT, lang_IW, lang_JA, lang_KH, lang_KN, lang_KO, lang_KZ, lang_LT,
+ lang_LO, lang_LV, lang_MN, lang_MY,
+ lang_NE, lang_NL, lang_NO, lang_PL, lang_PT, lang_PT_BR, lang_RO,
+ lang_RU, lang_SI, lang_SL, lang_SR, lang_SV, lang_SW, lang_TE, lang_TH, lang_TR,
+ lang_UK, lang_UR, lang_VI)
+
+CONVERTER_CLASSES = {
+ 'ar': lang_AR.Num2Word_AR(),
+ 'az': lang_AZ.Num2Word_AZ(),
+ 'bn': lang_BN.Num2Word_BN(),
+ 'bg': lang_BG.Num2Word_BG(),
+ 'ca': lang_CA.Num2Word_CA(),
+ 'cz': lang_CZ.Num2Word_CZ(),
+ 'en': lang_EN.Num2Word_EN(),
+ 'el': lang_EL.Num2Word_EL(),
+ 'en_IN': lang_EN_IN.Num2Word_EN_IN(),
+ 'fr': lang_FR.Num2Word_FR(),
+ 'fr_CH': lang_FR_CH.Num2Word_FR_CH(),
+ 'fr_BE': lang_FR_BE.Num2Word_FR_BE(),
+ 'fr_DZ': lang_FR_DZ.Num2Word_FR_DZ(),
+ 'de': lang_DE.Num2Word_DE(),
+ 'fi': lang_FI.Num2Word_FI(),
+ 'es': lang_ES.Num2Word_ES(),
+ 'es_CO': lang_ES_CO.Num2Word_ES_CO(),
+ 'es_NI': lang_ES_NI.Num2Word_ES_NI(),
+ 'es_VE': lang_ES_VE.Num2Word_ES_VE(),
+ 'id': lang_ID.Num2Word_ID(),
+ 'iw': lang_IW.Num2Word_IW(),
+ 'ja': lang_JA.Num2Word_JA(),
+ 'kh': lang_KH.Num2Word_KH(),
+ 'kn': lang_KN.Num2Word_KN(),
+ 'ko': lang_KO.Num2Word_KO(),
+ 'kz': lang_KZ.Num2Word_KZ(),
+ 'lo': lang_LO.Num2Word_LO(),
+ 'lt': lang_LT.Num2Word_LT(),
+ 'lv': lang_LV.Num2Word_LV(),
+ 'mn': lang_MN.Num2Word_MN(),
+ 'my': lang_MY.Num2Word_MY(),
+ 'pl': lang_PL.Num2Word_PL(),
+ 'ro': lang_RO.Num2Word_RO(),
+ 'ru': lang_RU.Num2Word_RU(),
+ 'si': lang_SI.Num2Word_SI(),
+ 'sl': lang_SL.Num2Word_SL(),
+ 'sr': lang_SR.Num2Word_SR(),
+ 'sv': lang_SV.Num2Word_SV(),
+ 'sw': lang_SW.Num2Word_SW(),
+ 'no': lang_NO.Num2Word_NO(),
+ 'dk': lang_DK.Num2Word_DK(),
+ 'pt': lang_PT.Num2Word_PT(),
+ 'pt_BR': lang_PT_BR.Num2Word_PT_BR(),
+ 'he': lang_HE.Num2Word_HE(),
+ 'it': lang_IT.Num2Word_IT(),
+ 'vi': lang_VI.Num2Word_VI(),
+ 'th': lang_TH.Num2Word_TH(),
+ 'tr': lang_TR.Num2Word_TR(),
+ 'ne': lang_NE.Num2Word_NE(),
+ 'nl': lang_NL.Num2Word_NL(),
+ 'uk': lang_UK.Num2Word_UK(),
+ 'ur': lang_UR.Num2Word_UR(),
+ 'te': lang_TE.Num2Word_TE(),
+ 'hu': lang_HU.Num2Word_HU()
+}
+
+CONVERTES_TYPES = ['cardinal', 'ordinal', 'ordinal_num', 'year', 'currency']
+
+'ar': lang_AR.Num2Word_AR(),
+'az': lang_AZ.Num2Word_AZ(),
+'bn': lang_BN.Num2Word_BN(),
+'bg': lang_BG.Num2Word_BG(),
+'ca': lang_CA.Num2Word_CA(),
+'cz': lang_CZ.Num2Word_CZ(),
+'en': lang_EN.Num2Word_EN(),
+'el': lang_EL.Num2Word_EL(),
+'en_IN': lang_EN_IN.Num2Word_EN_IN(),
+'fr': lang_FR.Num2Word_FR(),
+'fr_CH': lang_FR_CH.Num2Word_FR_CH(),
+'fr_BE': lang_FR_BE.Num2Word_FR_BE(),
+'fr_DZ': lang_FR_DZ.Num2Word_FR_DZ(),
+'de': lang_DE.Num2Word_DE(),
+'fi': lang_FI.Num2Word_FI(),
+'es': lang_ES.Num2Word_ES(),
+'es_CO': lang_ES_CO.Num2Word_ES_CO(),
+'es_NI': lang_ES_NI.Num2Word_ES_NI(),
+'es_VE': lang_ES_VE.Num2Word_ES_VE(),
+'id': lang_ID.Num2Word_ID(),
+'iw': lang_IW.Num2Word_IW(),
+'ja': lang_JA.Num2Word_JA(),
+'kh': lang_KH.Num2Word_KH(),
+'kn': lang_KN.Num2Word_KN(),
+'ko': lang_KO.Num2Word_KO(),
+'kz': lang_KZ.Num2Word_KZ(),
+'lo': lang_LO.Num2Word_LO(),
+'lt': lang_LT.Num2Word_LT(),
+'lv': lang_LV.Num2Word_LV(),
+'mn': lang_MN.Num2Word_MN(),
+'my': lang_MY.Num2Word_MY(),
+'pl': lang_PL.Num2Word_PL(),
+'ro': lang_RO.Num2Word_RO(),
+'ru': lang_RU.Num2Word_RU(),
+'si': lang_SI.Num2Word_SI(),
+'sl': lang_SL.Num2Word_SL(),
+'sr': lang_SR.Num2Word_SR(),
+'sv': lang_SV.Num2Word_SV(),
+'sw': lang_SW.Num2Word_SW(),
+'no': lang_NO.Num2Word_NO(),
+'dk': lang_DK.Num2Word_DK(),
+'pt': lang_PT.Num2Word_PT(),
+'pt_BR': lang_PT_BR.Num2Word_PT_BR(),
+'he': lang_HE.Num2Word_HE(),
+'it': lang_IT.Num2Word_IT(),
+'vi': lang_VI.Num2Word_VI(),
+'th': lang_TH.Num2Word_TH(),
+'tr': lang_TR.Num2Word_TR(),
+'ne': lang_NE.Num2Word_NE(),
+'nl': lang_NL.Num2Word_NL(),
+'uk': lang_UK.Num2Word_UK(),
+'ur': lang_UR.Num2Word_UR(),
+'te': lang_TE.Num2Word_TE(),
+'hu': lang_HU.Num2Word_HU()
+def num2words(number, ordinal=False, lang='en', to='cardinal', **kwargs):
+ # We try the full language first
+ if lang not in CONVERTER_CLASSES:
+ # ... and then try only the first 2 letters
+ lang = lang[:2]
+ if lang not in CONVERTER_CLASSES:
+ raise NotImplementedError()
+ converter = CONVERTER_CLASSES[lang]
+
+ if isinstance(number, str):
+ number = converter.str_to_number(number)
+
+ # backwards compatible
+ if ordinal:
+ return converter.to_ordinal(number)
+
+ if to not in CONVERTES_TYPES:
+ raise NotImplementedError()
+
+ return getattr(converter, 'to_{}'.format(to))(number, **kwargs)
diff --git a/fun_text_processing/num2words/num2words/base.py b/fun_text_processing/num2words/num2words/base.py
new file mode 100644
index 0000000..738eef2
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/base.py
@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import math
+from collections import OrderedDict
+from decimal import Decimal
+
+from .compat import to_s
+from .currency import parse_currency_parts, prefix_currency
+
+
+class Num2Word_Base(object):
+ CURRENCY_FORMS = {}
+ CURRENCY_ADJECTIVES = {}
+
+ def __init__(self):
+ self.is_title = False
+ self.precision = 2
+ self.exclude_title = []
+ self.negword = "(-) "
+ self.pointword = "(.)"
+ self.errmsg_nonnum = "type(%s) not in [long, int, float]"
+ self.errmsg_floatord = "Cannot treat float %s as ordinal."
+ self.errmsg_negord = "Cannot treat negative num %s as ordinal."
+ self.errmsg_toobig = "abs(%s) must be less than %s."
+
+ self.setup()
+
+ # uses cards
+ if any(hasattr(self, field) for field in
+ ['high_numwords', 'mid_numwords', 'low_numwords']):
+ self.cards = OrderedDict()
+ self.set_numwords()
+ self.MAXVAL = 1000 * list(self.cards.keys())[0]
+
+ def set_numwords(self):
+ self.set_high_numwords(self.high_numwords)
+ self.set_mid_numwords(self.mid_numwords)
+ self.set_low_numwords(self.low_numwords)
+ # self.set_labeled_numbers(self.labeled_numbers)
+
+ def set_high_numwords(self, *args):
+ raise NotImplementedError
+
+ # def set_labeled_numbers(self, labeled_numbers):
+ # for key, val in labeled_numbers.items():
+ # self.cards[key] = val
+
+ def set_mid_numwords(self, mid):
+ for key, val in mid:
+ self.cards[key] = val
+
+ def set_low_numwords(self, numwords):
+ for word, n in zip(numwords, range(len(numwords) - 1, -1, -1)):
+ self.cards[n] = word
+
+ def splitnum(self, value):
+ for elem in self.cards:
+ if elem > value:
+ continue
+
+ out = []
+ if value == 0:
+ div, mod = 1, 0
+ else:
+ div, mod = divmod(value, elem)
+
+ if div == 1:
+ out.append((self.cards[1], 1))
+ else:
+ if div == value: # The system tallies, eg Roman Numerals
+ return [(div * self.cards[elem], div*elem)]
+ out.append(self.splitnum(div))
+
+ out.append((self.cards[elem], elem))
+
+ if mod:
+ out.append(self.splitnum(mod))
+
+ return out
+
+ def parse_minus(self, num_str):
+ """Detach minus and return it as symbol with new num_str."""
+ if num_str.startswith('-'):
+ # Extra spacing to compensate if there is no minus.
+ return '%s ' % self.negword, num_str[1:]
+ return '', num_str
+
+ def str_to_number(self, value):
+ return Decimal(value)
+
+ def to_cardinal(self, value):
+ try:
+ assert int(value) == value
+ except (ValueError, TypeError, AssertionError):
+ return self.to_cardinal_float(value)
+
+ out = ""
+ if value < 0:
+ value = abs(value)
+ out = self.negword
+
+ if value >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig % (value, self.MAXVAL))
+
+ val = self.splitnum(value)
+ words, num = self.clean(val)
+ return self.title(out + words)
+
+ def float2tuple(self, value):
+ pre = int(value)
+
+ # Simple way of finding decimal places to update the precision
+ self.precision = abs(Decimal(str(value)).as_tuple().exponent)
+
+ post = abs(value - pre) * 10**self.precision
+ if abs(round(post) - post) < 0.01:
+ # We generally floor all values beyond our precision (rather than
+ # rounding), but in cases where we have something like 1.239999999,
+ # which is probably due to python's handling of floats, we actually
+ # want to consider it as 1.24 instead of 1.23
+ post = int(round(post))
+ else:
+ post = int(math.floor(post))
+
+ return pre, post
+
+ def to_cardinal_float(self, value):
+ try:
+ float(value) == value
+ except (ValueError, TypeError, AssertionError, AttributeError):
+ raise TypeError(self.errmsg_nonnum % value)
+
+ pre, post = self.float2tuple(float(value))
+
+ post = str(post)
+ post = '0' * (self.precision - len(post)) + post
+
+ out = [self.to_cardinal(pre)]
+ if self.precision:
+ out.append(self.title(self.pointword))
+
+ for i in range(self.precision):
+ curr = int(post[i])
+ out.append(to_s(self.to_cardinal(curr)))
+
+ return " ".join(out)
+
+ def merge(self, curr, next):
+ raise NotImplementedError
+
+ def clean(self, val):
+ out = val
+ while len(val) != 1:
+ out = []
+ left, right = val[:2]
+ if isinstance(left, tuple) and isinstance(right, tuple):
+ out.append(self.merge(left, right))
+ if val[2:]:
+ out.append(val[2:])
+ else:
+ for elem in val:
+ if isinstance(elem, list):
+ if len(elem) == 1:
+ out.append(elem[0])
+ else:
+ out.append(self.clean(elem))
+ else:
+ out.append(elem)
+ val = out
+ return out[0]
+
+ def title(self, value):
+ if self.is_title:
+ out = []
+ value = value.split()
+ for word in value:
+ if word in self.exclude_title:
+ out.append(word)
+ else:
+ out.append(word[0].upper() + word[1:])
+ value = " ".join(out)
+ return value
+
+ def verify_ordinal(self, value):
+ if not value == int(value):
+ raise TypeError(self.errmsg_floatord % value)
+ if not abs(value) == value:
+ raise TypeError(self.errmsg_negord % value)
+
+ def to_ordinal(self, value):
+ return self.to_cardinal(value)
+
+ def to_ordinal_num(self, value):
+ return value
+
+ # Trivial version
+ def inflect(self, value, text):
+ text = text.split("/")
+ if value == 1:
+ return text[0]
+ return "".join(text)
+
+ # //CHECK: generalise? Any others like pounds/shillings/pence?
+ def to_splitnum(self, val, hightxt="", lowtxt="", jointxt="",
+ divisor=100, longval=True, cents=True):
+ out = []
+
+ if isinstance(val, float):
+ high, low = self.float2tuple(val)
+ else:
+ try:
+ high, low = val
+ except TypeError:
+ high, low = divmod(val, divisor)
+
+ if high:
+ hightxt = self.title(self.inflect(high, hightxt))
+ out.append(self.to_cardinal(high))
+ if low:
+ if longval:
+ if hightxt:
+ out.append(hightxt)
+ if jointxt:
+ out.append(self.title(jointxt))
+ elif hightxt:
+ out.append(hightxt)
+
+ if low:
+ if cents:
+ out.append(self.to_cardinal(low))
+ else:
+ out.append("%02d" % low)
+ if lowtxt and longval:
+ out.append(self.title(self.inflect(low, lowtxt)))
+
+ return " ".join(out)
+
+ def to_year(self, value, **kwargs):
+ return self.to_cardinal(value)
+
+ def pluralize(self, n, forms):
+ """
+ Should resolve gettext form:
+ http://docs.translatehouse.org/projects/localization-guide/en/latest/l10n/pluralforms.html
+ """
+ raise NotImplementedError
+
+ def _money_verbose(self, number, currency):
+ return self.to_cardinal(number)
+
+ def _cents_verbose(self, number, currency):
+ return self.to_cardinal(number)
+
+ def _cents_terse(self, number, currency):
+ return "%02d" % number
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=',',
+ adjective=False):
+ """
+ Args:
+ val: Numeric value
+ currency (str): Currency code
+ cents (bool): Verbose cents
+ separator (str): Cent separator
+ adjective (bool): Prefix currency name with adjective
+ Returns:
+ str: Formatted string
+
+ """
+ left, right, is_negative = parse_currency_parts(val)
+
+ try:
+ cr1, cr2 = self.CURRENCY_FORMS[currency]
+
+ except KeyError:
+ raise NotImplementedError(
+ 'Currency code "%s" not implemented for "%s"' %
+ (currency, self.__class__.__name__))
+
+ if adjective and currency in self.CURRENCY_ADJECTIVES:
+ cr1 = prefix_currency(self.CURRENCY_ADJECTIVES[currency], cr1)
+
+ minus_str = "%s " % self.negword if is_negative else ""
+ money_str = self._money_verbose(left, currency)
+ cents_str = self._cents_verbose(right, currency) \
+ if cents else self._cents_terse(right, currency)
+
+ return u'%s%s %s%s %s %s' % (
+ minus_str,
+ money_str,
+ self.pluralize(left, cr1),
+ separator,
+ cents_str,
+ self.pluralize(right, cr2)
+ )
+
+ def setup(self):
+ pass
diff --git a/fun_text_processing/num2words/num2words/compat.py b/fun_text_processing/num2words/num2words/compat.py
new file mode 100644
index 0000000..ef030ee
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/compat.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+
+try:
+ strtype = basestring
+except NameError:
+ strtype = str
+
+
+def to_s(val):
+ try:
+ return unicode(val)
+ except NameError:
+ return str(val)
diff --git a/fun_text_processing/num2words/num2words/currency.py b/fun_text_processing/num2words/num2words/currency.py
new file mode 100644
index 0000000..d7f999f
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/currency.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division
+
+from decimal import ROUND_HALF_UP, Decimal
+
+
+def parse_currency_parts(value, is_int_with_cents=True):
+ if isinstance(value, int):
+ if is_int_with_cents:
+ # assume cents if value is integer
+ negative = value < 0
+ value = abs(value)
+ integer, cents = divmod(value, 100)
+ else:
+ negative = value < 0
+ integer, cents = abs(value), 0
+
+ else:
+ value = Decimal(value)
+ value = value.quantize(
+ Decimal('.01'),
+ rounding=ROUND_HALF_UP
+ )
+ negative = value < 0
+ value = abs(value)
+ integer, fraction = divmod(value, 1)
+ integer = int(integer)
+ cents = int(fraction * 100)
+
+ return integer, cents, negative
+
+
+def prefix_currency(prefix, base):
+ return tuple("%s %s" % (prefix, i) for i in base)
diff --git a/fun_text_processing/num2words/num2words/lang_AR.py b/fun_text_processing/num2words/num2words/lang_AR.py
new file mode 100644
index 0000000..a0027a8
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_AR.py
@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2003, Taro Ogawa. All Rights Reserved.
+# Copyright (c) 2013, Savoir-faire Linux inc. All Rights Reserved.
+# Copyright (c) 2018, Abdullah Alhazmy, Alhazmy13. All Rights Reserved.
+
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301 USA
+
+import re
+from decimal import Decimal
+from math import floor
+
+CURRENCY_SR = [("乇賷丕賱", "乇賷丕賱丕賳", "乇賷丕賱丕鬲", "乇賷丕賱丕賸"),
+ ("賴賱賱丞", "賴賱賱鬲丕賳", "賴賱賱丕鬲", "賴賱賱丞")]
+CURRENCY_EGP = [("噩賳賷賴", "噩賳賷賴丕賳", "噩賳賷賴丕鬲", "噩賳賷賴丕賸"),
+ ("賯乇卮", "賯乇卮丕賳", "賯乇賵卮", "賯乇卮")]
+CURRENCY_KWD = [("丿賷賳丕乇", "丿賷賳丕乇丕賳", "丿賷賳丕乇丕鬲", "丿賷賳丕乇丕賸"),
+ ("賮賱爻", "賮賱爻丕賳", "賮賱爻", "賮賱爻")]
+
+ARABIC_ONES = [
+ "", "賵丕丨丿", "丕孬賳丕賳", "孬賱丕孬丞", "兀乇亘毓丞", "禺賲爻丞", "爻鬲丞", "爻亘毓丞", "孬賲丕賳賷丞",
+ "鬲爻毓丞",
+ "毓卮乇丞", "兀丨丿 毓卮乇", "丕孬賳丕 毓卮乇", "孬賱丕孬丞 毓卮乇", "兀乇亘毓丞 毓卮乇", "禺賲爻丞 毓卮乇",
+ "爻鬲丞 毓卮乇", "爻亘毓丞 毓卮乇", "孬賲丕賳賷丞 毓卮乇",
+ "鬲爻毓丞 毓卮乇"
+]
+
+
+class Num2Word_AR(object):
+ errmsg_too_big = "Too large"
+ max_num = 10 ** 36
+
+ def __init__(self):
+ self.number = 0
+ self.arabicPrefixText = ""
+ self.arabicSuffixText = ""
+ self.integer_value = 0
+ self._decimalValue = ""
+ self.partPrecision = 2
+ self.currency_unit = CURRENCY_SR[0]
+ self.currency_subunit = CURRENCY_SR[1]
+ self.isCurrencyPartNameFeminine = True
+ self.isCurrencyNameFeminine = False
+ self.separator = '賵'
+
+ self.arabicOnes = ARABIC_ONES
+ self.arabicFeminineOnes = [
+ "", "廿丨丿賶", "丕孬賳鬲丕賳", "孬賱丕孬", "兀乇亘毓", "禺賲爻", "爻鬲", "爻亘毓", "孬賲丕賳",
+ "鬲爻毓",
+ "毓卮乇", "廿丨丿賶 毓卮乇丞", "丕孬賳鬲丕 毓卮乇丞", "孬賱丕孬 毓卮乇丞", "兀乇亘毓 毓卮乇丞",
+ "禺賲爻 毓卮乇丞", "爻鬲 毓卮乇丞", "爻亘毓 毓卮乇丞", "孬賲丕賳賷 毓卮乇丞",
+ "鬲爻毓 毓卮乇丞"
+ ]
+ self.arabicOrdinal = [
+ "", "丕賵賱", "孬丕賳賷", "孬丕賱孬", "乇丕亘毓", "禺丕賲爻", "爻丕丿爻", "爻丕亘毓", "孬丕賲賳",
+ "鬲丕爻毓", "毓丕卮乇", "丨丕丿賷 毓卮乇", "孬丕賳賷 毓卮乇", "孬丕賱孬 毓卮乇", "乇丕亘毓 毓卮乇",
+ "禺丕賲爻 毓卮乇", "爻丕丿爻 毓卮乇", "爻丕亘毓 毓卮乇", "孬丕賲賳 毓卮乇", "鬲丕爻毓 毓卮乇"
+ ]
+ self.arabicTens = [
+ "毓卮乇賵賳", "孬賱丕孬賵賳", "兀乇亘毓賵賳", "禺賲爻賵賳", "爻鬲賵賳", "爻亘毓賵賳", "孬賲丕賳賵賳",
+ "鬲爻毓賵賳"
+ ]
+ self.arabicHundreds = [
+ "", "賲丕卅丞", "賲卅鬲丕賳", "孬賱丕孬賲丕卅丞", "兀乇亘毓賲丕卅丞", "禺賲爻賲丕卅丞", "爻鬲賲丕卅丞",
+ "爻亘毓賲丕卅丞", "孬賲丕賳賲丕卅丞", "鬲爻毓賲丕卅丞"
+ ]
+ self.arabicAppendedTwos = [
+ "賲卅鬲丕", "兀賱賮丕", "賲賱賷賵賳丕", "賲賱賷丕乇丕", "鬲乇賷賱賷賵賳丕", "賰賵丕丿乇賷賱賷賵賳丕",
+ "賰賵賷賳鬲賱賷賵賳丕", "爻賰爻鬲賷賱賷賵賳丕"
+ ]
+ self.arabicTwos = [
+ "賲卅鬲丕賳", "兀賱賮丕賳", "賲賱賷賵賳丕賳", "賲賱賷丕乇丕賳", "鬲乇賷賱賷賵賳丕賳",
+ "賰賵丕丿乇賷賱賷賵賳丕賳", "賰賵賷賳鬲賱賷賵賳丕賳", "爻賰爻鬲賷賱賷賵賳丕賳"
+ ]
+ self.arabicGroup = [
+ "賲丕卅丞", "兀賱賮", "賲賱賷賵賳", "賲賱賷丕乇", "鬲乇賷賱賷賵賳", "賰賵丕丿乇賷賱賷賵賳",
+ "賰賵賷賳鬲賱賷賵賳", "爻賰爻鬲賷賱賷賵賳"
+ ]
+ self.arabicAppendedGroup = [
+ "", "兀賱賮丕賸", "賲賱賷賵賳丕賸", "賲賱賷丕乇丕賸", "鬲乇賷賱賷賵賳丕賸", "賰賵丕丿乇賷賱賷賵賳丕賸",
+ "賰賵賷賳鬲賱賷賵賳丕賸", "爻賰爻鬲賷賱賷賵賳丕賸"
+ ]
+ self.arabicPluralGroups = [
+ "", "丌賱丕賮", "賲賱丕賷賷賳", "賲賱賷丕乇丕鬲", "鬲乇賷賱賷賵賳丕鬲", "賰賵丕丿乇賷賱賷賵賳丕鬲",
+ "賰賵賷賳鬲賱賷賵賳丕鬲", "爻賰爻鬲賷賱賷賵賳丕鬲"
+ ]
+
+ def number_to_arabic(self, arabic_prefix_text, arabic_suffix_text):
+ self.arabicPrefixText = arabic_prefix_text
+ self.arabicSuffixText = arabic_suffix_text
+ self.extract_integer_and_decimal_parts()
+
+ def extract_integer_and_decimal_parts(self):
+ re.split('\\.', str(self.number))
+ splits = re.split('\\.', str(self.number))
+
+ self.integer_value = int(splits[0])
+ if len(splits) > 1:
+ self._decimalValue = int(self.decimal_value(splits[1]))
+ else:
+ self._decimalValue = 0
+
+ def decimal_value(self, decimal_part):
+
+ if self.partPrecision is not len(decimal_part):
+ decimal_part_length = len(decimal_part)
+
+ decimal_part_builder = decimal_part
+ for i in range(0, self.partPrecision - decimal_part_length):
+ decimal_part_builder += '0'
+ decimal_part = decimal_part_builder
+
+ if len(decimal_part) <= self.partPrecision:
+ dec = len(decimal_part)
+ else:
+ dec = self.partPrecision
+ result = decimal_part[0: dec]
+ else:
+ result = decimal_part
+
+ for i in range(len(result), self.partPrecision):
+ result += '0'
+ return result
+
+ def digit_feminine_status(self, digit, group_level):
+ if group_level == -1:
+ if self.isCurrencyPartNameFeminine:
+ return self.arabicFeminineOnes[int(digit)]
+ else:
+ return self.arabicOnes[int(digit)]
+ elif group_level == 0:
+ if self.isCurrencyNameFeminine:
+ return self.arabicFeminineOnes[int(digit)]
+ else:
+ return self.arabicOnes[int(digit)]
+
+ else:
+ return self.arabicOnes[int(digit)]
+
+ def process_arabic_group(self, group_number, group_level,
+ remaining_number):
+ tens = Decimal(group_number) % Decimal(100)
+ hundreds = Decimal(group_number) / Decimal(100)
+ ret_val = ""
+
+ if int(hundreds) > 0:
+ if tens == 0 and int(hundreds) == 2:
+ ret_val = "{}".format(self.arabicAppendedTwos[0])
+ else:
+ ret_val = "{}".format(self.arabicHundreds[int(hundreds)])
+
+ if tens > 0:
+ if tens < 20:
+ if tens == 2 and int(hundreds) == 0 and group_level > 0:
+ if self.integer_value in [2000, 2000000, 2000000000,
+ 2000000000000, 2000000000000000,
+ 2000000000000000000]:
+ ret_val = "{}".format(
+ self.arabicAppendedTwos[int(group_level)])
+ else:
+ ret_val = "{}".format(
+ self.arabicTwos[int(group_level)])
+ else:
+ if ret_val != "":
+ ret_val += " 賵 "
+
+ if tens == 1 and group_level > 0 and hundreds == 0:
+ ret_val += ""
+ elif (tens == 1 or tens == 2) and (
+ group_level == 0 or group_level == -1) and \
+ hundreds == 0 and remaining_number == 0:
+ ret_val += ""
+ else:
+ ret_val += self.digit_feminine_status(int(tens),
+ group_level)
+ else:
+ ones = tens % 10
+ tens = (tens / 10) - 2
+ if ones > 0:
+ if ret_val != "" and tens < 4:
+ ret_val += " 賵 "
+
+ ret_val += self.digit_feminine_status(ones, group_level)
+ if ret_val != "" and ones != 0:
+ ret_val += " 賵 "
+
+ ret_val += self.arabicTens[int(tens)]
+
+ return ret_val
+
+ def convert(self, value):
+ self.number = "{:.9f}".format(value)
+ self.number_to_arabic(self.arabicPrefixText, self.arabicSuffixText)
+ return self.convert_to_arabic()
+
+ def convert_to_arabic(self):
+ temp_number = Decimal(self.number)
+
+ if temp_number == Decimal(0):
+ return "氐賮乇"
+
+ decimal_string = self.process_arabic_group(self._decimalValue,
+ -1,
+ Decimal(0))
+ ret_val = ""
+ group = 0
+
+ while temp_number > Decimal(0):
+
+ number_to_process = int(
+ Decimal(str(temp_number)) % Decimal(str(1000)))
+ temp_number = int(Decimal(temp_number) / Decimal(1000))
+
+ group_description = \
+ self.process_arabic_group(number_to_process,
+ group,
+ Decimal(floor(temp_number)))
+ if group_description != '':
+ if group > 0:
+ if ret_val != "":
+ ret_val = "{} 賵 {}".format("", ret_val)
+ if number_to_process != 2:
+ if number_to_process % 100 != 1:
+ if 3 <= number_to_process <= 10:
+ ret_val = "{} {}".format(
+ self.arabicPluralGroups[group], ret_val)
+ else:
+ if ret_val != "":
+ ret_val = "{} {}".format(
+ self.arabicAppendedGroup[group],
+ ret_val)
+ else:
+ ret_val = "{} {}".format(
+ self.arabicGroup[group], ret_val)
+
+ else:
+ ret_val = "{} {}".format(self.arabicGroup[group],
+ ret_val)
+ ret_val = "{} {}".format(group_description, ret_val)
+ group += 1
+ formatted_number = ""
+ if self.arabicPrefixText != "":
+ formatted_number += "{} ".format(self.arabicPrefixText)
+ formatted_number += ret_val
+ if self.integer_value != 0:
+ remaining100 = int(self.integer_value % 100)
+
+ if remaining100 == 0:
+ formatted_number += self.currency_unit[0]
+ elif remaining100 == 1:
+ formatted_number += self.currency_unit[0]
+ elif remaining100 == 2:
+ if self.integer_value == 2:
+ formatted_number += self.currency_unit[1]
+ else:
+ formatted_number += self.currency_unit[0]
+ elif 3 <= remaining100 <= 10:
+ formatted_number += self.currency_unit[2]
+ elif 11 <= remaining100 <= 99:
+ formatted_number += self.currency_unit[3]
+ if self._decimalValue != 0:
+ formatted_number += " {} ".format(self.separator)
+ formatted_number += decimal_string
+
+ if self._decimalValue != 0:
+ formatted_number += " "
+ remaining100 = int(self._decimalValue % 100)
+
+ if remaining100 == 0:
+ formatted_number += self.currency_subunit[0]
+ elif remaining100 == 1:
+ formatted_number += self.currency_subunit[0]
+ elif remaining100 == 2:
+ formatted_number += self.currency_subunit[1]
+ elif 3 <= remaining100 <= 10:
+ formatted_number += self.currency_subunit[2]
+ elif 11 <= remaining100 <= 99:
+ formatted_number += self.currency_subunit[3]
+
+ if self.arabicSuffixText != "":
+ formatted_number += " {}".format(self.arabicSuffixText)
+
+ return formatted_number
+
+ def validate_number(self, number):
+ if number >= self.max_num:
+ raise OverflowError(self.errmsg_too_big)
+ return number
+
+ def set_currency_prefer(self, currency):
+ if currency == 'EGP':
+ self.currency_unit = CURRENCY_EGP[0]
+ self.currency_subunit = CURRENCY_EGP[1]
+ elif currency == 'KWD':
+ self.currency_unit = CURRENCY_KWD[0]
+ self.currency_subunit = CURRENCY_KWD[1]
+ else:
+ self.currency_unit = CURRENCY_SR[0]
+ self.currency_subunit = CURRENCY_SR[1]
+
+ def to_currency(self, value, currency='SR', prefix='', suffix=''):
+ self.set_currency_prefer(currency)
+ self.isCurrencyNameFeminine = False
+ self.separator = "賵"
+ self.arabicOnes = ARABIC_ONES
+ self.arabicPrefixText = prefix
+ self.arabicSuffixText = suffix
+ return self.convert(value=value)
+
+ def to_ordinal(self, number, prefix=''):
+ if number <= 19:
+ return "{}".format(self.arabicOrdinal[number])
+ if number < 100:
+ self.isCurrencyNameFeminine = True
+ else:
+ self.isCurrencyNameFeminine = False
+ self.currency_subunit = ('', '', '', '')
+ self.currency_unit = ('', '', '', '')
+ self.arabicPrefixText = prefix
+ self.arabicSuffixText = ""
+ return "{}".format(self.convert(abs(number)).strip())
+
+ def to_year(self, value):
+ value = self.validate_number(value)
+ return self.to_cardinal(value)
+
+ def to_ordinal_num(self, value):
+ return self.to_ordinal(value).strip()
+
+ def to_cardinal(self, number):
+ number = self.validate_number(number)
+ minus = ''
+ if number < 0:
+ minus = '爻丕賱亘 '
+ self.separator = ','
+ self.currency_subunit = ('', '', '', '')
+ self.currency_unit = ('', '', '', '')
+ self.arabicPrefixText = ""
+ self.arabicSuffixText = ""
+ self.arabicOnes = ARABIC_ONES
+ return minus + self.convert(value=abs(number)).strip()
diff --git a/fun_text_processing/num2words/num2words/lang_AZ.py b/fun_text_processing/num2words/num2words/lang_AZ.py
new file mode 100644
index 0000000..a16d4bf
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_AZ.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_AZ(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word + " milyon"
+
+ def setup(self):
+ super(Num2Word_AZ, self).setup()
+
+ self.negword = "m蓹nfi "
+ self.pointword = "n枚qt蓹"
+ self.exclude_title = ["v蓹", "n枚qt蓹", "m蓹nfi"]
+
+ self.mid_numwords = [(1000, "min"), (100, "y眉z"),
+ (90, "doxsan"), (80, "s蓹ks蓹n"), (70, "yetmi艧"),
+ (60, "altm谋艧"), (50, "蓹lli"), (40, "q谋rx"),
+ (30, "otuz")]
+ self.low_numwords = ['iyirmi', 'on doqquz', 'on s蓹kkiz', 'on yeddi', 'on alt谋', 'on be艧', 'on d枚rd', 'on 眉莽', 'on iki', 'on bir', 'on', 'doqquz', 's蓹kkiz', 'yeddi', 'alt谋', 'be艧', 'd枚rd', '眉莽', 'iki', 'bir', 's谋f谋r']
+ self.ords = {'bir': 'birinci',
+ 'iki': 'ikinci',
+ '眉莽': '眉莽眉nc眉',
+ 'd枚rd': 'd枚rd眉nc眉',
+ 'be艧': 'be艧inci',
+ 'alt谋': 'alt谋nc谋',
+ 'yeddi': 'yeddinci',
+ 's蓹kkiz': 's蓹kkizinci',
+ 'doqquz': 'doqquzuncu',
+ 'on': 'onuncu'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s v蓹 %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("-")
+ lastword = lastwords[-1].lower()
+ try:
+ lastword = self.ords[lastword]
+ except KeyError:
+ if lastword[-1] == "y":
+ lastword = lastword[:-1] + "ie"
+ lastword += "th"
+ lastwords[-1] = self.title(lastword)
+ outwords[-1] = "-".join(lastwords)
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '蓹srd蓹n 蓹vv蓹l ' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "y眉z"
+ elif low < 10:
+ lowtext = "oh-%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_BG.py b/fun_text_processing/num2words/num2words/lang_BG.py
new file mode 100644
index 0000000..804502b
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_BG.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+# Bulgarian num2words
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_BG(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word + " 屑懈谢懈芯薪邪"
+
+ def setup(self):
+ super(Num2Word_BG, self).setup()
+
+ self.negword = "屑懈薪褍褋 "
+ self.pointword = "褌芯褔泻邪"
+ self.exclude_title = ["懈", "褌芯褔泻邪", "屑懈薪褍褋"]
+
+ self.mid_numwords = [(1000, "褏懈谢褟写邪"), (100, "褋褌芯"),
+ (90, "写械胁械褌写械褋械褌"), (80, "芯褋械屑写械褋械褌"), (70, "褋械写械屑写械褋械褌"),
+ (60, "褕械褋褌写械褋械褌"), (50, "锌械褌写械褋械褌"), (40, "褔械褌懈褉懈写械褋械褌"),
+ (30, "褌褉懈写械褋械褌")]
+ self.low_numwords = ['写胁邪写械褋械褌', '写械胁械褌薪邪泄褋械褌', '芯褋械屑薪邪写械褋械褌', '褋械写械屑薪邪写械褋械褌', '褕械褋褌薪邪写械褋械褌', '锌械褌薪邪写械褋械褌', '褔械褌懈褉懈薪邪写械褋械褌', '褌褉懈薪邪写械褋械褌', '写胁邪薪邪写械褋械褌', '械写懈薪邪写械褋械褌', '写械褋械褌', '写械胁械褌', '芯褋械屑', '褋械写械屑', '褕械褋褌', '锌械褌', '褔械褌懈褉懈', '褌褉懈', '写胁械', '械写懈薪', '薪褍谢邪']
+ self.ords = {'械写懈薪': '锌褗褉胁懈',
+'写胁械': '胁褌芯褉芯',
+'褌褉懈': '褌褉械褌懈',
+'褔械褌懈褉懈': '褔械褌胁褗褉褌芯',
+'锌械褌': '锌械褌懈',
+'褕械褋褌': '褕械褋褌芯',
+'褋械写械屑': '褋械写屑芯',
+'芯褋械屑': '芯褋屑芯',
+'写械胁械褌': '写械胁械褌懈',
+'写械褋械褌': '写械褋械褌懈'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s 懈 %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s, %s" % (ltext, rtext), lnum + rnum)
+
+ # def to_ordinal(self, value):
+ # self.verify_ordinal(value)
+ # outwords = self.to_cardinal(value).split(" ")
+ # lastwords = outwords[-1].split("-")
+ # lastword = lastwords[-1].lower()
+ # try:
+ # lastword = self.ords[lastword]
+ # except KeyError:
+ # if lastword[-1] == "y":
+ # lastword = lastword[:-1] + "ie"
+ # lastword += "th"
+ # lastwords[-1] = self.title(lastword)
+ # outwords[-1] = "-".join(lastwords)
+ # return " ".join(outwords)
+
+ # def to_ordinal_num(self, value):
+ # self.verify_ordinal(value)
+ # return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '谐. 锌褉.薪.械' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "褋褌芯"
+ elif low < 10:
+ lowtext = "oh-%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_BN.py b/fun_text_processing/num2words/num2words/lang_BN.py
new file mode 100644
index 0000000..a308e73
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_BN.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+# Bengali
+
+class Num2Word_BN(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_BN, self).setup()
+
+ self.negword = "唳唳唰嬥 "
+ self.pointword = "唳Ο唳监唳ㄠ唳�"
+ self.exclude_title = ["唳忇Μ唳�", "唳Ο唳监唳ㄠ唳�", "唳唳唰嬥"]
+
+ self.high_numwords = [(100000000000, "唳熰唳班唳侧唳唳�"),
+ (1000000000, "唳忇 唳熰唳班唳侧唳唳�"),
+ (100000000, "唳唳侧唳唳�"),
+ (10000000, "唳忇 唳多Δ 唳唳侧唳唳�"),
+ (1000000, "唳曕唳熰"),
+ (100000, "唳︵Χ 唳侧唰嵿Ψ"),
+ (10000, "唳︵Χ 唳灌唳溹唳�")]
+
+ self.mid_numwords = [(1000, "唳灌唳溹唳�"),
+ (100, "唳多Δ"),
+ (90, "唳ㄠΜ唰嵿Μ唳�"), (80, "唳嗋Χ唳�"), (70, "唳膏Δ唰嵿Δ唳�"),
+ (60, "唳粪唳�"), (50, "唳唰嵿唳距Χ"), (40, "唳氞Σ唰嵿Σ唳苦Χ"),
+ (30, "唳む唳班唳�")]
+ self.low_numwords = ["唳唳�","唳夃Θ唳苦Χ","唳嗋唳距Π","唳膏Δ唰囙Π","唳粪唳�","唳Θ唰囙Π","唳氞唳︵唳�","唳む唳班","唳唳班","唳忇唳距Π唰�","唳︵Χ","唳ㄠΟ唳监唳�","唳嗋","唳膏唳�","唳涏Ο唳�","唳唳佮","唳氞唳�","唳む唳�","唳︵唳�","唳忇","唳多唳ㄠ唳�"]
+ self.ords = {"唳忇": "唳唳班Ε唳�",
+ "唳︵唳�": "唳︵唳唳む唳",
+ "唳む唳�": "唳む唳む唳",
+ "唳氞唳�": "唳氞Δ唰佮Π唰嵿Ε",
+ "唳唳佮": "唳唰嵿唳�",
+ "唳涏Ο唳�": "唳粪Ψ唰嵿",
+ "唳膏唳�": "唳膏Κ唰嵿Δ唳�",
+ "唳嗋": "唳呧Ψ唰嵿唳�",
+ "唳ㄠΟ唳监唳�": "唳ㄠΜ唳�",
+ "唳︵Χ": "唳︵Χ唳�",
+ "唳忇唳距Π唰�": "唳忇唳距Ζ唳�",
+ "唳唳班": "唳︵唳唳︵Χ",
+ "唳む唳班": "唳む唳班Ο唳监唳︵Χ",
+ "唳氞唳︵唳�": "唳氞Δ唰佮Π唰嵿Ζ唳�",
+ "唳Θ唰囙Π": "唳唰嵿唳︵Χ",
+ "唳粪唳�": "唳粪唳∴唳�",
+ "唳膏Δ唰囙Π": "唳膏Κ唰嵿Δ唳︵Χ唳むΞ",
+ "唳嗋唳距Π": "唳呧Ψ唰嵿唳距Ζ唳�",
+ "唳夃Θ唳苦Χ": "唳夃Θ唳苦Χ唳むΞ",
+ "唳唳�": "唳唳傕Χ唳むΞ"}
+
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("-")
+ lastword = lastwords[-1].lower()
+ try:
+ lastword = self.ords[lastword]
+ except KeyError:
+ lastword += "唳むΞ"
+ lastwords[-1] = self.title(lastword)
+ outwords[-1] = " ".join(lastwords)
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '唳栢唳班唳膏唳熰Κ唰傕Π唰嵿Μ' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "唳多Δ"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_CA.py b/fun_text_processing/num2words/num2words/lang_CA.py
new file mode 100644
index 0000000..1910e55
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_CA.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_CA(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word + "illions"
+
+ def setup(self):
+ super(Num2Word_CA, self).setup()
+
+ self.negword = "menys "
+ self.pointword = "punt"
+ self.exclude_title = ["i", "punt", "menys"]
+
+ self.mid_numwords = [(1000, "mil"), (100, "cent"),
+ (90, "noranta"), (80, "vuitanta"), (70, "setanta"),
+ (60, "seixanta"), (50, "cinquanta"), (40, "quaranta"),
+ (30, "trenta")]
+ self.low_numwords = ['vint', 'dinou', 'divuit anys', 'disset', 'setze', 'quinze', 'catorze', 'tretze', 'dotze', 'onze', 'deu', 'nou', 'vuit', 'set', 'sis', 'cinc', 'quatre', 'tres', 'dos', 'un', 'zero']
+ self.ords = {'un': 'primer',
+ 'dos': 'segon',
+ 'tres': 'tercer',
+ 'quatre': 'quart',
+ 'cinc': 'cinqu猫',
+ 'sis': 'sis猫',
+ 'set': 'set猫',
+ 'vuit': 'vuit猫',
+ 'nou': 'nov猫',
+ 'deu': 'des猫'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s i %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s, %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("-")
+ lastword = lastwords[-1].lower()
+ try:
+ lastword = self.ords[lastword]
+ except KeyError:
+ if lastword[-1] == "y":
+ lastword = lastword[:-1] + "ie"
+ lastword += "th"
+ lastwords[-1] = self.title(lastword)
+ outwords[-1] = "-".join(lastwords)
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "cent"
+ elif low < 10:
+ lowtext = "oh-%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_CZ.py b/fun_text_processing/num2words/num2words/lang_CZ.py
new file mode 100644
index 0000000..634f361
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_CZ.py
@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('nula',)
+
+ONES = {
+ 1: ('jedna',),
+ 2: ('dva',),
+ 3: ('t艡i',),
+ 4: ('膷ty艡i',),
+ 5: ('p臎t',),
+ 6: ('拧est',),
+ 7: ('sedm',),
+ 8: ('osm',),
+ 9: ('dev臎t',),
+}
+
+TENS = {
+ 0: ('deset',),
+ 1: ('jeden谩ct',),
+ 2: ('dvan谩ct',),
+ 3: ('t艡in谩ct',),
+ 4: ('膷trn谩ct',),
+ 5: ('patn谩ct',),
+ 6: ('拧estn谩ct',),
+ 7: ('sedmn谩ct',),
+ 8: ('osmn谩ct',),
+ 9: ('devaten谩ct',),
+}
+
+TWENTIES = {
+ 2: ('dvacet',),
+ 3: ('t艡icet',),
+ 4: ('膷ty艡icet',),
+ 5: ('pades谩t',),
+ 6: ('拧edes谩t',),
+ 7: ('sedmdes谩t',),
+ 8: ('osmdes谩t',),
+ 9: ('devades谩t',),
+}
+
+HUNDREDS = {
+ 1: ('sto',),
+ 2: ('dv臎st臎',),
+ 3: ('t艡ista',),
+ 4: ('膷ty艡ista',),
+ 5: ('p臎tset',),
+ 6: ('拧estset',),
+ 7: ('sedmset',),
+ 8: ('osmset',),
+ 9: ('dev臎tset',),
+}
+
+THOUSANDS = {
+ 1: ('tis铆c', 'tis铆ce', 'tis铆c'), # 10^3
+ 2: ('milion', 'miliony', 'milion暖'), # 10^6
+ 3: ('miliarda', 'miliardy', 'miliard'), # 10^9
+ 4: ('bilion', 'biliony', 'bilion暖'), # 10^12
+ 5: ('biliarda', 'biliardy', 'biliard'), # 10^15
+ 6: ('trilion', 'triliony', 'trilion暖'), # 10^18
+ 7: ('triliarda', 'triliardy', 'triliard'), # 10^21
+ 8: ('kvadrilion', 'kvadriliony', 'kvadrilion暖'), # 10^24
+ 9: ('kvadriliarda', 'kvadriliardy', 'kvadriliard'), # 10^27
+ 10: ('quintillion', 'quintilliony', 'quintillion暖'), # 10^30
+}
+
+
+class Num2Word_CZ(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'CZK': (
+ ('koruna', 'koruny', 'korun'), ('hal铆艡', 'hal铆艡e', 'hal茅艡暖')
+ ),
+ 'EUR': (
+ ('euro', 'euro', 'euro'), ('cent', 'centy', 'cent暖')
+ ),
+ }
+
+ def setup(self):
+ self.negword = "m铆nus"
+ self.pointword = "cel谩"
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return u'%s %s %s' % (
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return self._int2word(int(n))
+
+ def pluralize(self, n, forms):
+ if n == 1:
+ form = 0
+ elif 5 > n % 10 > 1 and (n % 100 < 10 or n % 100 > 20):
+ form = 1
+ else:
+ form = 2
+ return forms[form]
+
+ def to_ordinal(self, number):
+ raise NotImplementedError()
+
+ def _int2word(self, n):
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0 and not (i > 0 and x == 1):
+ words.append(ONES[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
diff --git a/fun_text_processing/num2words/num2words/lang_DE.py b/fun_text_processing/num2words/num2words/lang_DE.py
new file mode 100644
index 0000000..5e905ee
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_DE.py
@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+import re
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_DE(Num2Word_EU):
+ CURRENCY_FORMS = {
+ 'EUR': (('Euro', 'Euro'), ('Cent', 'Cent')),
+ 'GBP': (('Pfund', 'Pfund'), ('Penny', 'Pence')),
+ 'USD': (('Dollar', 'Dollar'), ('Cent', 'Cent')),
+ 'CNY': (('Yuan', 'Yuan'), ('Jiao', 'Fen')),
+ 'DEM': (('Mark', 'Mark'), ('Pfennig', 'Pfennig')),
+ }
+
+ GIGA_SUFFIX = "illiarde"
+ MEGA_SUFFIX = "illion"
+
+ def setup(self):
+ self.negword = "minus "
+ self.pointword = "Komma"
+ # "Cannot treat float %s as ordinal."
+ self.errmsg_floatord = (
+ "Die Gleitkommazahl %s kann nicht in eine Ordnungszahl " +
+ "konvertiert werden."
+ )
+ # "type(((type(%s)) ) not in [long, int, float]"
+ self.errmsg_nonnum = (
+ "Nur Zahlen (type(%s)) k枚nnen in W枚rter konvertiert werden."
+ )
+ # "Cannot treat negative num %s as ordinal."
+ self.errmsg_negord = (
+ "Die negative Zahl %s kann nicht in eine Ordnungszahl " +
+ "konvertiert werden."
+ )
+ # "abs(%s) must be less than %s."
+ self.errmsg_toobig = "Die Zahl %s muss kleiner als %s sein."
+ self.exclude_title = []
+
+ lows = ["Non", "Okt", "Sept", "Sext", "Quint", "Quadr", "Tr", "B", "M"]
+ units = ["", "un", "duo", "tre", "quattuor", "quin", "sex", "sept",
+ "okto", "novem"]
+ tens = ["dez", "vigint", "trigint", "quadragint", "quinquagint",
+ "sexagint", "septuagint", "oktogint", "nonagint"]
+ self.high_numwords = (
+ ["zent"] + self.gen_high_numwords(units, tens, lows)
+ )
+ self.mid_numwords = [(1000, "tausend"), (100, "hundert"),
+ (90, "neunzig"), (80, "achtzig"), (70, "siebzig"),
+ (60, "sechzig"), (50, "f\xFCnfzig"),
+ (40, "vierzig"), (30, "drei\xDFig")]
+ self.low_numwords = ["zwanzig", "neunzehn", "achtzehn", "siebzehn",
+ "sechzehn", "f\xFCnfzehn", "vierzehn", "dreizehn",
+ "zw\xF6lf", "elf", "zehn", "neun", "acht",
+ "sieben", "sechs", "f\xFCnf", "vier", "drei",
+ "zwei", "eins", "null"]
+ self.ords = {"eins": "ers",
+ "drei": "drit",
+ "acht": "ach",
+ "sieben": "sieb",
+ "ig": "igs",
+ "ert": "erts",
+ "end": "ends",
+ "ion": "ions",
+ "nen": "ns",
+ "rde": "rds",
+ "rden": "rds"}
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum == 100 or nnum == 1000:
+ return ("ein" + ntext, nnum)
+ elif nnum < 10 ** 6:
+ return next
+ ctext = "eine"
+
+ if nnum > cnum:
+ if nnum >= 10 ** 6:
+ if cnum > 1:
+ if ntext.endswith("e"):
+ ntext += "n"
+ else:
+ ntext += "en"
+ ctext += " "
+ val = cnum * nnum
+ else:
+ if nnum < 10 < cnum < 100:
+ if nnum == 1:
+ ntext = "ein"
+ ntext, ctext = ctext, ntext + "und"
+ elif cnum >= 10 ** 6:
+ ctext += " "
+ val = cnum + nnum
+
+ word = ctext + ntext
+ return (word, val)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outword = self.to_cardinal(value).lower()
+ for key in self.ords:
+ if outword.endswith(key):
+ outword = outword[:len(outword) - len(key)] + self.ords[key]
+ break
+
+ res = outword + "te"
+
+ # Exception: "hundertste" is usually preferred over "einhundertste"
+ if res == "eintausendste" or res == "einhundertste":
+ res = res.replace("ein", "", 1)
+ # ... similarly for "millionste" etc.
+ res = re.sub(r'eine ([a-z]+(illion|illiard)ste)$',
+ lambda m: m.group(1), res)
+ # Ordinals involving "Million" etc. are written without a space.
+ # see https://de.wikipedia.org/wiki/Million#Sprachliches
+ res = re.sub(r' ([a-z]+(illion|illiard)ste)$',
+ lambda m: m.group(1), res)
+
+ return res
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return str(value) + "."
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=' und',
+ adjective=False):
+ result = super(Num2Word_DE, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ # Handle exception, in german is "ein Euro" and not "eins Euro"
+ return result.replace("eins ", "ein ")
+
+ def to_year(self, val, longval=True):
+ if not (val // 100) % 10:
+ return self.to_cardinal(val)
+ return self.to_splitnum(val, hightxt="hundert", longval=longval)\
+ .replace(' ', '')
diff --git a/fun_text_processing/num2words/num2words/lang_DK.py b/fun_text_processing/num2words/num2words/lang_DK.py
new file mode 100644
index 0000000..6b527cf
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_DK.py
@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_DK(lang_EU.Num2Word_EU):
+ GIGA_SUFFIX = "illarder"
+ MEGA_SUFFIX = "illioner"
+
+ def setup(self):
+ super(Num2Word_DK, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "komma"
+ self.exclude_title = ["og", "komma", "minus"]
+
+ self.mid_numwords = [(1000, "tusind"), (100, "hundrede"),
+ (90, "halvfems"), (80, "firs"),
+ (70, "halvfjerds"), (60, "treds"),
+ (50, "halvtreds"), (40, "fyrre"), (30, "tredive")]
+ self.low_numwords = ["tyve", "nitten", "atten", "sytten",
+ "seksten", "femten", "fjorten", "tretten",
+ "tolv", "elleve", "ti", "ni", "otte",
+ "syv", "seks", "fem", "fire", "tre", "to",
+ "et", "nul"]
+ self.ords = {"nul": "nul",
+ "et": "f\xf8rste",
+ "to": "anden",
+ "tre": "tredje",
+ "fire": "fjerde",
+ "fem": "femte",
+ "seks": "sjette",
+ "syv": "syvende",
+ "otte": "ottende",
+ "ni": "niende",
+ "ti": "tiende",
+ "elleve": "ellevte",
+ "tolv": "tolvte",
+ "tretten": "trett",
+ "fjorten": "fjort",
+ "femten": "femt",
+ "seksten": "sekst",
+ "sytten": "sytt",
+ "atten": "att",
+ "nitten": "nitt",
+ "tyve": "tyv"}
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+ if next[1] == 100 or next[1] == 1000:
+ lst = list(next)
+ lst[0] = 'et' + lst[0]
+ next = tuple(lst)
+
+ if cnum == 1:
+ if nnum < 10 ** 6 or self.ordflag:
+ return next
+ ctext = "en"
+ if nnum > cnum:
+ if nnum >= 10 ** 6:
+ ctext += " "
+ val = cnum * nnum
+ else:
+ if cnum >= 100 and cnum < 1000:
+ ctext += " og "
+ elif cnum >= 1000 and cnum <= 100000:
+ ctext += "e og "
+ if nnum < 10 < cnum < 100:
+ if nnum == 1:
+ ntext = "en"
+ ntext, ctext = ctext, ntext + "og"
+ elif cnum >= 10 ** 6:
+ ctext += " "
+ val = cnum + nnum
+ word = ctext + ntext
+ return (word, val)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ self.ordflag = True
+ outword = self.to_cardinal(value)
+ self.ordflag = False
+ for key in self.ords:
+ if outword.endswith(key):
+ outword = outword[:len(outword) - len(key)] + self.ords[key]
+ break
+ if value % 100 >= 30 and value % 100 <= 39 or value % 100 == 0:
+ outword += "te"
+ elif value % 100 > 12 or value % 100 == 0:
+ outword += "ende"
+ return outword
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ vaerdte = (0, 1, 5, 6, 11, 12)
+ if value % 100 >= 30 and value % 100 <= 39 or value % 100 in vaerdte:
+ return str(value) + "te"
+ elif value % 100 == 2:
+ return str(value) + "en"
+ return str(value) + "ende"
+
+ def to_currency(self, val, longval=True):
+ if val // 100 == 1 or val == 1:
+ ret = self.to_splitnum(val, hightxt="kr", lowtxt="\xf8re",
+ jointxt="og", longval=longval)
+ return "en " + ret[3:]
+ return self.to_splitnum(val, hightxt="kr", lowtxt="\xf8re",
+ jointxt="og", longval=longval)
+
+ def to_year(self, val, longval=True):
+ if val == 1:
+ return 'en'
+ if not (val // 100) % 10:
+ return self.to_cardinal(val)
+ return self.to_splitnum(val, hightxt="hundrede", longval=longval)
diff --git a/fun_text_processing/num2words/num2words/lang_EL.py b/fun_text_processing/num2words/num2words/lang_EL.py
new file mode 100644
index 0000000..c08cb13
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_EL.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_EL(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word # Todo: check "-illion"
+
+ def setup(self):
+ super(Num2Word_EL, self).setup()
+ # lang_EU.Num2Word_EU.setup(self)
+
+ self.negword = "渭蔚委慰谓 "
+ self.pointword = "蟽畏渭蔚委慰"
+ self.exclude_title = ["魏伪喂", "蟽畏渭蔚委慰", "渭蔚委慰谓"]
+
+ self.high_numwords = [(1000000000, "苇谓伪 未喂蟽蔚魏伪蟿慰渭渭蠉蟻喂慰"),
+ (100000000, "蔚魏伪蟿蠈 蔚魏伪蟿慰渭渭蠉蟻喂伪"),
+ (10000000, "未苇魏伪 蔚魏伪蟿慰渭渭蠉蟻喂伪"),
+ (1000000, "苇谓伪 蔚魏伪蟿慰渭渭蠉蟻喂慰"),
+ (100000, "蔚魏伪蟿蠈 蠂喂位喂维未蔚蟼"),
+ (10000, "未苇魏伪 蠂喂位喂维未蔚蟼")]
+
+ self.mid_numwords = [(1000, "蠂委位喂伪"), (100, "蔚魏伪蟿蠈"),
+ (90, "蔚谓蔚谓萎谓蟿伪"), (80, "慰纬未蠈谓蟿伪"), (70, "蔚尾未慰渭萎谓蟿伪"),
+ (60, "蔚尉萎谓蟿伪"), (50, "蟺蔚谓萎谓蟿伪"), (40, "蟽伪蟻维谓蟿伪"),
+ (30, "蟿蟻喂维谓蟿伪")]
+
+ self.low_numwords = ["蔚委魏慰蟽喂",
+ "未蔚魏伪蔚谓谓苇伪",
+ "未蔚魏伪慰蠂蟿蠋",
+ "未蔚魏伪蔚蟺蟿维",
+ "未蔚魏伪苇尉喂",
+ "未蔚魏伪蟺苇谓蟿蔚",
+ "未蔚魏伪蟿苇蟽蟽蔚蟻伪",
+ "未蔚魏伪蟿蟻委伪",
+ "未蠋未蔚魏伪",
+ "苇谓蟿蔚魏伪",
+ "未苇魏伪",
+ "蔚谓谓喂维",
+ "慰蠂蟿蠋",
+ "蔚蟺蟿维",
+ "苇尉喂",
+ "蟺苇谓蟿蔚",
+ "蟿苇蟽蟽蔚蟻伪",
+ "蟿蟻委伪",
+ "未蠉慰",
+ "苇谓伪",
+ "渭畏未苇谓"]
+
+ self.ords = {"苇谓伪": "蟺蟻蠋蟿慰蟼",
+ "未蠉慰": "未蔚蠉蟿蔚蟻慰蟼",
+ "蟿蟻委伪": "蟿蟻委蟿慰蟼",
+ "蟿苇蟽蟽蔚蟻伪": "蟿苇蟿伪蟻蟿慰蟼",
+ "蟺苇谓蟿蔚": "蟺苇渭蟺蟿慰蟼",
+ "苇尉喂": "苇魏蟿慰蟼",
+ "蔚蟺蟿维": "苇尾未慰渭慰蟼",
+ "慰蠂蟿蠋": "蠈纬未慰慰蟼",
+ "蔚谓谓喂维": "苇谓伪蟿慰蟼",
+ "未苇魏伪": "未苇魏伪蟿慰蟼",
+ "苇谓蟿蔚魏伪": "蔚谓未苇魏伪蟿慰蟼",
+ "未蠋未蔚魏伪": "未蠅未苇魏伪蟿慰蟼"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value + "蟿慰蟼"
+ return ordinal_word
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '蟺.围' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "蔚魏伪蟿蠈"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_EN.py b/fun_text_processing/num2words/num2words/lang_EN.py
new file mode 100644
index 0000000..408c8cf
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_EN.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_EN(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word + "illion"
+
+ def setup(self):
+ super(Num2Word_EN, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "point"
+ self.exclude_title = ["and", "point", "minus"]
+
+ self.mid_numwords = [(1000, "thousand"), (100, "hundred"),
+ (90, "ninety"), (80, "eighty"), (70, "seventy"),
+ (60, "sixty"), (50, "fifty"), (40, "forty"),
+ (30, "thirty")]
+ self.low_numwords = ["twenty", "nineteen", "eighteen", "seventeen",
+ "sixteen", "fifteen", "fourteen", "thirteen",
+ "twelve", "eleven", "ten", "nine", "eight",
+ "seven", "six", "five", "four", "three", "two",
+ "one", "zero"]
+ self.ords = {"one": "first",
+ "two": "second",
+ "three": "third",
+ "four": "fourth",
+ "five": "fifth",
+ "six": "sixth",
+ "seven": "seventh",
+ "eight": "eighth",
+ "nine": "ninth",
+ "ten": "tenth",
+ "eleven": "eleventh",
+ "twelve": "twelfth"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s and %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s, %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("-")
+ lastword = lastwords[-1].lower()
+ try:
+ lastword = self.ords[lastword]
+ except KeyError:
+ if lastword[-1] == "y":
+ lastword = lastword[:-1] + "ie"
+ lastword += "th"
+ lastwords[-1] = self.title(lastword)
+ outwords[-1] = "-".join(lastwords)
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "hundred"
+ elif low < 10:
+ lowtext = "oh-%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_EN_IN.py b/fun_text_processing/num2words/num2words/lang_EN_IN.py
new file mode 100644
index 0000000..0736c27
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_EN_IN.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .lang_EN import Num2Word_EN
+
+
+class Num2Word_EN_IN(Num2Word_EN):
+ def set_high_numwords(self, high):
+ self.cards[10 ** 7] = "crore"
+ self.cards[10 ** 5] = "lakh"
diff --git a/fun_text_processing/num2words/num2words/lang_ES.py b/fun_text_processing/num2words/num2words/lang_ES.py
new file mode 100644
index 0000000..0c2c985
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_ES.py
@@ -0,0 +1,351 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+import math
+
+from .lang_EU import Num2Word_EU
+
+GENERIC_DOLLARS = ('dolar', 'd贸lares')
+GENERIC_CENTS = ('centavo', 'centavos')
+CURRENCIES_UNA = ('SLL', 'SEK', 'NOK', 'CZK', 'DKK', 'ISK',
+ 'SKK', 'GBP', 'CYP', 'EGP', 'FKP', 'GIP',
+ 'LBP', 'SDG', 'SHP', 'SSP', 'SYP', 'INR',
+ 'IDR', 'LKR', 'MUR', 'NPR', 'PKR', 'SCR',
+ 'ESP')
+
+
+class Num2Word_ES(Num2Word_EU):
+ CURRENCY_FORMS = {
+ 'EUR': (('euro', 'euros'), ('c茅ntimo', 'c茅ntimos')),
+ 'ESP': (('peseta', 'pesetas'), ('c茅ntimo', 'c茅ntimos')),
+ 'USD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'PEN': (('sol', 'soles'), ('c茅ntimo', 'c茅ntimos')),
+ 'CRC': (('col贸n', 'colones'), GENERIC_CENTS),
+ 'AUD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'CAD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'GBP': (('libra', 'libras'), ('penny', 'pence')),
+ 'RUB': (('rublo', 'rublos'), ('kopeyka', 'kopeykas')),
+ 'SEK': (('corona', 'coronas'), ('枚re', '枚re')),
+ 'NOK': (('corona', 'coronas'), ('酶re', '酶re')),
+ 'PLN': (('zloty', 'zlotys'), ('grosz', 'groszy')),
+ 'MXN': (('peso', 'pesos'), GENERIC_CENTS),
+ 'RON': (('leu', 'leus'), ('ban', 'bani')),
+ 'INR': (('rupia', 'rupias'), ('paisa', 'paisas')),
+ 'HUF': (('flor铆n', 'florines'), ('fill茅r', 'fill茅r')),
+ 'FRF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'CNY': (('yuan', 'yuanes'), ('fen', 'jiaos')),
+ 'CZK': (('corona', 'coronas'), ('hal茅艡', 'hal茅艡')),
+ 'NIO': (('c贸rdoba', 'c贸rdobas'), GENERIC_CENTS),
+ 'VES': (('bol铆var', 'bol铆vares'), ('c茅ntimo', 'c茅ntimos')),
+ 'BRL': (('real', 'reales'), GENERIC_CENTS),
+ 'CHF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'JPY': (('yen', 'yenes'), ('sen', 'sen')),
+ 'KRW': (('won', 'wones'), ('jeon', 'jeon')),
+ 'KPW': (('won', 'wones'), ('chon', 'chon')),
+ 'TRY': (('lira', 'liras'), ('kuru艧', 'kuru艧')),
+ 'ZAR': (('rand', 'rands'), ('c茅ntimo', 'c茅ntimos')),
+ 'KZT': (('tenge', 'tenges'), ('t茂谋n', 't茂谋n')),
+ 'UAH': (('hryvnia', 'hryvnias'), ('kopiyka', 'kopiykas')),
+ 'THB': (('baht', 'bahts'), ('satang', 'satang')),
+ 'AED': (('dirham', 'dirhams'), ('fils', 'fils')),
+ 'AFN': (('afghani', 'afghanis'), ('pul', 'puls')),
+ 'ALL': (('lek ', 'leke'), ('qindark毛', 'qindarka')),
+ 'AMD': (('dram', 'drams'), ('luma', 'lumas')),
+ 'ANG': (('flor铆n', 'florines'), GENERIC_CENTS),
+ 'AOA': (('kwanza', 'kwanzas'), ('c茅ntimo', 'c茅ntimos')),
+ 'ARS': (('peso', 'pesos'), GENERIC_CENTS),
+ 'AWG': (('flor铆n', 'florines'), GENERIC_CENTS),
+ 'AZN': (('manat', 'manat'), ('q蓹pik', 'q蓹pik')),
+ 'BBD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BDT': (('taka', 'takas'), ('paisa', 'paisas')),
+ 'BGN': (('lev', 'leva'), ('stotinka', 'stotinki')),
+ 'BHD': (('dinar', 'dinares'), ('fils', 'fils')),
+ 'BIF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'BMD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BND': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BOB': (('boliviano', 'bolivianos'), GENERIC_CENTS),
+ 'BSD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BTN': (('ngultrum', 'ngultrum'), ('chetrum', 'chetrum')),
+ 'BWP': (('pula', 'pulas'), ('thebe', 'thebes')),
+ 'BYN': (('rublo', 'rublos'), ('k贸pek', 'k贸peks')),
+ 'BYR': (('rublo', 'rublos'), ('k贸pek', 'k贸peks')),
+ 'BZD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'CDF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'CLP': (('peso', 'pesos'), GENERIC_CENTS),
+ 'COP': (('peso', 'pesos'), GENERIC_CENTS),
+ 'CUP': (('peso', 'pesos'), GENERIC_CENTS),
+ 'CVE': (('escudo', 'escudos'), GENERIC_CENTS),
+ 'CYP': (('libra', 'libras'), ('c茅ntimo', 'c茅ntimos')),
+ 'DJF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'DKK': (('corona', 'coronas'), ('酶re', '酶re')),
+ 'DOP': (('peso', 'pesos'), GENERIC_CENTS),
+ 'DZD': (('dinar', 'dinares'), ('c茅ntimo', 'c茅ntimos')),
+ 'ECS': (('sucre', 'sucres'), GENERIC_CENTS),
+ 'EGP': (('libra', 'libras'), ('piastra', 'piastras')),
+ 'ERN': (('nakfa', 'nakfas'), ('c茅ntimo', 'c茅ntimos')),
+ 'ETB': (('birr', 'birrs'), ('c茅ntimo', 'c茅ntimos')),
+ 'FJD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'FKP': (('libra', 'libras'), ('penny', 'peniques')),
+ 'GEL': (('lari', 'laris'), ('tetri', 'tetris')),
+ 'GHS': (('cedi', 'cedis'), ('pesewa', 'pesewas')),
+ 'GIP': (('libra', 'libras'), ('penique', 'peniques')),
+ 'GMD': (('dalasi', 'dalasis'), ('butut', 'bututs')),
+ 'GNF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'GTQ': (('quetzal', 'quetzales'), GENERIC_CENTS),
+ 'GYD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'HKD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'HNL': (('lempira', 'lempiras'), GENERIC_CENTS),
+ 'HRK': (('kuna', 'kunas'), ('lipa', 'lipas')),
+ 'HTG': (('gourde', 'gourdes'), ('c茅ntimo', 'c茅ntimos')),
+ 'IDR': (('rupia', 'rupias'), ('c茅ntimo', 'c茅ntimos')),
+ 'ILS': (('s茅quel', 's茅queles'), ('agora', 'agoras')),
+ 'IQD': (('dinar', 'dinares'), ('fils', 'fils')),
+ 'IRR': (('rial', 'riales'), ('dinar', 'dinares')),
+ 'ISK': (('corona', 'coronas'), ('eyrir', 'aurar')),
+ 'ITL': (('lira', 'liras'), ('c茅ntimo', 'c茅ntimos')),
+ 'JMD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'JOD': (('dinar', 'dinares'), ('piastra', 'piastras')),
+ 'KES': (('chel铆n', 'chelines'), ('c茅ntimo', 'c茅ntimos')),
+ 'KGS': (('som', 'som'), ('tyiyn', 'tyiyn')),
+ 'KHR': (('riel', 'rieles'), ('c茅ntimo', 'c茅ntimos')),
+ 'KMF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'KWD': (('dinar', 'dinares'), ('fils', 'fils')),
+ 'KYD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'LAK': (('kip', 'kips'), ('att', 'att')),
+ 'LBP': (('libra', 'libras'), ('piastra', 'piastras')),
+ 'LKR': (('rupia', 'rupias'), ('c茅ntimo', 'c茅ntimos')),
+ 'LRD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'LSL': (('loti', 'lotis'), ('c茅ntimo', 'c茅ntimos')),
+ 'LTL': (('lita', 'litas'), ('c茅ntimo', 'c茅ntimos')),
+ 'LVL': (('lat', 'lats'), ('c茅ntimo', 'c茅ntimos')),
+ 'LYD': (('dinar', 'dinares'), ('d铆rham', 'd铆rhams')),
+ 'MAD': (('d铆rham', 'dirhams'), ('c茅ntimo', 'c茅ntimos')),
+ 'MDL': (('leu', 'lei'), ('ban', 'bani')),
+ 'MGA': (('ariary', 'ariaris'), ('iraimbilanja', 'iraimbilanja')),
+ 'MKD': (('denar', 'denares'), ('deni', 'denis')),
+ 'MMK': (('kiat', 'kiats'), ('pya', 'pyas')),
+ 'MNT': (('tugrik', 'tugriks'), ('m枚ng枚', 'm枚ng枚')),
+ 'MOP': (('pataca', 'patacas'), ('avo', 'avos')),
+ 'MRO': (('ouguiya', 'ouguiyas'), ('khoums', 'khoums')),
+ 'MRU': (('ouguiya', 'ouguiyas'), ('khoums', 'khoums')),
+ 'MUR': (('rupia', 'rupias'), ('c茅ntimo', 'c茅ntimos')),
+ 'MVR': (('rufiyaa', 'rufiyaas'), ('laari', 'laari')),
+ 'MWK': (('kuacha', 'kuachas'), ('tambala', 'tambalas')),
+ 'MYR': (('ringgit', 'ringgit'), ('c茅ntimo', 'c茅ntimos')),
+ 'MZN': (('metical', 'metical'), GENERIC_CENTS),
+ 'NAD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'NGN': (('naira', 'nairas'), ('kobo', 'kobo')),
+ 'NPR': (('rupia', 'rupias'), ('paisa', 'paisas')),
+ 'NZD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'OMR': (('rial', 'riales'), ('baisa', 'baisa')),
+ 'PAB': (('balboa', 'balboas'), ('cent茅simo', 'cent茅simos')),
+ 'PGK': (('kina', 'kinas'), ('toea', 'toea')),
+ 'PHP': (('peso', 'pesos'), GENERIC_CENTS),
+ 'PKR': (('rupia', 'rupias'), ('paisa', 'paisas')),
+ 'PLZ': (('zloty', 'zlotys'), ('grosz', 'groszy')),
+ 'PYG': (('guaran铆', 'guaran铆s'), ('c茅ntimo', 'c茅ntimos')),
+ 'QAR': (('rial', 'riales'), ('d铆rham', 'd铆rhams')),
+ 'QTQ': (('quetzal', 'quetzales'), GENERIC_CENTS),
+ 'RSD': (('dinar', 'dinares'), ('para', 'para')),
+ 'RUR': (('rublo', 'rublos'), ('kopek', 'kopeks')),
+ 'RWF': (('franco', 'francos'), ('c茅ntimo', 'c茅ntimos')),
+ 'SAR': (('riyal', 'riales'), ('halala', 'halalas')),
+ 'SBD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'SCR': (('rupia', 'rupias'), ('c茅ntimo', 'c茅ntimos')),
+ 'SDG': (('libra', 'libras'), ('piastra', 'piastras')),
+ 'SGD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'SHP': (('libra', 'libras'), ('penny', 'peniques')),
+ 'SKK': (('corona', 'coronas'), ('halier', 'haliers')),
+ 'SLL': (('leona', 'leonas'), ('c茅ntimo', 'c茅ntimos')),
+ 'SRD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'SSP': (('libra', 'libras'), ('piastra', 'piastras')),
+ 'STD': (('dobra', 'dobras'), ('c茅ntimo', 'c茅ntimos')),
+ 'SVC': (('col贸n', 'colones'), GENERIC_CENTS),
+ 'SYP': (('libra', 'libras'), ('piastra', 'piastras')),
+ 'SZL': (('lilangeni', 'emalangeni'), ('c茅ntimo', 'c茅ntimos')),
+ 'TJS': (('somoni', 'somonis'), ('dirame', 'dirames')),
+ 'TMT': (('manat', 'manat'), ('tenge', 'tenge')),
+ 'TND': (('dinar', 'dinares'), ('mil茅simo', 'mil茅simos')),
+ 'TOP': (('paanga', 'paangas'), ('c茅ntimo', 'c茅ntimos')),
+ 'TTD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'TWD': (('nuevo d贸lar', 'nuevos dolares'), ('c茅ntimo', 'c茅ntimos')),
+ 'TZS': (('chel铆n', 'chelines'), ('c茅ntimo', 'c茅ntimos')),
+ 'UAG': (('hryvnia', 'hryvnias'), ('kopiyka', 'kopiykas')),
+ 'UGX': (('chel铆n', 'chelines'), ('c茅ntimo', 'c茅ntimos')),
+ 'UYU': (('peso', 'pesos'), ('cent茅simo', 'cent茅simos')),
+ 'UZS': (('sum', 'sum'), ('tiyin', 'tiyin')),
+ 'VEF': (('bol铆var fuerte', 'bol铆vares fuertes'),
+ ('c茅ntimo', 'c茅ntimos')),
+ 'VND': (('dong', 'dongs'), ('xu', 'xu')),
+ 'VUV': (('vatu', 'vatu'), ('nenhum', 'nenhum')),
+ 'WST': (('tala', 'tala'), GENERIC_CENTS),
+ 'XAF': (('franco CFA', 'francos CFA'), ('c茅ntimo', 'c茅ntimos')),
+ 'XCD': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'XOF': (('franco CFA', 'francos CFA'), ('c茅ntimo', 'c茅ntimos')),
+ 'XPF': (('franco CFP', 'francos CFP'), ('c茅ntimo', 'c茅ntimos')),
+ 'YER': (('rial', 'riales'), ('fils', 'fils')),
+ 'YUM': (('dinar', 'dinares'), ('para', 'para')),
+ 'ZMW': (('kwacha', 'kwachas'), ('ngwee', 'ngwee')),
+ 'ZRZ': (('zaire', 'zaires'), ('likuta', 'makuta')),
+ 'ZWL': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ 'ZWL': (GENERIC_DOLLARS, ('c茅ntimo', 'c茅ntimos')),
+ }
+
+ # //CHECK: Is this sufficient??
+ GIGA_SUFFIX = None
+ MEGA_SUFFIX = "ill贸n"
+
+ def setup(self):
+ lows = ["cuatr", "tr", "b", "m"]
+ self.high_numwords = self.gen_high_numwords([], [], lows)
+ self.negword = "menos "
+ self.pointword = "punto"
+ self.errmsg_nonnum = "Solo n煤meros pueden ser convertidos a palabras."
+ self.errmsg_toobig = (
+ "Numero muy grande para ser convertido a palabras."
+ )
+ self.gender_stem = "o"
+ self.exclude_title = ["y", "menos", "punto"]
+ self.mid_numwords = [(1000, "mil"), (100, "cien"), (90, "noventa"),
+ (80, "ochenta"), (70, "setenta"), (60, "sesenta"),
+ (50, "cincuenta"), (40, "cuarenta"),
+ (30, "treinta")]
+ self.low_numwords = ["veintinueve", "veintiocho", "veintisiete",
+ "veintis茅is", "veinticinco", "veinticuatro",
+ "veintitr茅s", "veintid贸s", "veintiuno",
+ "veinte", "diecinueve", "dieciocho", "diecisiete",
+ "dieciseis", "quince", "catorce", "trece", "doce",
+ "once", "diez", "nueve", "ocho", "siete", "seis",
+ "cinco", "cuatro", "tres", "dos", "uno", "cero"]
+ self.ords = {1: "primer",
+ 2: "segund",
+ 3: "tercer",
+ 4: "cuart",
+ 5: "quint",
+ 6: "sext",
+ 7: "s茅ptim",
+ 8: "octav",
+ 9: "noven",
+ 10: "d茅cim",
+ 20: "vig茅sim",
+ 30: "trig茅sim",
+ 40: "quadrag茅sim",
+ 50: "quincuag茅sim",
+ 60: "sexag茅sim",
+ 70: "septuag茅sim",
+ 80: "octog茅sim",
+ 90: "nonag茅sim",
+ 100: "cent茅sim",
+ 200: "ducent茅sim",
+ 300: "tricent茅sim",
+ 400: "cuadrigent茅sim",
+ 500: "quingent茅sim",
+ 600: "sexcent茅sim",
+ 700: "septigent茅sim",
+ 800: "octigent茅sim",
+ 900: "noningent茅sim",
+ 1e3: "mil茅sim",
+ 1e6: "millon茅sim",
+ 1e9: "billon茅sim",
+ 1e12: "trillon茅sim",
+ 1e15: "cuadrillon茅sim"}
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+ ctext = "un"
+ elif cnum == 100 and not nnum % 1000 == 0:
+ ctext += "t" + self.gender_stem
+
+ if nnum < cnum:
+ if cnum < 100:
+ return "%s y %s" % (ctext, ntext), cnum + nnum
+ return "%s %s" % (ctext, ntext), cnum + nnum
+ elif (not nnum % 1000000) and cnum > 1:
+ ntext = ntext[:-3] + "lones"
+
+ if nnum == 100:
+ if cnum == 5:
+ ctext = "quinien"
+ ntext = ""
+ elif cnum == 7:
+ ctext = "sete"
+ elif cnum == 9:
+ ctext = "nove"
+ ntext += "t" + self.gender_stem + "s"
+ else:
+ ntext = " " + ntext
+
+ return (ctext + ntext, cnum * nnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ if value == 0:
+ text = ""
+ elif value <= 10:
+ text = "%s%s" % (self.ords[value], self.gender_stem)
+ elif value <= 12:
+ text = (
+ "%s%s%s" % (self.ords[10], self.gender_stem,
+ self.to_ordinal(value - 10))
+ )
+ elif value <= 100:
+ dec = (value // 10) * 10
+ text = (
+ "%s%s %s" % (self.ords[dec], self.gender_stem,
+ self.to_ordinal(value - dec))
+ )
+ elif value <= 1e3:
+ cen = (value // 100) * 100
+ text = (
+ "%s%s %s" % (self.ords[cen], self.gender_stem,
+ self.to_ordinal(value - cen))
+ )
+ elif value < 1e18:
+ # Round down to the nearest 1e(3n)
+ # dec contains the following:
+ # [ 1e3, 1e6): 1e3
+ # [ 1e6, 1e9): 1e6
+ # [ 1e9, 1e12): 1e9
+ # [1e12, 1e15): 1e12
+ # [1e15, 1e18): 1e15
+ dec = 1000 ** int(math.log(int(value), 1000))
+
+ # Split the parts before and after the word for 'dec'
+ # eg (12, 345) = divmod(12_345, 1_000)
+ high_part, low_part = divmod(value, dec)
+
+ cardinal = self.to_cardinal(high_part) if high_part != 1 else ""
+ text = (
+ "%s%s%s %s" % (cardinal, self.ords[dec], self.gender_stem,
+ self.to_ordinal(low_part))
+ )
+ else:
+ text = self.to_cardinal(value)
+ return text.strip()
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, "潞" if self.gender_stem == 'o' else "陋")
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=' con',
+ adjective=False):
+ result = super(Num2Word_ES, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ # Handle exception, in spanish is "un euro" and not "uno euro"
+ # except in this currencies: leona, corona,
+ # libra, rupia, lempira, peseta, is 'una'
+ # but only when it's first word, otherwise
+ # it's replaced in others words like 'veintiun'
+ if currency in CURRENCIES_UNA:
+ list_result = result.split(" ")
+ if list_result[0] == "uno":
+ list_result[0] = list_result[0].replace("uno", "una")
+ result = " ".join(list_result)
+ result = result.replace("uno", "un")
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_ES_CO.py b/fun_text_processing/num2words/num2words/lang_ES_CO.py
new file mode 100644
index 0000000..f27b17b
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_ES_CO.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_ES import Num2Word_ES
+
+
+class Num2Word_ES_CO(Num2Word_ES):
+
+ def to_currency(self, val, longval=True, old=False):
+ result = self.to_splitnum(val, hightxt="peso/s", lowtxt="centavo/s",
+ divisor=1, jointxt="y", longval=longval)
+ # Handle exception, in spanish is "un euro" and not "uno euro"
+ return result.replace("uno", "un")
diff --git a/fun_text_processing/num2words/num2words/lang_ES_NI.py b/fun_text_processing/num2words/num2words/lang_ES_NI.py
new file mode 100644
index 0000000..5537eab
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_ES_NI.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_ES import Num2Word_ES
+
+
+class Num2Word_ES_NI(Num2Word_ES):
+ CURRENCY_FORMS = {
+ 'NIO': (('c贸rdoba', 'c贸rdobas'), ('centavo', 'centavos')),
+ }
+
+ def to_currency(self, val, currency='NIO', cents=True, separator=' con',
+ adjective=False):
+ result = super(Num2Word_ES, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ return result.replace("uno", "un")
diff --git a/fun_text_processing/num2words/num2words/lang_ES_VE.py b/fun_text_processing/num2words/num2words/lang_ES_VE.py
new file mode 100644
index 0000000..d73acc2
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_ES_VE.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_ES import Num2Word_ES
+
+
+class Num2Word_ES_VE(Num2Word_ES):
+
+ def to_currency(self, val, longval=True, old=False):
+ hightxt = "bol铆var/es" if old else "bol铆var/es fuerte/s"
+ result = self.to_splitnum(
+ val, hightxt=hightxt, lowtxt="centavo/s",
+ divisor=1, jointxt="y", longval=longval
+ )
+ # Handle exception, in spanish is "un euro" and not "uno euro"
+ return result.replace("uno", "un")
diff --git a/fun_text_processing/num2words/num2words/lang_EU.py b/fun_text_processing/num2words/num2words/lang_EU.py
new file mode 100644
index 0000000..4e64921
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_EU.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+
+GENERIC_DOLLARS = ('dollar', 'dollars')
+GENERIC_CENTS = ('cent', 'cents')
+
+
+class Num2Word_EU(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'AUD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'CAD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ # repalced by EUR
+ 'EEK': (('kroon', 'kroons'), ('sent', 'senti')),
+ 'EUR': (('euro', 'euro'), GENERIC_CENTS),
+ 'GBP': (('pound sterling', 'pounds sterling'), ('penny', 'pence')),
+ # replaced by EUR
+ 'LTL': (('litas', 'litas'), GENERIC_CENTS),
+ # replaced by EUR
+ 'LVL': (('lat', 'lats'), ('santim', 'santims')),
+ 'USD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'RUB': (('rouble', 'roubles'), ('kopek', 'kopeks')),
+ 'SEK': (('krona', 'kronor'), ('枚re', '枚re')),
+ 'NOK': (('krone', 'kroner'), ('酶re', '酶re')),
+ 'PLN': (('zloty', 'zlotys', 'zlotu'), ('grosz', 'groszy')),
+ 'MXN': (('peso', 'pesos'), GENERIC_CENTS),
+ 'RON': (('leu', 'lei', 'de lei'), ('ban', 'bani', 'de bani')),
+ 'INR': (('rupee', 'rupees'), ('paisa', 'paise')),
+ 'HUF': (('forint', 'forint'), ('fill茅r', 'fill茅r'))
+ }
+
+ CURRENCY_ADJECTIVES = {
+ 'AUD': 'Australian',
+ 'CAD': 'Canadian',
+ 'EEK': 'Estonian',
+ 'USD': 'US',
+ 'RUB': 'Russian',
+ 'NOK': 'Norwegian',
+ 'MXN': 'Mexican',
+ 'RON': 'Romanian',
+ 'INR': 'Indian',
+ 'HUF': 'Hungarian'
+ }
+
+ GIGA_SUFFIX = "illiard"
+ MEGA_SUFFIX = "illion"
+
+ def set_high_numwords(self, high):
+ cap = 3 + 6 * len(high)
+
+ for word, n in zip(high, range(cap, 3, -6)):
+ if self.GIGA_SUFFIX:
+ self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ if self.MEGA_SUFFIX:
+ self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+
+ def gen_high_numwords(self, units, tens, lows):
+ out = [u + t for t in tens for u in units]
+ out.reverse()
+ return out + lows
+
+ def pluralize(self, n, forms):
+ form = 0 if n == 1 else 1
+ return forms[form]
+
+ def setup(self):
+ lows = ["non", "oct", "sept", "sext", "quint", "quadr", "tr", "b", "m"]
+ units = ["", "un", "duo", "tre", "quattuor", "quin", "sex", "sept",
+ "octo", "novem"]
+ tens = ["dec", "vigint", "trigint", "quadragint", "quinquagint",
+ "sexagint", "septuagint", "octogint", "nonagint"]
+ self.high_numwords = ["cent"] + self.gen_high_numwords(units, tens,
+ lows)
diff --git a/fun_text_processing/num2words/num2words/lang_FI.py b/fun_text_processing/num2words/num2words/lang_FI.py
new file mode 100644
index 0000000..86f9e09
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_FI.py
@@ -0,0 +1,721 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from collections import OrderedDict
+
+from . import lang_EU
+
+GENERIC_CENTS = ('sentti', 'sentti盲')
+GENERIC_CENTAVOS = ('centavo', 'centavoa')
+
+# grammatical cases
+NOM = 10 # nominative: the dictionary form
+GEN = 11 # genitive: ~of/'s
+ACC = 12 # accusative: not used; either nominative or genitive
+PTV = 13 # partitive: as an object
+# locative cases (internal)
+INE = 14 # inessive: in
+ELA = 15 # elative: from/out of
+ILL = 16 # illative: into
+# locative cases (external)
+ADE = 17 # adessive: at/on
+ABL = 18 # ablative: from (after being at/on, not in)
+ALL = 19 # allative: to
+# essive
+ESS = 20 # essive: as (in the role of)
+TRANSL = 21 # translative: to (the role of; being sth)
+# rare
+INSTRUC = 22 # instructive: with (plural is the same as singular)
+ABE = 23 # abessive: without
+COM = 24 # comitative: together with (plural = singular)
+
+NAME_TO_CASE = {
+ 'nominative': NOM,
+ 'genitive': GEN,
+ 'accusative': ACC,
+ 'partitive': PTV,
+ 'inessive': INE,
+ 'elative': ELA,
+ 'illative': ILL,
+ 'adessive': ADE,
+ 'ablative': ABL,
+ 'allative': ALL,
+ 'essive': ESS,
+ 'translative': TRANSL,
+ 'instructive': INSTRUC,
+ 'abessive': ABE,
+ 'comitative': COM,
+}
+
+# https://en.wikibooks.org/wiki/Finnish/Grammar-Vowel_harmony
+BACK_TO_FRONT = {
+ 'a': '盲',
+ 'o': '枚',
+ 'u': 'y',
+}
+
+# https://en.wiktionary.org/wiki/Appendix:Finnish_nominal_inflection
+# CASE: (SINGULAR_SUFFIX+, PLURAL_SUFFIX+)
+KOTUS_TYPE = {
+
+ # Kotus type 5/risti, no gradation
+ 5: {
+ # grammatical
+ NOM: ('i', 'it'),
+ GEN: ('in', 'ien'),
+ PTV: ('ia', 'eja'),
+ # locative, internal
+ INE: ('issa', 'eissa'),
+ ELA: ('ista', 'eista'),
+ ILL: ('iin', 'eihin'),
+ # locative, external
+ ADE: ('illa', 'eilla'),
+ ABL: ('ilta', 'eilta'),
+ ALL: ('ille', 'eille'),
+ # essive
+ ESS: ('ina', 'eina'),
+ TRANSL: ('iksi', 'eiksi'),
+ # rare
+ INSTRUC: ('ein', 'ein'),
+ ABE: ('itta', 'eitta'),
+ COM: ('eine', 'eine'), # works better
+ },
+
+ # Kotus type 7/ovi, no gradation
+ 7: {
+ # grammatical
+ NOM: ('i', 'et'),
+ GEN: ('en', 'ien'),
+ PTV: ('ea', 'ia'),
+ # locative, internal
+ INE: ('essa', 'issa'),
+ ELA: ('esta', 'ista'),
+ ILL: ('een', 'iin'),
+ # locative, external
+ ADE: ('ella', 'illa'),
+ ABL: ('elta', 'ilta'),
+ ALL: ('elle', 'ille'),
+ # essive
+ ESS: ('ena', 'ina'),
+ TRANSL: ('eksi', 'iksi'),
+ # rare
+ INSTRUC: ('in', 'in'),
+ ABE: ('etta', 'itta'),
+ COM: ('ine', 'ine'), # works better
+ },
+
+ # Kotus type 8/nalle, no gradation
+ 8: {
+ # grammatical
+ NOM: ('e', 'et'),
+ GEN: ('en', ('ejen', 'ein')),
+ PTV: ('ea', 'eja'),
+ # locative, internal
+ INE: ('essa', 'eissa'),
+ ELA: ('esta', 'eista'),
+ ILL: ('een', 'eihin'),
+ # locative, external
+ ADE: ('ella', 'eilla'),
+ ABL: ('elta', 'eilta'),
+ ALL: ('elle', 'eille'),
+ # essive
+ ESS: ('ena', 'eina'),
+ TRANSL: ('eksi', 'eiksi'),
+ # rare
+ INSTRUC: ('ein', 'ein'),
+ ABE: ('etta', 'eitta'),
+ COM: ('eine', 'eine'), # works better
+ },
+
+ # Kotus type 9/kala, t-d gradation (sata)
+ 109: {
+ # grammatical
+ NOM: ('ta', 'dat'),
+ GEN: ('dan', ('tojen', 'tain')),
+ PTV: ('taa', 'toja'),
+ # locative, internal
+ INE: ('dassa', 'doissa'),
+ ELA: ('dasta', 'doista'),
+ ILL: ('taan', 'toihin'),
+ # locative, external
+ ADE: ('dalla', 'doilla'),
+ ABL: ('dalta', 'doilta'),
+ ALL: ('dalle', 'doille'),
+ # essive
+ ESS: ('tana', 'toina'),
+ TRANSL: ('daksi', 'doiksi'),
+ # rare
+ INSTRUC: ('doin', 'doin'),
+ ABE: ('datta', 'doitta'),
+ COM: ('toine', 'toine'), # works better
+ },
+
+ # Kotus type 10/koira, no gradation
+ 10: {
+ # grammatical
+ NOM: ('a', 'at'),
+ GEN: ('an', ('ien', 'ain')),
+ PTV: ('aa', 'ia'),
+ # locative, internal
+ INE: ('assa', 'issa'),
+ ELA: ('asta', 'ista'),
+ ILL: ('aan', 'iin'),
+ # locative, external
+ ADE: ('alla', 'illa'),
+ ABL: ('alta', 'ilta'),
+ ALL: ('alle', 'ille'),
+ # essive
+ ESS: ('ana', 'ina'),
+ TRANSL: ('aksi', 'iksi'),
+ # rare
+ INSTRUC: ('in', 'in'),
+ ABE: ('atta', 'itta'),
+ COM: ('ine', 'ine'), # works better
+ },
+
+ # Kotus type 27/k盲si, t-d gradation
+ 27: {
+ # grammatical
+ NOM: ('si', 'det'),
+ GEN: ('den', ('sien', 'tten')),
+ PTV: ('tta', 'sia'),
+ # locative, internal
+ INE: ('dessa', 'sissa'),
+ ELA: ('desta', 'sista'),
+ ILL: ('teen', 'siin'),
+ # locative, external
+ ADE: ('della', 'silla'),
+ ABL: ('delta', 'silta'),
+ ALL: ('delle', 'sille'),
+ # essive
+ ESS: ('tena', 'sina'),
+ TRANSL: ('deksi', 'siksi'),
+ # rare
+ INSTRUC: ('sin', 'sin'),
+ ABE: ('detta', 'sitta'),
+ COM: ('sine', 'sine'), # works better
+ },
+
+ # Kotus type 31/kaksi, t-d gradation
+ 31: {
+ # grammatical
+ NOM: ('ksi', 'hdet'),
+ GEN: ('hden', 'ksien'),
+ PTV: ('hta', 'ksia'),
+ # locative, internal
+ INE: ('hdessa', 'ksissa'),
+ ELA: ('hdesta', 'ksista'),
+ ILL: ('hteen', 'ksiin'),
+ # locative, external
+ ADE: ('hdella', 'ksilla'),
+ ABL: ('hdelta', 'ksilta'),
+ ALL: ('hdelle', 'ksille'),
+ # essive
+ ESS: ('htena', 'ksina'),
+ TRANSL: ('hdeksi', 'ksiksi'),
+ # rare
+ INSTRUC: ('ksin', 'ksin'),
+ ABE: ('hdetta', 'ksitta'),
+ COM: ('ksine', 'ksine'), # works better
+ },
+
+ # Kotus type 32/sisar, no gradation
+ 32: {
+ # grammatical
+ NOM: ('', 'et'),
+ GEN: ('en', ('ien', 'ten')),
+ PTV: ('ta', 'ia'),
+ # locative, internal
+ INE: ('essa', 'issa'),
+ ELA: ('esta', 'ista'),
+ ILL: ('een', 'iin'),
+ # locative, external
+ ADE: ('ella', 'illa'),
+ ABL: ('elta', 'ilta'),
+ ALL: ('elle', 'ille'),
+ # essive
+ ESS: ('ena', 'ina'),
+ TRANSL: ('eksi', 'iksi'),
+ # rare
+ INSTRUC: ('in', 'in'),
+ ABE: ('etta', 'itta'),
+ COM: ('ine', 'ine'), # works better
+ },
+
+ # Kotus type 38/nainen, no gradation
+ 38: {
+ # grammatical
+ NOM: ('nen', 'set'),
+ GEN: ('sen', ('sten', 'sien')),
+ PTV: ('sta', 'sia'),
+ # locative, internal
+ INE: ('sessa', 'sissa'),
+ ELA: ('sesta', 'sista'),
+ ILL: ('seen', 'siin'),
+ # locative, external
+ ADE: ('sella', 'silla'),
+ ABL: ('selta', 'silta'),
+ ALL: ('selle', 'sille'),
+ # essive
+ ESS: ('sena', 'sina'),
+ TRANSL: ('seksi', 'siksi'),
+ # rare
+ INSTRUC: ('sin', 'sin'),
+ ABE: ('setta', 'sitta'),
+ COM: ('sine', 'sine'), # works better
+ },
+
+ # Kotus type 45/kahdeksas, nt-nn gradation
+ 45: {
+ # grammatical
+ NOM: ('s', 'nnet'),
+ GEN: ('nnen', 'nsien'),
+ PTV: ('tta', 'nsia'),
+ # locative, internal
+ INE: ('nnessa', 'nsissa'),
+ ELA: ('nnesta', 'nsista'),
+ ILL: ('nteen', 'nsiin'),
+ # locative, external
+ ADE: ('nnella', 'nsilla'),
+ ABL: ('nnelta', 'nsilta'),
+ ALL: ('nnelle', 'nsille'),
+ # essive
+ ESS: ('ntena', 'nsina'),
+ TRANSL: ('nneksi', 'nsiksi'),
+ # rare
+ INSTRUC: ('nsin', 'nsin'),
+ ABE: ('nnetta', 'nsitta'),
+ COM: ('nsine', 'nsine'), # works better
+ },
+
+ # Kotus type 46/tuhat, nt-nn gradation
+ 46: {
+ # grammatical
+ NOM: ('t', 'nnet'),
+ GEN: ('nnen', ('nsien', 'nten')),
+ PTV: ('tta', 'nsia'),
+ # locative, internal
+ INE: ('nnessa', 'nsissa'),
+ ELA: ('nnesta', 'nsista'),
+ ILL: ('nteen', 'nsiin'),
+ # locative, external
+ ADE: ('nnella', 'nsilla'),
+ ABL: ('nnelta', 'nsilta'),
+ ALL: ('nnelle', 'nsille'),
+ # essive
+ ESS: ('ntena', 'nsina'),
+ TRANSL: ('nneksi', 'nsiksi'),
+ # rare
+ INSTRUC: ('nsin', 'nsin'),
+ ABE: ('nnetta', 'nsitta'),
+ COM: ('nsine', 'nsine'), # works better
+ },
+}
+
+# kolme
+KOTUS_TYPE[108] = {
+ c: (KOTUS_TYPE[8][c][0], KOTUS_TYPE[7][c][1])
+ for c in KOTUS_TYPE[8]
+}
+KOTUS_TYPE[108][INSTRUC] = ('en', 'in')
+KOTUS_TYPE[108][ABE] = ('etta', 'itta')
+KOTUS_TYPE[108][COM] = ('ine', 'ine')
+
+# seitsem盲n, kahdeksan, yhdeks盲n
+KOTUS_TYPE[110] = KOTUS_TYPE[10].copy()
+KOTUS_TYPE[110][NOM] = ('an', 'at')
+
+# kymmenen
+KOTUS_TYPE[132] = KOTUS_TYPE[32].copy()
+KOTUS_TYPE[132][NOM] = ('en', 'et')
+
+
+def inflect(parts, options):
+ if not isinstance(parts, list):
+ parts = [parts]
+
+ out = ''
+ for part in parts:
+ # part is plain text, concat and continue
+ if not isinstance(part, tuple):
+ out += part
+ continue
+ # predefined case (kaksikymment盲, ...)
+ tmp_case = options.case
+ if len(part) == 3:
+ # override singular nominative only
+ if options.case == NOM and not options.plural:
+ tmp_case = part[2]
+ part = part[:2]
+ # stem and suffix
+ stem, kotus_type = part
+ suffix = KOTUS_TYPE[kotus_type][tmp_case][options.plural]
+ # many choices, choose preferred or first
+ if isinstance(suffix, tuple):
+ common = set(suffix) & set(options.prefer or set())
+ if len(common) == 1:
+ suffix = common.pop()
+ else:
+ suffix = suffix[0]
+ # apply vowel harmony
+ if not set(BACK_TO_FRONT) & set(stem):
+ for back, front in BACK_TO_FRONT.items():
+ suffix = suffix.replace(back, front)
+ # concat
+ out += stem + suffix
+
+ return out
+
+
+class Options(object):
+ def __init__(self, ordinal, case, plural, prefer):
+ self.ordinal = ordinal
+ self.case = case
+ self.plural = plural
+ self.prefer = prefer
+
+ def variation(self, ordinal=None, case=None, plural=None, prefer=None):
+ return Options(
+ ordinal if ordinal is not None else self.ordinal,
+ case if case is not None else self.case,
+ plural if plural is not None else self.plural,
+ prefer if prefer is not None else self.prefer,
+ )
+
+
+class Num2Word_FI(lang_EU.Num2Word_EU):
+ CURRENCY_FORMS = {
+ 'BRL': (('real', 'realia'), GENERIC_CENTAVOS),
+ 'CHF': (('frangi', 'frangia'), ('rappen', 'rappenia')),
+ 'CNY': (('juan', 'juania'), ('fen', 'feni盲')),
+ 'EUR': (('euro', 'euroa'), GENERIC_CENTS),
+ 'FIM': (('markka', 'markkaa'), ('penni', 'penni盲')), # historical
+ 'INR': (('rupia', 'rupiaa'), ('paisa', 'paisaa')),
+ 'JPY': (('jeni', 'jeni盲'), ('sen', 'seni盲')), # rare subunit
+ 'KRW': (('won', 'wonia'), ('jeon', 'jeonia')), # rare subunit
+ 'KPW': (('won', 'wonia'), ('chon', 'chonia')), # rare subunit
+ 'MXN': (('peso', 'pesoa'), GENERIC_CENTAVOS),
+ 'RUB': (('rupla', 'ruplaa'), ('kopeekka', 'kopeekkaa')),
+ 'TRY': (('liira', 'liiraa'), ('kuru艧', 'kuru艧ia')),
+ 'ZAR': (('randi', 'randia'), GENERIC_CENTS),
+ }
+
+ # crowns
+ for curr_code in 'DKK', 'ISK', 'NOK', 'SEK':
+ CURRENCY_FORMS[curr_code] = (('kruunu', 'kruunua'), ('盲yri', '盲yri盲'))
+
+ # dollars
+ for curr_code in 'AUD', 'CAD', 'HKD', 'NZD', 'SGD', 'USD':
+ CURRENCY_FORMS[curr_code] = (
+ ('dollari', 'dollaria'), GENERIC_CENTS)
+
+ # pounds
+ for curr_code in ('GBP',):
+ CURRENCY_FORMS[curr_code] = (('punta', 'puntaa'), ('penny', 'penny盲'))
+
+ CURRENCY_ADJECTIVES = {
+ 'AUD': 'Australian',
+ 'BRL': 'Brasilian',
+ 'CAD': 'Kanadan',
+ 'CHF': 'Sveitsin',
+ 'DKK': 'Tanskan',
+ 'FIM': 'Suomen', # historical
+ 'GBP': 'Englannin',
+ 'HKD': 'Hongkongin',
+ 'INR': 'Intian',
+ 'ISK': 'Islannin',
+ 'KRW': 'Etel盲-Korean',
+ 'KPW': 'Pohjois-Korean',
+ 'MXN': 'Meksikon',
+ 'NOK': 'Norjan',
+ 'NZD': 'Uuden-Seelannin',
+ 'RUB': 'Ven盲j盲n',
+ 'SEK': 'Ruotsin',
+ 'SGD': 'Singaporen',
+ 'TRY': 'Turkin',
+ 'USD': 'Yhdysvaltain',
+ 'ZAR': 'Etel盲-Afrikan',
+ }
+
+ def __init__(self):
+ self.ords = OrderedDict()
+ super(Num2Word_FI, self).__init__()
+
+ def set_numwords(self):
+ self.set_high_numwords(self.high_numwords)
+ self.set_mid_numwords(self.mid_numwords, self.mid_ords)
+ self.set_low_numwords(self.low_numwords, self.low_ords)
+
+ def set_high_numwords(self, high):
+ # references:
+ # https://fi.wikipedia.org/wiki/Suurten_lukujen_nimet
+ # https://en.wikipedia.org/wiki/Names_of_large_numbers#Standard_dictionary_numbers
+
+ # translate to Finnish
+ replacements = [
+ ("qu", "kv"),
+ ("x", "ks"),
+ ("c", "k"),
+ ("kent", "sent"), # applied after c -> k to cent
+ ]
+ translated = []
+ for i, numword in enumerate(high):
+ # notes:
+ # - 1e6**9 can be either noviljoona or noniljoona
+ # - 1e6**38 and above are untested
+
+ # 1e6**6 is sekstiljoona but 1e6**16 is sedekiljoona
+ if numword.startswith("sex") and numword != "sext":
+ numword = numword.replace("sex", "se")
+ # 1e6**7 is septiljoona but 1e6**17 is septendekiljoona
+ elif numword.startswith("sept") and numword != "sept":
+ numword = "septen" + numword[len("sept"):]
+ # 1e6**8 is oktiljoona but 1e6**18 is duodevigintiljoona
+ # (2 from 20)
+ elif numword.startswith("octo"):
+ numword = high[i + -10]
+ numword = "duode" + numword[len("octo"):]
+ # 1e6**9 is noniljoona but 1e6**19 is undevigintiljoona (1 from 20)
+ elif numword.startswith("nove"):
+ numword = high[i + -10]
+ numword = "unde" + numword[len("nove") + 1:]
+
+ # apply general replacements to all numwords
+ for repl in replacements:
+ numword = numword.replace(repl[0], repl[1])
+ translated.append(numword)
+
+ max = 6 * len(translated)
+ for word, n in zip(translated, range(max, 0, -6)):
+ if n == 6:
+ # irregularity considering short scale and long scale
+ self.cards[10 ** 9] = ("miljard", 5)
+ self.ords[10 ** 9] = ("miljardi", 45)
+ self.cards[10 ** n] = (word + "iljoon", 10)
+ self.ords[10 ** n] = (word + "iljoona", 45)
+
+ def set_mid_numwords(self, cards, ords):
+ for key, val in cards:
+ self.cards[key] = val
+ for key, val in ords:
+ self.ords[key] = val
+
+ def set_low_numwords(self, cards, ords):
+ for key, val in cards:
+ self.cards[key] = val
+ for key, val in ords:
+ self.ords[key] = val
+
+ def setup(self):
+ super(Num2Word_FI, self).setup()
+
+ self.negword = "miinus "
+ self.pointword = "pilkku"
+ self.exclude_title = ["pilkku", "miinus"]
+
+ self.mid_numwords = [
+ (1000, ("tuha", 46)),
+ (100, ("sa", 109)),
+ (90, [("yhdeks", 110), ("kymmen", 132, PTV)]),
+ (80, [("kahdeks", 110), ("kymmen", 132, PTV)]),
+ (70, [("seitsem", 110), ("kymmen", 132, PTV)]),
+ (60, [("kuu", 27), ("kymmen", 132, PTV)]),
+ (50, [("vii", 27), ("kymmen", 132, PTV)]),
+ (40, [("nelj", 10), ("kymmen", 132, PTV)]),
+ (30, [("kolm", 108), ("kymmen", 132, PTV)]),
+ ]
+
+ self.mid_ords = [
+ (1000, ("tuhanne", 45)),
+ (100, ("sada", 45)),
+ (90, [("yhdeks盲", 45), ("kymmene", 45)]),
+ (80, [("kahdeksa", 45), ("kymmene", 45)]),
+ (70, [("seitsem盲", 45), ("kymmene", 45)]),
+ (60, [("kuude", 45), ("kymmene", 45)]),
+ (50, [("viide", 45), ("kymmene", 45)]),
+ (40, [("nelj盲", 45), ("kymmene", 45)]),
+ (30, [("kolma", 45), ("kymmene", 45)]),
+ ]
+
+ self.low_numwords = [
+ (20, [("ka", 31), ("kymmen", 132, PTV)]),
+ (19, [("yhdeks", 110), "toista"]),
+ (18, [("kahdeks", 110), "toista"]),
+ (17, [("seitsem", 110), "toista"]),
+ (16, [("kuu", 27), "toista"]),
+ (15, [("vii", 27), "toista"]),
+ (14, [("nelj", 10), "toista"]),
+ (13, [("kolm", 108), "toista"]),
+ (12, [("ka", 31), "toista"]),
+ (11, [("y", 31), "toista"]),
+ (10, ("kymmen", 132)),
+ (9, ("yhdeks", 110)),
+ (8, ("kahdeks", 110)),
+ (7, ("seitsem", 110)),
+ (6, ("kuu", 27)),
+ (5, ("vii", 27)),
+ (4, ("nelj", 10)),
+ (3, ("kolm", 108)),
+ (2, ("ka", 31)),
+ (1, ("y", 31)),
+ (0, ("noll", 10)),
+ ]
+
+ self.low_ords = [
+ (20, [("kahde", 45), ("kymmene", 45)]),
+ (19, [("yhdeks盲", 45), "toista"]),
+ (18, [("kahdeksa", 45), "toista"]),
+ (17, [("seitsem盲", 45), "toista"]),
+ (16, [("kuude", 45), "toista"]),
+ (15, [("viide", 45), "toista"]),
+ (14, [("nelj盲", 45), "toista"]),
+ (13, [("kolma", 45), "toista"]),
+ (12, [("kahde", 45), "toista"]),
+ (11, [("yhde", 45), "toista"]),
+ (10, ("kymmene", 45)),
+ (9, ("yhdeks盲", 45)),
+ (8, ("kahdeksa", 45)),
+ (7, ("seitsem盲", 45)),
+ (6, ("kuude", 45)),
+ (5, ("viide", 45)),
+ (4, ("nelj盲", 45)),
+ (3, ("kolma", 45)),
+ (2, ("toi", 38)),
+ (1, ("ensimm盲i", 38)),
+ (0, ("nolla", 45)),
+ ]
+
+ def merge(self, lpair, rpair, options):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+
+ # http://www.kielitoimistonohjepankki.fi/ohje/49
+ fmt = "%s%s"
+ # ignore lpair if lnum is 1
+ if lnum == 1:
+ rtext = inflect(rtext, options)
+ return (rtext, rnum)
+ # rnum is added to lnum
+ elif lnum > rnum:
+ ltext = inflect(ltext, options)
+ rtext = inflect(rtext, options)
+ # separate groups with space
+ if lnum >= 1000:
+ fmt = "%s %s"
+ return (fmt % (ltext, rtext), lnum + rnum)
+ # rnum is multiplied by lnum
+ elif lnum < rnum:
+ if options.ordinal:
+ # kahdessadas, not toinensadas
+ if lnum == 2:
+ ltext = ("kahde", 45)
+ rtext = inflect(rtext, options)
+ else:
+ # kaksituhatta but kahdettuhannet
+ rcase = options.case
+ if options.case == NOM and not options.plural:
+ rcase = PTV
+ rtext = inflect(rtext, options.variation(case=rcase))
+ ltext = inflect(ltext, options)
+ return (fmt % (ltext, rtext), lnum * rnum)
+
+ def to_cardinal(self, value, case='nominative', plural=False, prefer=None):
+ case = NAME_TO_CASE[case]
+ options = Options(False, case, plural, prefer)
+ try:
+ assert int(value) == value
+ except (ValueError, TypeError, AssertionError):
+ if case != NOM:
+ raise NotImplementedError(
+ "Cases other than nominative are not implemented for "
+ "cardinal floating point numbers.")
+ return self.to_cardinal_float(value)
+
+ out = ""
+ if value < 0:
+ value = abs(value)
+ out = self.negword
+
+ if value >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig % (value, self.MAXVAL))
+
+ val = self.splitnum(value, options)
+ words, num = self.clean(val, options)
+ return self.title(out + words)
+
+ def to_ordinal(self, value, case='nominative', plural=False, prefer=None):
+ case = NAME_TO_CASE[case]
+ options = Options(True, case, plural, prefer)
+
+ self.verify_ordinal(value)
+ if value >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig % (value, self.MAXVAL))
+
+ val = self.splitnum(value, options)
+ words, num = self.clean(val, options)
+ return self.title(words)
+
+ def to_ordinal_num(self, value, case='nominative', plural=False):
+ case = NAME_TO_CASE[case]
+ raise NotImplementedError
+
+ def to_year(self, val, suffix=None, longval=True):
+ suffix = suffix or ""
+ if val < 0:
+ val = abs(val)
+ suffix = suffix or " ennen ajanlaskun alkua"
+ return self.to_cardinal(val).replace(" ", "") + suffix
+
+ def to_currency(self, val, currency="EUR", cents=True, separator=" ja",
+ adjective=False):
+ return super(Num2Word_FI, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+
+ def splitnum(self, value, options):
+ elems = self.ords if options.ordinal else self.cards
+ for elem in elems:
+ if elem > value:
+ continue
+
+ out = []
+ if value == 0:
+ div, mod = 1, 0
+ else:
+ div, mod = divmod(value, elem)
+
+ if div == 1:
+ out.append((elems[1], 1))
+ else:
+ if div == value: # The system tallies, eg Roman Numerals
+ return [(div * elems[elem], div*elem)]
+ out.append(self.splitnum(div, options))
+
+ out.append((elems[elem], elem))
+
+ if mod:
+ out.append(self.splitnum(mod, options))
+
+ return out
+
+ def clean(self, val, options):
+ out = val
+ while len(val) != 1:
+ out = []
+ left, right = val[:2]
+ if isinstance(left, tuple) and isinstance(right, tuple):
+ out.append(self.merge(left, right, options))
+ if val[2:]:
+ out.append(val[2:])
+ else:
+ for elem in val:
+ if isinstance(elem, list):
+ if len(elem) == 1:
+ out.append(elem[0])
+ else:
+ out.append(self.clean(elem, options))
+ else:
+ out.append(elem)
+ val = out
+ return out[0]
diff --git a/fun_text_processing/num2words/num2words/lang_FR.py b/fun_text_processing/num2words/num2words/lang_FR.py
new file mode 100644
index 0000000..133d802
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_FR.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_FR(Num2Word_EU):
+ CURRENCY_FORMS = {
+ 'EUR': (('euro', 'euros'), ('centime', 'centimes')),
+ 'USD': (('dollar', 'dollars'), ('cent', 'cents')),
+ 'FRF': (('franc', 'francs'), ('centime', 'centimes')),
+ 'GBP': (('livre', 'livres'), ('penny', 'pence')),
+ 'CNY': (('yuan', 'yuans'), ('fen', 'jiaos')),
+ }
+
+ def setup(self):
+ Num2Word_EU.setup(self)
+
+ self.negword = "moins "
+ self.pointword = "virgule"
+ self.errmsg_nonnum = (
+ u"Seulement des nombres peuvent 锚tre convertis en mots."
+ )
+ self.errmsg_toobig = u"Nombre trop grand pour 锚tre converti en mots."
+ self.exclude_title = ["et", "virgule", "moins"]
+ self.mid_numwords = [(1000, "mille"), (100, "cent"),
+ (80, "quatre-vingts"), (60, "soixante"),
+ (50, "cinquante"), (40, "quarante"),
+ (30, "trente")]
+ self.low_numwords = ["vingt", "dix-neuf", "dix-huit", "dix-sept",
+ "seize", "quinze", "quatorze", "treize", "douze",
+ "onze", "dix", "neuf", "huit", "sept", "six",
+ "cinq", "quatre", "trois", "deux", "un", "z茅ro"]
+ self.ords = {
+ "cinq": "cinqui猫me",
+ "neuf": "neuvi猫me",
+ }
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+ else:
+ if (not (cnum - 80) % 100
+ or (not cnum % 100 and cnum < 1000))\
+ and nnum < 1000000 \
+ and ctext[-1] == "s":
+ ctext = ctext[:-1]
+ if cnum < 1000 and nnum != 1000 and \
+ ntext[-1] != "s" and not nnum % 100:
+ ntext += "s"
+
+ if nnum < cnum < 100:
+ if nnum % 10 == 1 and cnum != 80:
+ return ("%s et %s" % (ctext, ntext), cnum + nnum)
+ return ("%s-%s" % (ctext, ntext), cnum + nnum)
+ if nnum > cnum:
+ return ("%s %s" % (ctext, ntext), cnum * nnum)
+ return ("%s %s" % (ctext, ntext), cnum + nnum)
+
+ # Is this right for such things as 1001 - "mille uni猫me" instead of
+ # "mille premier"?? "milli猫me"??
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ if value == 1:
+ return "premier"
+ word = self.to_cardinal(value)
+ for src, repl in self.ords.items():
+ if word.endswith(src):
+ word = word[:-len(src)] + repl
+ break
+ else:
+ if word[-1] == "e":
+ word = word[:-1]
+ word = word + "i猫me"
+ return word
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ out = str(value)
+ out += "er" if value == 1 else "me"
+ return out
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=' et',
+ adjective=False):
+ result = super(Num2Word_FR, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_FR_BE.py b/fun_text_processing/num2words/num2words/lang_FR_BE.py
new file mode 100644
index 0000000..feecd36
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_FR_BE.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_FR import Num2Word_FR
+
+
+class Num2Word_FR_BE(Num2Word_FR):
+ def setup(self):
+ Num2Word_FR.setup(self)
+
+ self.mid_numwords = [(1000, "mille"), (100, "cent"), (90, "nonante"),
+ (80, "quatre-vingt"), (70, "septante"),
+ (60, "soixante"), (50, "cinquante"),
+ (40, "quarante"), (30, "trente")]
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+
+ if cnum < 1000 and nnum != 1000 and\
+ ntext[-1] != "s" and not nnum % 100:
+ ntext += "s"
+
+ if nnum < cnum < 100:
+ if nnum % 10 == 1:
+ return ("%s et %s" % (ctext, ntext), cnum + nnum)
+ return ("%s-%s" % (ctext, ntext), cnum + nnum)
+ if nnum > cnum:
+ return ("%s %s" % (ctext, ntext), cnum * nnum)
+ return ("%s %s" % (ctext, ntext), cnum + nnum)
diff --git a/fun_text_processing/num2words/num2words/lang_FR_CH.py b/fun_text_processing/num2words/num2words/lang_FR_CH.py
new file mode 100644
index 0000000..72b98a1
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_FR_CH.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+from .lang_FR import Num2Word_FR
+
+
+class Num2Word_FR_CH(Num2Word_FR):
+ def setup(self):
+ Num2Word_FR.setup(self)
+
+ self.mid_numwords = [(1000, "mille"), (100, "cent"), (90, "nonante"),
+ (80, "huitante"), (70, "septante"),
+ (60, "soixante"), (50, "cinquante"),
+ (40, "quarante"), (30, "trente")]
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+
+ if cnum < 1000 and nnum != 1000 and\
+ ntext[-1] != "s" and not nnum % 100:
+ ntext += "s"
+
+ if nnum < cnum < 100:
+ if nnum % 10 == 1:
+ return ("%s et %s" % (ctext, ntext), cnum + nnum)
+ return ("%s-%s" % (ctext, ntext), cnum + nnum)
+ if nnum > cnum:
+ return ("%s %s" % (ctext, ntext), cnum * nnum)
+ return ("%s %s" % (ctext, ntext), cnum + nnum)
diff --git a/fun_text_processing/num2words/num2words/lang_FR_DZ.py b/fun_text_processing/num2words/num2words/lang_FR_DZ.py
new file mode 100644
index 0000000..9920cb6
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_FR_DZ.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .lang_FR import Num2Word_FR
+
+
+class Num2Word_FR_DZ(Num2Word_FR):
+ CURRENCY_FORMS = {
+ 'DIN': (('dinard', 'dinards'), ('centime', 'centimes')),
+ }
+
+ def to_currency(self, val, currency='DIN', cents=True, separator=' et',
+ adjective=False):
+ result = super(Num2Word_FR, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_HE.py b/fun_text_processing/num2words/num2words/lang_HE.py
new file mode 100644
index 0000000..3117301
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_HE.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+
+
+from __future__ import print_function, unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = (u'讗驻住',)
+
+ONES = {
+ 1: (u'讗讞转',),
+ 2: (u'砖转讬诐',),
+ 3: (u'砖诇砖',),
+ 4: (u'讗专讘注',),
+ 5: (u'讞诪砖',),
+ 6: (u'砖砖',),
+ 7: (u'砖讘注',),
+ 8: (u'砖诪讜谞讛',),
+ 9: (u'转砖注',),
+}
+
+TENS = {
+ 0: (u'注砖专',),
+ 1: (u'讗讞转 注砖专讛',),
+ 2: (u'砖转讬诐 注砖专讛',),
+ 3: (u'砖诇砖 注砖专讛',),
+ 4: (u'讗专讘注 注砖专讛',),
+ 5: (u'讞诪砖 注砖专讛',),
+ 6: (u'砖砖 注砖专讛',),
+ 7: (u'砖讘注 注砖专讛',),
+ 8: (u'砖诪讜谞讛 注砖专讛',),
+ 9: (u'转砖注 注砖专讛',),
+}
+
+TWENTIES = {
+ 2: (u'注砖专讬诐',),
+ 3: (u'砖诇砖讬诐',),
+ 4: (u'讗专讘注讬诐',),
+ 5: (u'讞诪讬砖讬诐',),
+ 6: (u'砖砖讬诐',),
+ 7: (u'砖讘注讬诐',),
+ 8: (u'砖诪讜谞讬诐',),
+ 9: (u'转砖注讬诐',),
+}
+
+HUNDRED = {
+ 1: (u'诪讗讛',),
+ 2: (u'诪讗转讬讬诐',),
+ 3: (u'诪讗讜转',)
+}
+
+THOUSANDS = {
+ 1: (u'讗诇祝',),
+ 2: (u'讗诇驻讬讬诐',),
+ 3: (u'砖诇砖转 讗诇驻讬诐',),
+ 4: (u'讗专讘注转 讗诇驻讬诐',),
+ 5: (u'讞诪砖转 讗诇驻讬诐',),
+ 6: (u'砖砖转 讗诇驻讬诐',),
+ 7: (u'砖讘注转 讗诇驻讬诐',),
+ 8: (u'砖诪讜谞转 讗诇驻讬诐',),
+ 9: (u'转砖注转 讗诇驻讬诐',),
+}
+
+AND = u'讜'
+
+
+def pluralize(n, forms):
+ # gettext implementation:
+ # (n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2)
+
+ form = 0 if (n % 10 == 1 and n % 100 != 11) else 1 if n != 0 else 2
+
+ return forms[form]
+
+
+def int2word(n):
+ if n > 9999: # doesn't yet work for numbers this big
+ raise NotImplementedError()
+
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if i > 0:
+ words.append(THOUSANDS[n1][0])
+ continue
+
+ if n3 > 0:
+ if n3 <= 2:
+ words.append(HUNDRED[n3][0])
+ else:
+ words.append(ONES[n3][0] + ' ' + HUNDRED[3][0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0 and not (i > 0 and x == 1):
+ words.append(ONES[n1][0])
+
+ if i > 0:
+ words.append(THOUSANDS[i][0])
+
+ # source: https://hebrew-academy.org.il/2017/01/30/讜-讛讞讬讘讜专-讘诪住驻专讬诐/
+ if len(words) > 1:
+ words[-1] = AND + words[-1]
+
+ return ' '.join(words)
+
+
+def n2w(n):
+ return int2word(int(n))
+
+
+def to_currency(n, currency='EUR', cents=True, separator=','):
+ raise NotImplementedError()
+
+
+class Num2Word_HE(Num2Word_Base):
+ def to_cardinal(self, number):
+ return n2w(number)
+
+ def to_ordinal(self, number):
+ raise NotImplementedError()
+
+
+if __name__ == '__main__':
+ yo = Num2Word_HE()
+ nums = [1, 11, 21, 24, 99, 100, 101, 200, 211, 345, 1000, 1011]
+ for num in nums:
+ print(num, yo.to_cardinal(num))
diff --git a/fun_text_processing/num2words/num2words/lang_HU.py b/fun_text_processing/num2words/num2words/lang_HU.py
new file mode 100644
index 0000000..32591e9
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_HU.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+ZERO = 'nulla'
+
+
+class Num2Word_HU(lang_EU.Num2Word_EU):
+ GIGA_SUFFIX = "illi谩rd"
+ MEGA_SUFFIX = "illi贸"
+
+ def setup(self):
+ super(Num2Word_HU, self).setup()
+
+ self.negword = "m铆nusz "
+ self.pointword = "eg茅sz"
+
+ self.mid_numwords = [(1000, "ezer"), (100, "sz谩z"), (90, "kilencven"),
+ (80, "nyolcvan"), (70, "hetven"), (60, "hatvan"),
+ (50, "枚tven"), (40, "negyven"), (30, "harminc")]
+
+ low_numwords = ["kilenc", "nyolc", "h茅t", "hat", "枚t", "n茅gy", "h谩rom",
+ "kett艖", "egy"]
+ self.low_numwords = (['tizen' + w for w in low_numwords]
+ + ['t铆z']
+ + low_numwords)
+ self.low_numwords = (['huszon' + w for w in low_numwords]
+ + ['h煤sz']
+ + self.low_numwords
+ + [ZERO])
+
+ self.partial_ords = {
+ 'nulla': 'nullad',
+ 'egy': 'egyed',
+ 'kett艖': 'ketted',
+ 'h谩rom': 'harmad',
+ 'n茅gy': 'negyed',
+ '枚t': '枚t枚d',
+ 'hat': 'hatod',
+ 'h茅t': 'heted',
+ 'nyolc': 'nyolcad',
+ 'kilenc': 'kilenced',
+ 't铆z': 'tized',
+ 'h煤sz': 'huszad',
+ 'harminc': 'harmincad',
+ 'negyven': 'negyvened',
+ '枚tven': '枚tvened',
+ 'hatvan': 'hatvanad',
+ 'hetven': 'hetvened',
+ 'nyolcvan': 'nyolcvanad',
+ 'kilencven': 'kilencvened',
+ 'sz谩z': 'sz谩zad',
+ 'ezer': 'ezred',
+ 'illi贸': 'milliomod',
+ 'illi谩rd': 'milli谩rdod'
+ }
+
+ def to_cardinal(self, value, zero=ZERO):
+ if int(value) != value:
+ return self.to_cardinal_float(value)
+ elif value < 0:
+ out = self.negword + self.to_cardinal(-value)
+ elif value == 0:
+ out = zero
+ elif zero == '' and value == 2:
+ out = 'k茅t'
+ elif value < 30:
+ out = self.cards[value]
+ elif value < 100:
+ out = self.tens_to_cardinal(value)
+ elif value < 1000:
+ out = self.hundreds_to_cardinal(value)
+ elif value < 10**6:
+ out = self.thousands_to_cardinal(value)
+ else:
+ out = self.big_number_to_cardinal(value)
+ return out
+
+ def tens_to_cardinal(self, value):
+ try:
+ return self.cards[value]
+ except KeyError:
+ return self.cards[value // 10 * 10] + self.to_cardinal(value % 10)
+
+ def hundreds_to_cardinal(self, value):
+ hundreds = value // 100
+ prefix = "sz谩z"
+ if hundreds != 1:
+ prefix = self.to_cardinal(hundreds, zero="") + prefix
+ postfix = self.to_cardinal(value % 100, zero="")
+ return prefix + postfix
+
+ def thousands_to_cardinal(self, value):
+ thousands = value // 1000
+ prefix = "ezer"
+ if thousands != 1:
+ prefix = self.to_cardinal(thousands, zero="") + prefix
+ postfix = self.to_cardinal(value % 1000, zero="")
+ return prefix + ('' if value <= 2000 or not postfix else '-') + postfix
+
+ def big_number_to_cardinal(self, value):
+ digits = len(str(value))
+ digits = digits if digits % 3 != 0 else digits - 2
+ exp = 10 ** (digits // 3 * 3)
+ rest = self.to_cardinal(value % exp, '')
+ return (self.to_cardinal(value // exp, '') + self.cards[exp]
+ + ('-' + rest if rest else ''))
+
+ def to_ordinal(self, value):
+ if value < 0:
+ return self.negword + self.to_ordinal(-value)
+ if value == 1:
+ return 'els艖'
+ elif value == 2:
+ return 'm谩sodik'
+ else:
+ out = self.to_cardinal(value)
+ for card_word, ord_word in self.partial_ords.items():
+ if out[-len(card_word):] == card_word:
+ out = out[:-len(card_word)] + ord_word
+ break
+ return out + 'ik'
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return str(value) + '.'
+
+ def to_year(self, val, suffix=None, longval=True):
+ # suffix is prefix here
+ prefix = ''
+ if val < 0 or suffix is not None:
+ val = abs(val)
+ prefix = (suffix + ' ' if suffix is not None else 'i. e. ')
+ return prefix + self.to_cardinal(val)
+
+ def to_currency(self, val, currency='HUF', cents=True, separator=',',
+ adjective=False):
+ return super(Num2Word_HU, self).to_currency(
+ val, currency, cents, separator, adjective)
+
+ def to_cardinal_float(self, value):
+ if abs(value) != value:
+ return self.negword + self.to_cardinal_float(-value)
+ left, right = str(value).split('.')
+ return (self.to_cardinal(int(left))
+ + ' eg茅sz '
+ + self.to_cardinal(int(right))
+ + ' ' + self.partial_ords[self.cards[10 ** len(right)]])
diff --git a/fun_text_processing/num2words/num2words/lang_ID.py b/fun_text_processing/num2words/num2words/lang_ID.py
new file mode 100644
index 0000000..2648918
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_ID.py
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+
+class Num2Word_ID():
+ BASE = {0: [],
+ 1: ["satu"],
+ 2: ["dua"],
+ 3: ["tiga"],
+ 4: ["empat"],
+ 5: ["lima"],
+ 6: ["enam"],
+ 7: ["tujuh"],
+ 8: ["delapan"],
+ 9: ["sembilan"]}
+
+ TENS_TO = {3: "ribu",
+ 6: "juta",
+ 9: "miliar",
+ 12: "triliun",
+ 15: "kuadriliun",
+ 18: "kuantiliun",
+ 21: "sekstiliun",
+ 24: "septiliun",
+ 27: "oktiliun",
+ 30: "noniliun",
+ 33: "desiliun"}
+
+ errmsg_floatord = "Cannot treat float number as ordinal"
+ errmsg_negord = "Cannot treat negative number as ordinal"
+ errmsg_toobig = "Too large"
+ max_num = 10 ** 36
+
+ def split_by_koma(self, number):
+ return str(number).split('.')
+
+ def split_by_3(self, number):
+ """
+ starting here, it groups the number by three from the tail
+ '1234567' -> (('1',),('234',),('567',))
+ :param number:str
+ :rtype:tuple
+ """
+ blocks = ()
+ length = len(number)
+
+ if length < 3:
+ blocks += ((number,),)
+ else:
+ len_of_first_block = length % 3
+
+ if len_of_first_block > 0:
+ first_block = number[0:len_of_first_block],
+ blocks += first_block,
+
+ for i in range(len_of_first_block, length, 3):
+ next_block = (number[i:i + 3],),
+ blocks += next_block
+
+ return blocks
+
+ def spell(self, blocks):
+ """
+ it adds the list of spelling to the blocks
+ (
+ ('1',),('034',)) -> (('1',['satu']),('234',['tiga', 'puluh', 'empat'])
+ )
+ :param blocks: tuple
+ :rtype: tuple
+ """
+ word_blocks = ()
+ first_block = blocks[0]
+ if len(first_block[0]) == 1:
+ if first_block[0] == '0':
+ spelling = ['nol']
+ else:
+ spelling = self.BASE[int(first_block[0])]
+ elif len(first_block[0]) == 2:
+ spelling = self.puluh(first_block[0])
+ else:
+ spelling = (
+ self.ratus(first_block[0][0]) + self.puluh(first_block[0][1:3])
+ )
+
+ word_blocks += (first_block[0], spelling),
+
+ for block in blocks[1:]:
+ spelling = self.ratus(block[0][0]) + self.puluh(block[0][1:3])
+ block += spelling,
+ word_blocks += block,
+
+ return word_blocks
+
+ def ratus(self, number):
+ # it is used to spell
+ if number == '1':
+ return ['seratus']
+ elif number == '0':
+ return []
+ else:
+ return self.BASE[int(number)] + ['ratus']
+
+ def puluh(self, number):
+ # it is used to spell
+ if number[0] == '1':
+ if number[1] == '0':
+ return ['sepuluh']
+ elif number[1] == '1':
+ return ['sebelas']
+ else:
+ return self.BASE[int(number[1])] + ['belas']
+ elif number[0] == '0':
+ return self.BASE[int(number[1])]
+ else:
+ return (
+ self.BASE[int(number[0])] + ['puluh']
+ + self.BASE[int(number[1])]
+ )
+
+ def spell_float(self, float_part):
+ # spell the float number
+ word_list = []
+ for n in float_part:
+ if n == '0':
+ word_list += ['nol']
+ continue
+ word_list += self.BASE[int(n)]
+ return ' '.join(['', 'koma'] + word_list)
+
+ def join(self, word_blocks, float_part):
+ """
+ join the words by first join lists in the tuple
+ :param word_blocks: tuple
+ :rtype: str
+ """
+ word_list = []
+ length = len(word_blocks) - 1
+ first_block = word_blocks[0],
+ start = 0
+
+ if length == 1 and first_block[0][0] == '1':
+ word_list += ['seribu']
+ start = 1
+
+ for i in range(start, length + 1, 1):
+ word_list += word_blocks[i][1]
+ if not word_blocks[i][1]:
+ continue
+ if i == length:
+ break
+ word_list += [self.TENS_TO[(length - i) * 3]]
+
+ return ' '.join(word_list) + float_part
+
+ def to_cardinal(self, number):
+ if number >= self.max_num:
+ raise OverflowError(self.errmsg_toobig % (number, self.max_num))
+ minus = ''
+ if number < 0:
+ minus = 'min '
+ float_word = ''
+ n = self.split_by_koma(abs(number))
+ if len(n) == 2:
+ float_word = self.spell_float(n[1])
+ return minus + self.join(self.spell(self.split_by_3(n[0])), float_word)
+
+ def to_ordinal(self, number):
+ self.verify_ordinal(number)
+ out_word = self.to_cardinal(number)
+ if out_word == "satu":
+ return "pertama"
+ return "ke" + out_word
+
+ def to_ordinal_num(self, number):
+ self.verify_ordinal(number)
+ return "ke-" + str(number)
+
+ def to_currency(self, value):
+ return self.to_cardinal(value) + " rupiah"
+
+ def to_year(self, value):
+ return self.to_cardinal(value)
+
+ def verify_ordinal(self, value):
+ if not value == int(value):
+ raise TypeError(self.errmsg_floatord % value)
+ if not abs(value) == value:
+ raise TypeError(self.errmsg_negord % value)
diff --git a/fun_text_processing/num2words/num2words/lang_IT.py b/fun_text_processing/num2words/num2words/lang_IT.py
new file mode 100644
index 0000000..6966d73
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_IT.py
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2018-2019, Filippo Costa. All Rights Reserved.
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301 USA
+
+from __future__ import unicode_literals
+
+# Globals
+# -------
+
+ZERO = "zero"
+
+CARDINAL_WORDS = [
+ ZERO, "uno", "due", "tre", "quattro", "cinque", "sei", "sette", "otto",
+ "nove", "dieci", "undici", "dodici", "tredici", "quattordici", "quindici",
+ "sedici", "diciassette", "diciotto", "diciannove"
+]
+
+ORDINAL_WORDS = [
+ ZERO, "primo", "secondo", "terzo", "quarto", "quinto", "sesto", "settimo",
+ "ottavo", "nono", "decimo", "undicesimo", "dodicesimo", "tredicesimo",
+ "quattordicesimo", "quindicesimo", "sedicesimo", "diciassettesimo",
+ "diciottesimo", "diciannovesimo"
+]
+
+# The script can extrapolate the missing numbers from the base forms.
+STR_TENS = {2: "venti", 3: "trenta", 4: "quaranta", 6: "sessanta"}
+
+# These prefixes are used for extremely big numbers.
+EXPONENT_PREFIXES = [
+ ZERO, "m", "b", "tr", "quadr", "quint", "sest", "sett", "ott", "nov", "dec"
+]
+
+
+# Utils
+# =====
+
+def phonetic_contraction(string):
+ return (string
+ .replace("oo", "o") # ex. "centootto"
+ .replace("ao", "o") # ex. "settantaotto"
+ .replace("io", "o") # ex. "ventiotto"
+ .replace("au", "u") # ex. "trentauno"
+ .replace("iu", "u") # ex. "ventiunesimo"
+ )
+
+
+def exponent_length_to_string(exponent_length):
+ # We always assume `exponent` to be a multiple of 3. If it's not true, then
+ # Num2Word_IT.big_number_to_cardinal did something wrong.
+ prefix = EXPONENT_PREFIXES[exponent_length // 6]
+ if exponent_length % 6 == 0:
+ return prefix + "ilione"
+ else:
+ return prefix + "iliardo"
+
+
+def accentuate(string):
+ # This is inefficient: it may do several rewritings when deleting
+ # half-sentence accents. However, it is the easiest method and speed is
+ # not crucial (duh), so...
+ return " ".join(
+ # Deletes half-sentence accents and accentuates the last "tre"
+ [w.replace("tr茅", "tre")[:-3] + "tr茅"
+ # We shouldn't accentuate a single "tre": is has to be a composite
+ # word. ~~~~~~~~~~
+ if w[-3:] == "tre" and len(w) > 3
+ # Deletes half-sentence accents anyway
+ # ~~~~~~~~~~~~~~~~~~~~~~
+ else w.replace("tr茅", "tre")
+ for w in string.split()
+ ])
+
+
+def omitt_if_zero(number_to_string):
+ return "" if number_to_string == ZERO else number_to_string
+
+
+# Main class
+# ==========
+
+class Num2Word_IT:
+ MINUS_PREFIX_WORD = "meno "
+ FLOAT_INFIX_WORD = " virgola "
+
+ def __init__(self):
+ pass
+
+ def float_to_words(self, float_number, ordinal=False):
+ if ordinal:
+ prefix = self.to_ordinal(int(float_number))
+ else:
+ prefix = self.to_cardinal(int(float_number))
+ float_part = str(float_number).split('.')[1]
+ postfix = " ".join(
+ # Drops the trailing zero and comma
+ [self.to_cardinal(int(c)) for c in float_part]
+ )
+ return prefix + Num2Word_IT.FLOAT_INFIX_WORD + postfix
+
+ def tens_to_cardinal(self, number):
+ tens = number // 10
+ units = number % 10
+ if tens in STR_TENS:
+ prefix = STR_TENS[tens]
+ else:
+ prefix = CARDINAL_WORDS[tens][:-1] + "anta"
+ postfix = omitt_if_zero(CARDINAL_WORDS[units])
+ return phonetic_contraction(prefix + postfix)
+
+ def hundreds_to_cardinal(self, number):
+ hundreds = number // 100
+ prefix = "cento"
+ if hundreds != 1:
+ prefix = CARDINAL_WORDS[hundreds] + prefix
+ postfix = omitt_if_zero(self.to_cardinal(number % 100))
+ return phonetic_contraction(prefix + postfix)
+
+ def thousands_to_cardinal(self, number):
+ thousands = number // 1000
+ if thousands == 1:
+ prefix = "mille"
+ else:
+ prefix = self.to_cardinal(thousands) + "mila"
+ postfix = omitt_if_zero(self.to_cardinal(number % 1000))
+ # "mille" and "mila" don't need any phonetic contractions
+ return prefix + postfix
+
+ def big_number_to_cardinal(self, number):
+ digits = [c for c in str(number)]
+ length = len(digits)
+ if length >= 66:
+ raise NotImplementedError("The given number is too large.")
+ # This is how many digits come before the "illion" term.
+ # cento miliardi => 3
+ # dieci milioni => 2
+ # un miliardo => 1
+ predigits = length % 3 or 3
+ multiplier = digits[:predigits]
+ exponent = digits[predigits:]
+ # Default infix string: "milione", "biliardo", "sestilione", ecc.
+ infix = exponent_length_to_string(len(exponent))
+ if multiplier == ["1"]:
+ prefix = "un "
+ else:
+ prefix = self.to_cardinal(int("".join(multiplier)))
+ # Plural form ~~~~~~~~~~~
+ infix = " " + infix[:-1] + "i"
+ # Read as: Does the value of exponent equal 0?
+ if set(exponent) != set("0"):
+ postfix = self.to_cardinal(int("".join(exponent)))
+ if " e " in postfix:
+ infix += ", "
+ else:
+ infix += " e "
+ else:
+ postfix = ""
+ return prefix + infix + postfix
+
+ def to_cardinal(self, number):
+ if number < 0:
+ string = Num2Word_IT.MINUS_PREFIX_WORD + self.to_cardinal(-number)
+ elif isinstance(number, float):
+ string = self.float_to_words(number)
+ elif number < 20:
+ string = CARDINAL_WORDS[number]
+ elif number < 100:
+ string = self.tens_to_cardinal(number)
+ elif number < 1000:
+ string = self.hundreds_to_cardinal(number)
+ elif number < 1000000:
+ string = self.thousands_to_cardinal(number)
+ else:
+ string = self.big_number_to_cardinal(number)
+ return accentuate(string)
+
+ def to_ordinal(self, number):
+ tens = number % 100
+ # Italian grammar is poorly defined here 炉\_(銉�)_/炉:
+ # centodecimo VS centodieciesimo VS centesimo decimo?
+ is_outside_teens = not 10 < tens < 20
+ if number < 0:
+ return Num2Word_IT.MINUS_PREFIX_WORD + self.to_ordinal(-number)
+ elif number % 1 != 0:
+ return self.float_to_words(number, ordinal=True)
+ elif number < 20:
+ return ORDINAL_WORDS[number]
+ elif is_outside_teens and tens % 10 == 3:
+ # Gets ride of the accent ~~~~~~~~~~
+ return self.to_cardinal(number)[:-1] + "eesimo"
+ elif is_outside_teens and tens % 10 == 6:
+ return self.to_cardinal(number) + "esimo"
+ else:
+ string = self.to_cardinal(number)[:-1]
+ if string[-3:] == "mil":
+ string += "l"
+ return string + "esimo"
diff --git a/fun_text_processing/num2words/num2words/lang_IW.py b/fun_text_processing/num2words/num2words/lang_IW.py
new file mode 100644
index 0000000..c43941f
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_IW.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+# Hebrew num2words
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_IW(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 + 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10 ** n] = word + " 诪执讬诇执讬讜止谉"
+
+ def setup(self):
+ super(Num2Word_IW, self).setup()
+
+ self.negword = "驻讞讜转 "
+ self.pointword = "谞职拽讜旨讚指讛"
+ self.exclude_title = ["讜", "谞职拽讜旨讚指讛", "驻讞讜转"]
+
+ self.mid_numwords = [(1000, "讗诇祝"), (100, "诪讗讛"),
+ (90, "转执砖讈注执讬诐"), (80, "砖诪讜谞讬诐"), (70, "砖讈执讘注执讬诐"),
+ (60, "砖讈执砖讈执讬诐"), (50, "讞诪讬砖讬诐"), (40, "讗专讘注讬诐"),
+ (30, "砖讈职诇讜止砖讈执讬诐")]
+ self.low_numwords = ['注砖专讬诐', '转砖注 注砖专讛', '砖诪讜谞讛 注砖专讛', '砖讘注 注砖专讛', '砖砖 注砖专讛', '讞植诪值砖讈 注侄砖讉专值讛', '讗专讘注讛 注砖专', '砖讈职诇讜止砖讈 注侄砖讉专值讛', '砖转讬讬诐 注砖专讛', '讗讞讚 注砖专', '注砖专', '转值砖讈址注', '砖诪讜谞讛', '砖讘注', '砖讈值砖讈', '讞指诪值砖讈', '讗专讘注', '砖讈职诇讜止砖讈指讛', '砖转讬讬诐', '讗讞讚', '讗侄驻侄住']
+ self.ords = {'讗讞讚': '专讗砖讜谉',
+'砖转讬讬诐': '砖讈职谞执讬指讛',
+'砖讈职诇讜止砖讈指讛': '砖讈职诇执讬砖讈执讬',
+'讗专讘注': '专讘讬注讬',
+'讞指诪值砖讈': '讞诪讬砖讬',
+'砖讈值砖讈': '砖讈执砖讈执讬转',
+'砖讘注': '砖讈职讘执讬注执讬转',
+'砖诪讜谞讛': '砖诪讜谞讛',
+'转值砖讈址注': '讟',
+'注砖专': '注植砖讉执讬专执讬转'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s 讜 %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s, %s" % (ltext, rtext), lnum + rnum)
+
+ # def to_ordinal(self, value):
+ # self.verify_ordinal(value)
+ # outwords = self.to_cardinal(value).split(" ")
+ # lastwords = outwords[-1].split("-")
+ # lastword = lastwords[-1].lower()
+ # try:
+ # lastword = self.ords[lastword]
+ # except KeyError:
+ # if lastword[-1] == "y":
+ # lastword = lastword[:-1] + "ie"
+ # lastword += "th"
+ # lastwords[-1] = self.title(lastword)
+ # outwords[-1] = "-".join(lastwords)
+ # return " ".join(outwords)
+
+ # def to_ordinal_num(self, value):
+ # self.verify_ordinal(value)
+ # return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '诇执驻谞值讬 讛址住职驻执讬专指讛' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "诪值讗指讛"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_JA.py b/fun_text_processing/num2words/num2words/lang_JA.py
new file mode 100644
index 0000000..b3ade3e
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_JA.py
@@ -0,0 +1,574 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from .base import Num2Word_Base
+from .compat import strtype, to_s
+from .currency import parse_currency_parts, prefix_currency
+
+
+def select_text(text, reading=False, prefer=None):
+ """Select the correct text from the Japanese number, reading and
+ alternatives"""
+ # select kanji number or kana reading
+ if reading:
+ text = text[1]
+ else:
+ text = text[0]
+
+ # select the preferred one or the first one from multiple alternatives
+ if not isinstance(text, strtype):
+ common = set(text) & set(prefer or set())
+ if len(common) == 1:
+ text = common.pop()
+ else:
+ text = text[0]
+
+ return text
+
+
+def rendaku_merge_pairs(lpair, rpair):
+ """Merge lpair < rpair while applying semi-irregular rendaku rules"""
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum > rnum:
+ raise ValueError
+
+ if rpair == ("銇层們銇�", 100):
+ if lpair == ("銇曘倱", 3):
+ rtext = "銇炽們銇�"
+ elif lpair == ("銈嶃亸", 6):
+ ltext = "銈嶃仯"
+ rtext = "銇淬們銇�"
+ elif lpair == ("銇仭", 8):
+ ltext = "銇仯"
+ rtext = "銇淬們銇�"
+ elif rpair == ("銇涖倱", 1000):
+ if lpair == ("銇曘倱", 3):
+ rtext = "銇溿倱"
+ elif lpair == ("銇仭", 8):
+ ltext = "銇仯"
+ elif rpair == ("銇°倗銇�", 10**12):
+ if lpair == ("銇勩仭", 1):
+ ltext = "銇勩仯"
+ elif lpair == ("銇仭", 8):
+ ltext = "銇仯"
+ elif lpair == ("銇樸倕銇�", 10):
+ ltext = "銇樸倕銇�"
+ elif rpair == ("銇戙亜", 10**16):
+ if lpair == ("銇勩仭", 1):
+ ltext = "銇勩仯"
+ elif lpair == ("銈嶃亸", 6):
+ ltext = "銈嶃仯"
+ elif lpair == ("銇仭", 8):
+ ltext = "銇仯"
+ elif lpair == ("銇樸倕銇�", 10):
+ ltext = "銇樸倕銇�"
+ elif lpair == ("銇层們銇�", 100):
+ ltext = "銇层們銇�"
+
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+
+
+# Source: https://www.sljfaq.org/afaq/era-list.html
+# if there are multiple eras for the same year, use the last one
+ERA_START = [
+ (645, ("澶у寲", "銇熴亜銇�")),
+ (650, ("鐧介泬", "銇亸銇�")),
+ (686, ("鏈遍偿", "銇椼倕銇°倗銇�")),
+ (701, ("澶у疂", "銇熴亜銇汇亞")),
+ (704, ("鎱堕洸", "銇戙亜銇嗐倱")),
+ (708, ("鍜岄妳", "銈忋仼銇�")),
+ (715, ("闇婁簚", "銈屻亜銇�")),
+ (717, ("椁婅��", "銈堛亞銈嶃亞")),
+ (724, ("绁炰簚", "銇樸倱銇�")),
+ (729, ("澶╁钩", "銇︺倱銇淬倗銇�")),
+ (749, ("澶╁钩鎰熷疂", "銇︺倱銇淬倗銇嗐亱銈撱伣銇�")),
+ (749, ("澶╁钩鍕濆疂", "銇︺倱銇淬倗銇嗐仐銈囥亞銇汇亞")),
+ (757, ("澶╁钩瀹濆瓧", "銇︺倱銇淬倗銇嗐仒銈囥亞銇�")),
+ (765, ("澶╁钩绁炶", "銇︺倱銇淬倗銇嗐仒銈撱仈")),
+ (767, ("绁炶鏅洸", "銇樸倱銇斻亼銇勩亞銈�")),
+ (770, ("瀹濅簚", "銇汇亞銇�")),
+ (781, ("澶╁繙", "銇︺倱銇娿亞")),
+ (782, ("寤舵殾", "銇堛倱銈娿們銇�")),
+ (806, ("澶у悓", "銇犮亜銇┿亞")),
+ (810, ("寮樹粊", "銇撱亞銇倱")),
+ (823, ("澶╅暦", "銇︺倱銇°倗銇�")),
+ (834, ("鎵垮拰", "銇樸倗銇嗐倧")),
+ (848, ("鍢夌ゥ", "銇嬨仐銈囥亞")),
+ (851, ("浠佸", "銇倱銇樸倕")),
+ (855, ("鏂夎 ", "銇曘亜銇撱亞")),
+ (857, ("澶╁畨", "銇︺倱銇傘倱")),
+ (859, ("璨炶Τ", "銇樸倗銇嗐亴銈�")),
+ (877, ("鍏冩叾", "銇屻倱銇庛倗銇�")),
+ (885, ("浠佸拰", "銇倱銇�")),
+ (889, ("瀵涘钩", "銇嬨倱銇淬倗銇�")),
+ (898, ("鏄屾嘲", "銇椼倗銇嗐仧銇�")),
+ (901, ("寤跺枩", "銇堛倱銇�")),
+ (923, ("寤堕暦", "銇堛倱銇°倗銇�")),
+ (931, ("鎵垮钩", "銇樸倗銇嗐伕銇�")),
+ (938, ("澶╂叾", "銇︺倱銇庛倗銇�")),
+ (947, ("澶╂殾", "銇︺倱銈娿們銇�")),
+ (957, ("澶╁境", "銇︺倱銇ㄣ亸")),
+ (961, ("蹇滃拰", "銇娿亞銈�")),
+ (964, ("搴蜂繚", "銇撱亞銇汇亞")),
+ (968, ("瀹夊拰", "銇傘倱銇�")),
+ (970, ("澶╃", "銇︺倱銈嶃亸")),
+ (974, ("澶╁欢", "銇︺倱銇堛倱")),
+ (976, ("璨炲厓", "銇樸倗銇嗐亽銈�")),
+ (979, ("澶╁厓", "銇︺倱銇掋倱")),
+ (983, ("姘歌Τ", "銇堛亜銇嬨倱")),
+ (985, ("瀵涘拰", "銇嬨倱銇�")),
+ (987, ("姘稿欢", "銇堛亜銇堛倱")),
+ (989, ("姘哥", "銇堛亜銇�")),
+ (990, ("姝ｆ殾", "銇椼倗銇嗐倞銈冦亸")),
+ (995, ("闀峰境", "銇°倗銇嗐仺銇�")),
+ (999, ("闀蜂繚", "銇°倗銇嗐伝銇�")),
+ (1004, ("瀵涘紭", "銇嬨倱銇撱亞")),
+ (1013, ("闀峰拰", "銇°倗銇嗐倧")),
+ (1017, ("瀵涗粊", "銇嬨倱銇倱")),
+ (1021, ("娌诲畨", "銇樸亗銈�")),
+ (1024, ("涓囧", "銇俱倱銇樸倕")),
+ (1028, ("闀峰厓", "銇°倗銇嗐亽銈�")),
+ (1037, ("闀锋殾", "銇°倗銇嗐倞銈冦亸")),
+ (1040, ("闀蜂箙", "銇°倗銇嗐亶銈呫亞")),
+ (1045, ("瀵涘境", "銇嬨倱銇ㄣ亸")),
+ (1046, ("姘告壙", "銇堛亜銇椼倗銇�")),
+ (1053, ("澶╁枩", "銇︺倱銇�")),
+ (1058, ("搴峰钩", "銇撱亞銇搞亜")),
+ (1065, ("娌绘殾", "銇樸倞銈冦亸")),
+ (1069, ("寤朵箙", "銇堛倱銇嶃倕銇�")),
+ (1074, ("鎵夸繚", "銇樸倗銇嗐伝銇�")),
+ (1078, ("鎵挎殾", "銇樸倗銇嗐倞銈冦亸")),
+ (1081, ("姘镐繚", "銇堛亜銇汇亞")),
+ (1084, ("蹇滃境", "銇娿亞銇ㄣ亸")),
+ (1087, ("瀵涙不", "銇嬨倱銇�")),
+ (1095, ("鍢変繚", "銇嬨伝銇�")),
+ (1097, ("姘搁暦", "銇堛亜銇°倗銇�")),
+ (1098, ("鎵垮境", "銇樸倗銇嗐仺銇�")),
+ (1099, ("搴峰拰", "銇撱亞銈�")),
+ (1104, ("闀锋不", "銇°倗銇嗐仒")),
+ (1106, ("鍢夋壙", "銇嬨仒銈囥亞")),
+ (1108, ("澶╀粊", "銇︺倱銇倱")),
+ (1110, ("澶╂案", "銇︺倱銇亜")),
+ (1113, ("姘镐箙", "銇堛亜銇嶃倕銇�")),
+ (1118, ("鍏冩案", "銇掋倱銇堛亜")),
+ (1120, ("淇濆畨", "銇汇亞銇傘倱")),
+ (1124, ("澶╂不", "銇︺倱銇�")),
+ (1126, ("澶ф不", "銇犮亜銇�")),
+ (1131, ("澶╂壙", "銇︺倱銇椼倗銇�")),
+ (1132, ("闀锋壙", "銇°倗銇嗐仐銈囥亞")),
+ (1135, ("淇濆欢", "銇汇亞銇堛倱")),
+ (1141, ("姘告不", "銇堛亜銇�")),
+ (1142, ("搴锋不", "銇撱亞銇�")),
+ (1144, ("澶╅", "銇︺倱銈堛亞")),
+ (1145, ("涔呭畨", "銇嶃倕銇嗐亗銈�")),
+ (1151, ("浠佸钩", "銇倱銇恒亜")),
+ (1154, ("涔呭", "銇嶃倕銇嗐仒銈�")),
+ (1156, ("淇濆厓", "銇汇亞銇掋倱")),
+ (1159, ("骞虫不", "銇搞亜銇�")),
+ (1160, ("姘告殾", "銇堛亜銈娿們銇�")),
+ (1161, ("蹇滀繚", "銇娿亞銇汇亞")),
+ (1163, ("闀峰瘺", "銇°倗銇嗐亱銈�")),
+ (1165, ("姘镐竾", "銇堛亜銇俱倱")),
+ (1166, ("浠佸畨", "銇倱銇傘倱")),
+ (1169, ("鍢夊繙", "銇嬨亰銇�")),
+ (1171, ("鎵垮畨", "銇椼倗銇嗐亗銈�")),
+ (1175, ("瀹夊厓", "銇傘倱銇掋倱")),
+ (1177, ("娌绘壙", "銇樸仐銈囥亞")),
+ (1181, ("椁婂拰", "銈堛亞銈�")),
+ (1182, ("瀵挎案", "銇樸倕銇堛亜")),
+ (1184, ("鍏冩殾", "銇掋倱銈娿們銇�")),
+ (1185, ("鏂囨不", "銇躲倱銇�")),
+ (1190, ("寤轰箙", "銇戙倱銇嶃倕銇�")),
+ (1199, ("姝ｆ不", "銇椼倗銇嗐仒")),
+ (1201, ("寤轰粊", "銇戙倱銇倱")),
+ (1204, ("鍏冧箙", "銇掋倱銇嶃倕銇�")),
+ (1206, ("寤烘案", "銇戙倱銇堛亜")),
+ (1207, ("鎵垮厓", "銇樸倗銇嗐亽銈�")),
+ (1211, ("寤烘殾", "銇戙倱銈娿們銇�")),
+ (1214, ("寤轰繚", "銇戙倱銇姐亞")),
+ (1219, ("鎵夸箙", "銇樸倗銇嗐亶銈呫亞")),
+ (1222, ("璨炲繙", "銇樸倗銇嗐亰銇�")),
+ (1225, ("鍏冧粊", "銇掋倱銇倱")),
+ (1225, ("鍢夌", "銇嬨倣銇�")),
+ (1228, ("瀹夎矠", "銇傘倱銇︺亜")),
+ (1229, ("瀵涘枩", "銇嬨倱銇�")),
+ (1232, ("璨炴案", "銇樸倗銇嗐亪銇�")),
+ (1233, ("澶╃", "銇︺倱銇枫亸")),
+ (1235, ("鏂囨殾", "銇躲倱銈娿們銇�")),
+ (1235, ("鍢夌", "銇嬨仸銇�")),
+ (1239, ("鏆︿粊", "銈娿們銇忋伀銈�")),
+ (1239, ("寤跺繙", "銇堛倱銇娿亞")),
+ (1240, ("浠佹不", "銇倱銇�")),
+ (1243, ("瀵涘厓", "銇嬨倱銇掋倱")),
+ (1247, ("瀹濇不", "銇汇亞銇�")),
+ (1249, ("寤洪暦", "銇戙倱銇°倗銇�")),
+ (1256, ("搴峰厓", "銇撱亞銇掋倱")),
+ (1257, ("姝ｅ槈", "銇椼倗銇嗐亱")),
+ (1259, ("姝ｅ厓", "銇椼倗銇嗐亽銈�")),
+ (1260, ("鏂囧繙", "銇躲倱銇娿亞")),
+ (1261, ("寮橀暦", "銇撱亞銇°倗銇�")),
+ (1264, ("鏂囨案", "銇躲倱銇堛亜")),
+ (1275, ("鍋ユ不", "銇戙倱銇�")),
+ (1278, ("寮樺畨", "銇撱亞銇傘倱")),
+ (1288, ("姝ｅ繙", "銇椼倗銇嗐亰銇�")),
+ (1293, ("姘镐粊", "銇堛亜銇倱")),
+ (1299, ("姝ｅ畨", "銇椼倗銇嗐亗銈�")),
+ (1303, ("涔惧厓", "銇戙倱銇掋倱")),
+ (1303, ("鍢夊厓", "銇嬨亽銈�")),
+ (1307, ("寰虫不", "銇ㄣ亸銇�")),
+ (1308, ("寤舵叾", "銇堛倱銇嶃倗銇�")),
+ (1311, ("蹇滈暦", "銇娿亞銇°倗銇�")),
+ (1312, ("姝ｅ拰", "銇椼倗銇嗐倧")),
+ (1317, ("鏂囦繚", "銇躲倱銇姐亞")),
+ (1319, ("鍏冨繙", "銇掋倱銇娿亞")),
+ (1321, ("鍏冧酣", "銇掋倱銇撱亞")),
+ (1325, ("姝ｄ腑", "銇椼倗銇嗐仭銈�")),
+ (1326, ("鍢夋殾", "銇嬨倞銈冦亸")),
+ (1329, ("鍏冨境", "銇掋倱銇ㄣ亸")),
+ (1331, ("鍏冨紭", "銇掋倱銇撱亞")),
+ (1332, ("姝ｆ叾", "銇椼倗銇嗐亼銇�")),
+ (1334, ("寤烘", "銇戙倱銈�")),
+ (1336, ("寤跺厓", "銇堛亜銇掋倱")),
+ (1338, ("鏆﹀繙", "銈娿們銇忋亰銇�")),
+ (1340, ("鑸堝浗", "銇撱亞銇撱亸")),
+ (1342, ("搴锋案", "銇撱亞銇堛亜")),
+ (1345, ("璨炲拰", "銇樸倗銇嗐倧")),
+ (1347, ("姝ｅ钩", "銇椼倗銇嗐伕銇�")),
+ (1350, ("瑕冲繙", "銇嬨倱銇娿亞")),
+ (1352, ("鏂囧拰", "銇躲倱銇�")),
+ (1356, ("寤舵枃", "銇堛倱銇躲倱")),
+ (1361, ("搴峰畨", "銇撱亞銇傘倱")),
+ (1362, ("璨炴不", "銇樸倗銇嗐仒")),
+ (1368, ("蹇滃畨", "銇娿亞銇傘倱")),
+ (1370, ("寤哄境", "銇戙倱銇ㄣ亸")),
+ (1372, ("鏂囦腑", "銇躲倱銇°倕銇�")),
+ (1375, ("姘稿拰", "銇堛亜銈�")),
+ (1375, ("澶╂巿", "銇︺倱銇樸倕")),
+ (1379, ("搴锋殾", "銇撱亞銈娿們銇�")),
+ (1381, ("姘稿境", "銇堛亜銇ㄣ亸")),
+ (1381, ("寮樺拰", "銇撱亞銈�")),
+ (1384, ("鑷冲境", "銇椼仺銇�")),
+ (1384, ("鍏冧腑", "銇掋倱銇°倕銇�")),
+ (1387, ("鍢夋叾", "銇嬨亼銇�")),
+ (1389, ("搴峰繙", "銇撱亞銇娿亞")),
+ (1390, ("鏄庡境", "銈併亜銇ㄣ亸")),
+ (1394, ("蹇滄案", "銇娿亞銇堛亜")),
+ (1428, ("姝ｉ暦", "銇椼倗銇嗐仭銈囥亞")),
+ (1429, ("姘镐韩", "銇堛亜銇嶃倗銇�")),
+ (1441, ("鍢夊悏", "銇嬨亶銇�")),
+ (1444, ("鏂囧畨", "銇躲倱銇傘倱")),
+ (1449, ("瀹濆境", "銇汇亞銇ㄣ亸")),
+ (1452, ("浜境", "銇嶃倗銇嗐仺銇�")),
+ (1455, ("搴锋", "銇撱亞銇椼倗銇�")),
+ (1457, ("闀风", "銇°倗銇嗐倣銇�")),
+ (1461, ("瀵涙", "銇嬨倱銇椼倗銇�")),
+ (1466, ("鏂囨", "銇躲倱銇椼倗銇�")),
+ (1467, ("蹇滀粊", "銇娿亞銇倱")),
+ (1469, ("鏂囨槑", "銇躲倱銈併亜")),
+ (1487, ("闀蜂韩", "銇°倗銇嗐亶銈囥亞")),
+ (1489, ("寤跺境", "銇堛倱銇ㄣ亸")),
+ (1492, ("鏄庡繙", "銈併亜銇娿亞")),
+ (1501, ("鏂囦簚", "銇躲倱銇�")),
+ (1504, ("姘告", "銇堛亜銇椼倗銇�")),
+ (1521, ("澶ф案", "銇犮亜銇堛亜")),
+ (1528, ("浜", "銇嶃倗銇嗐倣銇�")),
+ (1532, ("澶╂枃", "銇︺倱銇躲倱")),
+ (1555, ("寮樻不", "銇撱亞銇�")),
+ (1558, ("姘哥", "銇堛亜銈嶃亸")),
+ (1570, ("鍏冧簚", "銇掋倱銇�")),
+ (1573, ("澶╂", "銇︺倱銇椼倗銇�")),
+ (1593, ("鏂囩", "銇躲倱銈嶃亸")),
+ (1596, ("鎱堕暦", "銇戙亜銇°倗銇�")),
+ (1615, ("鍏冨拰", "銇掋倱銇�")),
+ (1624, ("瀵涙案", "銇嬨倱銇堛亜")),
+ (1645, ("姝ｄ繚", "銇椼倗銇嗐伝銇�")),
+ (1648, ("鎱跺畨", "銇戙亜銇傘倱")),
+ (1652, ("鎵垮繙", "銇樸倗銇嗐亰銇�")),
+ (1655, ("鏄庢殾", "銈併亜銈屻亶")),
+ (1658, ("涓囨不", "銇俱倱銇�")),
+ (1661, ("瀵涙枃", "銇嬨倱銇躲倱")),
+ (1673, ("寤跺疂", "銇堛倱銇姐亞")),
+ (1681, ("澶╁拰", "銇︺倱銇�")),
+ (1684, ("璨炰韩", "銇樸倗銇嗐亶銈囥亞")),
+ (1688, ("鍏冪", "銇掋倱銈嶃亸")),
+ (1704, ("瀹濇案", "銇汇亞銇堛亜")),
+ (1711, ("姝ｅ境", "銇椼倗銇嗐仺銇�")),
+ (1716, ("浜繚", "銇嶃倗銇嗐伝銇�")),
+ (1736, ("鍏冩枃", "銇掋倱銇躲倱")),
+ (1741, ("瀵涗繚", "銇嬨倱銇姐亞")),
+ (1744, ("寤朵韩", "銇堛倱銇嶃倗銇�")),
+ (1748, ("瀵涘欢", "銇嬨倱銇堛倱")),
+ (1751, ("瀹濇殾", "銇汇亞銈屻亶")),
+ (1764, ("鏄庡拰", "銈併亜銈�")),
+ (1773, ("瀹夋案", "銇傘倱銇堛亜")),
+ (1781, ("澶╂槑", "銇︺倱銈併亜")),
+ (1801, ("瀵涙斂", "銇嬨倱銇涖亜")),
+ (1802, ("浜拰", "銇嶃倗銇嗐倧")),
+ (1804, ("鏂囧寲", "銇躲倱銇�")),
+ (1818, ("鏂囨斂", "銇躲倱銇涖亜")),
+ (1831, ("澶╀繚", "銇︺倱銇姐亞")),
+ (1845, ("寮樺寲", "銇撱亞銇�")),
+ (1848, ("鍢夋案", "銇嬨亪銇�")),
+ (1855, ("瀹夋斂", "銇傘倱銇涖亜")),
+ (1860, ("涓囧欢", "銇俱倱銇堛亜")),
+ (1861, ("鏂囦箙", "銇躲倱銇嶃倕銇�")),
+ (1864, ("鍏冩不", "銇掋倱銇�")),
+ (1865, ("鎱跺繙", "銇戙亜銇娿亞")),
+ (1868, ("鏄庢不", "銈併亜銇�")),
+ (1912, ("澶ф", "銇熴亜銇椼倗銇�")),
+ (1926, ("鏄拰", "銇椼倗銇嗐倧")),
+ (1989, ("骞虫垚", "銇搞亜銇涖亜")),
+]
+
+
+class Num2Word_JA(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'JPY': (('鍐�', '銇堛倱'), ()),
+ }
+
+ def set_high_numwords(self, high):
+ max = 4 * len(high)
+ for word, n in zip(high, range(max, 0, -4)):
+ self.cards[10 ** n] = word
+
+ def setup(self):
+ self.negword = "銉炪偆銉娿偣"
+ self.pointword = ("鐐�", "銇︺倱")
+ self.exclude_title = ["鐐�", "銉炪偆銉娿偣"]
+
+ self.high_numwords = [
+ ("涓�", "銇俱倱"), # 10**4 man
+ ("鍎�", "銇娿亸"), # 10**8 oku
+ ("鍏�", "銇°倗銇�"), # 10**12 ch艒
+ ("浜�", "銇戙亜"), # 10**16 kei
+ ("鍨�", "銇屻亜"), # 10**20 gai
+ ("绉�", "銇�"), # 10**24 shi
+ ("绌�", "銇樸倗銇�"), # 10**28 j艒
+ ("婧�", "銇撱亞"), # 10**32 k艒
+ ("婢�", "銇嬨倱"), # 10**36 kan
+ ("姝�", "銇涖亜"), # 10**40 sei
+ ("杓�", "銇曘亜"), # 10**44 sai
+ ("妤�", "銇斻亸"), # 10**48 goku
+ ]
+
+ self.high_numwords.reverse()
+
+ self.mid_numwords = [
+ (1000, ("鍗�", "銇涖倱")),
+ (100, ("鐧�", "銇层們銇�")),
+ ]
+
+ self.low_numwords = [
+ ("鍗�", "銇樸倕銇�"), # 10 j奴
+ ("涔�", "銇嶃倕銇�"), # 9 ky奴
+ ("鍏�", "銇仭"), # 8 hachi
+ ("涓�", ("銇仾", "銇椼仭")), # 7 nana, shichi
+ ("鍏�", "銈嶃亸"), # 6 roku
+ ("浜�", "銇�"), # 5 go
+ ("鍥�", ("銈堛倱", "銇�")), # 4 yon, shi
+ ("涓�", "銇曘倱"), # 3 san
+ ("浜�", "銇�"), # 2 ni
+ ("涓�", "銇勩仭"), # 1 ichi
+ # both are alternatives, 闆� doesn't map to 銈笺儹 or 銆� to 銈屻亜
+ (("闆�", "銆�"), ("銈笺儹", "銈屻亜")), # 0 ZERO, rei
+ ]
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+
+ fmt = "%s%s"
+ # ignore lpair if lnum is 1 and rnum is less than 10000
+ if lnum == 1 and rnum < 10000:
+ return rpair
+ # rnum is added to lnum
+ elif lnum > rnum:
+ return (fmt % (ltext, rtext), lnum + rnum)
+ # rnum is multiplied by lnum
+ elif lnum < rnum:
+ return rendaku_merge_pairs(lpair, rpair)
+
+ def _ordinal_suffix(self, reading, counter):
+ if reading:
+ if counter == "鐣�":
+ return "銇般倱銈�"
+ else:
+ raise NotImplementedError(
+ "Reading not implemented for %s" % counter)
+ else:
+ return counter + "鐩�"
+
+ def to_ordinal(self, value, reading=False, prefer=None, counter="鐣�"):
+ self.verify_ordinal(value)
+ base = self.to_cardinal(value, reading=reading, prefer=prefer)
+ return "%s%s" % (base, self._ordinal_suffix(reading, counter))
+
+ def to_ordinal_num(self, value, reading=False, counter="鐣�"):
+ return "%s%s" % (value, self._ordinal_suffix(reading, counter))
+
+ def to_year(self, val, suffix=None, longval=True, reading=False,
+ prefer=None, era=True):
+ year = val
+ # Gregorian calendar
+ if not era:
+ prefix = ""
+ if year < 0:
+ year = abs(year)
+ prefix = "銇嶃亽銈撱仠銈�" if reading else "绱�鍏冨墠"
+
+ year_words = self.to_cardinal(year, reading=reading, prefer=prefer)
+ if reading and year % 10 == 9:
+ year_words = year_words[:-3] + "銇�"
+
+ return "%s%s%s" % (prefix, year_words, "銇倱" if reading else "骞�")
+
+ # Era calendar (default)
+ min_year = ERA_START[0][0]
+ last_era_idx = len(ERA_START) - 1
+ if year < min_year:
+ raise ValueError(
+ "Can't convert years less than %s to era" % min_year)
+
+ first = 0
+ last = last_era_idx
+ era_idx = None
+ while era_idx is None:
+ mid = (first + last) // 2
+ if mid == last_era_idx or (ERA_START[mid][0] <= year and
+ ERA_START[mid + 1][0] > year):
+ era_idx = mid
+ # if an era lasting less than a year is preferred, choose it
+ if prefer:
+ i = mid - 1
+ while i >= 0 and ERA_START[i][0] == year:
+ # match kanji or hiragana
+ if set(ERA_START[i][1]) & set(prefer):
+ era_idx = i
+ break
+ i -= 1
+
+ # ends up at the last index where year >= ERA_START[mid][0]
+ if year < ERA_START[mid][0]:
+ last = mid - 1
+ else:
+ first = mid + 1
+
+ era = ERA_START[era_idx]
+ era_name = era[1][0]
+ era_year = year - era[0] + 1
+ fmt = "%s%s骞�"
+ if reading == "arabic":
+ era_year_words = str(era_year)
+ elif reading:
+ era_name = era[1][1]
+ era_year_words = (self.to_cardinal(era_year, reading=True,
+ prefer=prefer)
+ if era_year != 1 else "銇屻倱")
+ if era_year % 10 == 9:
+ era_year_words = era_year_words[:-3] + "銇�"
+ fmt = "%s%s銇倱"
+ else:
+ era_year_words = (self.to_cardinal(era_year, reading=False,
+ prefer=prefer)
+ if era_year != 1 else "鍏�")
+
+ return fmt % (era_name, era_year_words)
+
+ def to_currency(self, val, currency="JPY", cents=False, separator="",
+ adjective=False, reading=False, prefer=None):
+ left, right, is_negative = parse_currency_parts(
+ val, is_int_with_cents=cents)
+
+ try:
+ cr1, cr2 = self.CURRENCY_FORMS[currency]
+ if (cents or abs(val) != left) and not cr2:
+ raise ValueError('Decimals not supported for "%s"' % currency)
+ except KeyError:
+ raise NotImplementedError(
+ 'Currency code "%s" not implemented for "%s"' %
+ (currency, self.__class__.__name__))
+
+ if adjective and currency in self.CURRENCY_ADJECTIVES:
+ cr1 = prefix_currency(self.CURRENCY_ADJECTIVES[currency], cr1)
+
+ minus_str = self.negword if is_negative else ""
+
+ return '%s%s%s%s%s' % (
+ minus_str,
+ self.to_cardinal(left, reading=reading, prefer=prefer),
+ cr1[1] if reading else cr1[0],
+ self.to_cardinal(right, reading=reading, prefer=prefer)
+ if cr2 else '',
+ (cr2[1] if reading else cr2[0]) if cr2 else '',
+ )
+
+ def splitnum(self, value, reading, prefer):
+ for elem in self.cards:
+ if elem > value:
+ continue
+
+ out = []
+ if value == 0:
+ div, mod = 1, 0
+ else:
+ div, mod = divmod(value, elem)
+
+ if div == 1:
+ out.append((select_text(self.cards[1], reading, prefer), 1))
+ else:
+ if div == value: # The system tallies, eg Roman Numerals
+ return [(
+ div * select_text(self.cards[elem], reading, prefer),
+ div * elem)]
+ out.append(self.splitnum(div, reading, prefer))
+
+ out.append((select_text(self.cards[elem], reading, prefer), elem))
+
+ if mod:
+ out.append(self.splitnum(mod, reading, prefer))
+
+ return out
+
+ def to_cardinal(self, value, reading=False, prefer=None):
+ try:
+ assert int(value) == value
+ except (ValueError, TypeError, AssertionError):
+ return self.to_cardinal_float(value, reading=reading,
+ prefer=prefer)
+
+ out = ""
+ if value < 0:
+ value = abs(value)
+ out = self.negword
+
+ if value >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig % (value, self.MAXVAL))
+
+ val = self.splitnum(value, reading, prefer)
+ words, _ = self.clean(val)
+ return self.title(out + words)
+
+ def to_cardinal_float(self, value, reading=False, prefer=None):
+ prefer = prefer or ["銈屻亜"]
+ try:
+ float(value) == value
+ except (ValueError, TypeError, AssertionError):
+ raise TypeError(self.errmsg_nonnum % value)
+
+ pre, post = self.float2tuple(float(value))
+
+ post = str(post)
+ post = '0' * (self.precision - len(post)) + post
+
+ out = [self.to_cardinal(pre, reading=reading, prefer=prefer)]
+ if self.precision:
+ out.append(self.title(self.pointword[1 if reading else 0]))
+
+ for i in range(self.precision):
+ curr = int(post[i])
+ out.append(to_s(
+ self.to_cardinal(curr, reading=reading, prefer=prefer)))
+
+ return "".join(out)
diff --git a/fun_text_processing/num2words/num2words/lang_KH.py b/fun_text_processing/num2words/num2words/lang_KH.py
new file mode 100644
index 0000000..d7345dc
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_KH.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# Khmer
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+# Khmer
+
+class Num2Word_KH(lang_EU.Num2Word_EU):
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 3, -1)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+ # self.cards[10**n] = word + "釣涐灦釣�"
+ # try:
+ # ordinal_word = self.high_numwords[high]
+ # except KeyError:
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # print(word)
+ # print(n)
+ # self.cards[10 ** n] = word
+
+ def gen_high_numwords(self, units, tens, lows):
+ out = [u + t for t in tens for u in units]
+ out.reverse()
+ return out + lows
+
+ def setup(self):
+ super(Num2Word_KH, self).setup()
+
+ self.negword = "釣娽瀫 "
+ self.pointword = "釣呩焼釣庒灮釣�"
+ self.exclude_title = ["釣撫灧釣�", "釣呩焼釣庒灮釣�", "釣娽瀫"]
+ lows = [""]
+ units = [""]
+ tens = ["釣栣灦釣撫煁釣涐灦釣�","釣氠灆釣涐灦釣�","釣�釤勧瀶釣�","釣涐灦釣�","釣熱焸釣�","釣樶焿釣横灀"]
+ self.high_numwords = self.gen_high_numwords(units, tens, lows)
+
+ self.high_numwords = [(1000000000, "釣樶灲釣欋灃釣夺灀釤嬦灈釣夺灀"),
+ (100000000, "釣樶灲釣欋灇釣欋灈釣夺灀"),
+ (10000000, "釣樶灲釣欋瀫釤勧瀶釣�"),
+ (1000000, "釣樶灲釣欋灈釣夺灀"),
+ (100000, "釣樶灲釣欋灍釤傖灀"),
+ (10000, "釣樶灲釣欋灅釤夅灪釣�")]
+ # (2400000, "釣栣灨釣氠灈釣夺灀釣斸灲釣撫灅釤夅灪釣�")
+
+ self.mid_numwords = [(1000, "釣樶灲釣欋灃釣夺灀釤�"), (100, "釣樶灲釣欋灇釣�"),
+ (90, "釣�釤呩灍釣丰灁"), (80, "釣斸焿釤傖瀼釣熱灧釣�"), (70, "釣呩灧釣忈灍釣丰灁"),
+ (60, "釣犪灮釣�釣熱灧釣�"), (50, "釣犪灦釣熱灧釣�"), (40, "釣熱焸釣熱灧釣�"),
+ (30, "釣熱灦釣樶灍釣丰灁")]
+
+ self.low_numwords = ["釣樶煉釣椺焹",
+ "釣娽灁釤嬦灁釤掅灇釣夺焼釣斸灲釣�",
+ "釣娽灁釤嬦灁釤掅灇釣夺焼釣斸灨",
+ "釣娽灁釤嬦灁釤掅灇釣夺焼釣栣灨釣�",
+ "釣娽灁釤嬦灁釤掅灇釣夺焼釣樶灲釣�",
+ "釣娽灁釤嬦灁釤掅灇釣夺焼",
+ "釣娽灁釤嬦灁釣结灀",
+ "釣娽灁釤嬦灁釣�",
+ "釣娽灁釤嬦灃釣羔灇",
+ "釣娽灁釤嬦灅釣结灆",
+ "釣娽灁釤�",
+ "釣斸煉釣氠灦釤嗎灁釣结灀",
+ "釣斸煉釣氠灦釤嗎灁釣�",
+ "釣斸煉釣氠灦釤嗎灃釣羔灇",
+ "釣斸煉釣氠灦釤嗎灅釣结灆",
+ "釣斸煉釣氠灦釤�",
+ "釣斸灲釣�",
+ "釣斸灨",
+ "釣栣灨釣�",
+ "釣樶灲釣�",
+ "釣熱灱釣撫煉釣�"]
+
+ self.ords = {"釣樶灲釣�": "釣戓灨釣樶灲釣�",
+ "釣栣灨釣�": "釣戓灨釣栣灨釣�",
+ "釣斸灨": "釣戓灨釣斸灨",
+ "釣斸灲釣�": "釣戓灨釣斸灲釣�",
+ "釣斸煉釣氠灦釤�": "釣戓灨釣斸煉釣氠灦釤�",
+ "釣斸煉釣氠灦釤嗎灅釣结灆": "釣戓灨釣斸煉釣氠灦釤嗎灅釣结灆",
+ "釣斸煉釣氠灦釤嗎灃釣羔灇": "釣戓灨釣斸煉釣氠灦釤嗎灃釣羔灇",
+ "釣斸煉釣氠灦釤嗎灁釣�": "釣戓灨釣斸煉釣氠灦釤嗎灁釣�",
+ "釣斸煉釣氠灦釤嗎灁釣结灀": "釣戓灨釣斸煉釣氠灦釤嗎灁釣结灀",
+ "釣娽灁釤�": "釣戓灨釣娽灁釤�",
+ "釣娽灁釤嬦灅釣结灆": "釣戓灨釣娽灁釤嬦灅釣结灆",
+ "釣娽灁釤嬦灃釣羔灇": "釣戓灨釣娽灁釤嬦灃釣羔灇"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = "釣戓灨" + value
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '釣樶灮釣撫瀭' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "釣氠灆"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_KN.py b/fun_text_processing/num2words/num2words/lang_KN.py
new file mode 100644
index 0000000..2f89872
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_KN.py
@@ -0,0 +1,167 @@
+# -*- encoding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_KN(Num2Word_EU):
+ def set_high_numwords(self, high):
+ for n, word in self.high_numwords:
+ self.cards[10**n] = word
+
+ def setup(self):
+ self.low_numwords = [
+ "嗖む硦嗖傕铂嗖む硩嗖む硦嗖傕铂嗖む硩嗖む硜",
+ "嗖む硦嗖傕铂嗖む硩嗖む硢嗖傕矡喑�",
+ "嗖む硦嗖傕铂嗖む硩嗖む硣嗖赤硜",
+ "嗖む硦嗖傕铂嗖む硩嗖む簿嗖班硜",
+ "嗖む硦嗖傕铂嗖む硩嗖む硤嗖︵硜",
+ "嗖む硦嗖傕铂嗖む硩嗖� 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖む硦嗖傕铂嗖む硩嗖� 嗖硞嗖班硜",
+ "嗖む硦嗖傕铂嗖む硩嗖む硢嗖班病喑�",
+ "嗖む硦嗖傕铂嗖む硩嗖む硦嗖傕拨喑�",
+ "嗖む硢喑傕矀嗖菠喑嵿菠喑�",
+ "嗖庎矀嗖菠喑嵿菠喑娻矀嗖菠喑嵿菠喑�",
+ "嗖庎矀嗖菠喑嵿菠喑嗋矀嗖熰硜",
+ "嗖庎矀嗖菠喑嵿菠喑囙渤喑�",
+ "嗖庎矀嗖菠喑嵿菠嗖距舶喑�",
+ "嗖庎矀嗖菠喑嵿菠喑堗拨喑�",
+ "嗖庎矀嗖菠喑嵿菠喑� 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖庎矀嗖菠喑嵿菠喑� 嗖硞嗖班硜",
+ "嗖庎矀嗖菠喑嵿菠喑嗋舶嗖∴硜",
+ "嗖庎矀嗖菠喑嵿菠喑嗋硞嗖傕拨喑�",
+ "嗖庎矀嗖菠喑嵿菠喑�",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硦嗖傕铂嗖む硩嗖む硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硢嗖傕矡喑�",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硣嗖赤硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む簿嗖班硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硤嗖︵硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硩 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硩 嗖硞嗖班硜",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硢嗖班病喑�",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硦嗖傕拨喑�",
+ "嗖庎勃喑嵿勃嗖む硩嗖む硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑娻矀嗖菠喑嵿菠喑�",
+ "嗖呧舶嗖掂菠喑嵿菠喑嗋矀嗖熰硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑囙渤喑�",
+ "嗖呧舶嗖掂菠喑嵿菠嗖距舶喑�",
+ "嗖呧舶嗖掂菠喑嵿菠喑堗拨喑�",
+ "嗖呧舶嗖掂菠喑嵿菠喑� 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑� 嗖硞嗖班硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑嗋舶嗖∴硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑娻矀嗖︵硜",
+ "嗖呧舶嗖掂菠喑嵿菠喑�",
+ "嗖愢驳嗖む硩嗖む硦嗖傕铂嗖む硩嗖む硜",
+ "嗖愢驳嗖む硩嗖む硢嗖傕矡喑�",
+ "嗖愢驳嗖む硩嗖む硣嗖赤硜",
+ "嗖愢驳嗖む硩嗖む簿嗖班硜",
+ "嗖愢驳嗖む硩嗖む硤嗖︵硜",
+ "嗖愢驳嗖む硩嗖む硩嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖愢驳嗖む硩嗖む伯喑傕舶喑�",
+ "嗖愢驳嗖む硩嗖む硢嗖班病喑�",
+ "嗖愢驳嗖む硩嗖む硦嗖傕拨喑�",
+ "嗖愢驳嗖む硩嗖む硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑娻矀嗖菠喑嵿菠喑�",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑嗋矀嗖熰硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑囙渤喑�",
+ "嗖ㄠ膊嗖掂菠喑嵿菠嗖距舶喑�",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑嗋硸嗖︵硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑� 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑� 嗖硞嗖班硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑� 嗖庎舶嗖∴硜",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑嗋硞嗖傕拨喑�",
+ "嗖ㄠ膊嗖掂菠喑嵿菠喑�",
+ "嗖硞嗖掂菠喑嵿菠喑� 嗖掄矀嗖菠喑嵿菠喑�",
+ "嗖硞嗖掂菠喑嵿矌嗖傕矡喑�",
+ "嗖硞嗖掂菠喑嵿矎嗖赤硜",
+ "嗖硞嗖掂菠喑嵿矄嗖班硜",
+ "嗖硞嗖掂菠喑嵿菠喑� 嗖愢拨喑�",
+ "嗖硞嗖掂菠喑嵿菠喑� 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖硞嗖掂菠喑嵿菠喑� 嗖硞嗖班硜",
+ "嗖硞嗖掂菠喑嵿菠喑嵿矌嗖班病喑�",
+ "嗖硞嗖掂菠喑嵿菠喑嵿矑嗖傕拨喑�",
+ "嗖硞嗖掂菠喑嵿菠喑�",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩嗖掄矀嗖菠喑嵿菠喑�",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩嗖庎矀嗖熰硜",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩嗖忇渤喑�",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩嗖嗋舶喑�",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩 嗖愢拨喑�",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩 嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩 嗖硞嗖班硜",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩 嗖庎舶嗖∴硜",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硩 嗖掄矀嗖︵硜",
+ "嗖囙勃喑嵿勃嗖む硩嗖む硜",
+ "嗖灌菠喑嵿菠喑娻矀嗖菠喑嵿菠喑�",
+ "嗖灌拨嗖苦波喑嗋矀嗖熰硜",
+ "嗖灌拨嗖苦波喑嗋硶嗖赤硜",
+ "嗖灌拨嗖苦波嗖距舶喑�",
+ "嗖灌拨嗖苦波喑嗋硸嗖︵硜",
+ "嗖灌拨嗖苦波嗖距膊喑嵿矔喑�",
+ "嗖灌拨嗖苦伯喑傕舶喑�",
+ "嗖灌波喑嵿波喑嗋舶嗖∴硜",
+ "嗖灌波喑嵿波喑嗋硞嗖傕拨喑�",
+ "嗖灌菠喑嵿菠喑�",
+ "嗖掄矀嗖菠喑嵿菠喑�",
+ "嗖庎矀嗖熰硜",
+ "嗖忇渤喑�",
+ "嗖嗋舶喑�",
+ "嗖愢拨喑�",
+ "嗖ㄠ簿嗖侧硩嗖曕硜",
+ "嗖硞嗖班硜",
+ "嗖庎舶嗖∴硜",
+ "嗖掄矀嗖︵硜",
+ "嗖膏硦嗖ㄠ硩嗖ㄠ硢",
+ ]
+
+ self.mid_numwords = [(100, "嗖ㄠ硞嗖班硜")]
+
+ self.high_numwords = [(7, "嗖曕硦喑曕矡嗖�"), (5, "嗖掄矀嗖︵硜 嗖侧矔喑嵿卜"), (3, "嗖膏簿嗖掂部嗖�")]
+
+ self.pointword = "嗖部嗖傕拨喑�"
+
+ self.modifiers = [
+ "喑�",
+ "嗖�",
+ "嗖�",
+ "嗖苦硶",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ "喑�",
+ ]
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ if ltext[-1] in self.modifiers:
+ return ("%s %s" % (ltext[:-1], rtext), lnum + rnum)
+ else:
+ return ("%s %s" % (ltext + "嗖�", rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value)
+ if outwords[-1] in self.modifiers:
+ outwords = outwords[:-1]
+ ordinal_num = outwords + "嗖ㄠ硣"
+ return ordinal_num
diff --git a/fun_text_processing/num2words/num2words/lang_KO.py b/fun_text_processing/num2words/num2words/lang_KO.py
new file mode 100644
index 0000000..eeafaa2
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_KO.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from .base import Num2Word_Base
+from .currency import parse_currency_parts
+
+
+class Num2Word_KO(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'KRW': ('鞗�', None),
+ 'USD': ('雼煬', '靹柬姼'),
+ 'JPY': ('鞐�', None)
+ }
+
+ def set_high_numwords(self, high):
+ max = 4 * len(high)
+ for word, n in zip(high, range(max, 0, -4)):
+ self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_KO, self).setup()
+
+ self.negword = "毵堨澊雱堨姢 "
+ self.pointword = "鞝�"
+
+ self.high_numwords = [
+ '氍措焿雽�靾�',
+ '攵堦皜靷潣',
+ '雮橃湢韮�',
+ '鞎勳姽旮�',
+ '頃晿靷�',
+ '攴�',
+ '鞛�',
+ '鞝�',
+ '臧�',
+ '甑�',
+ '鞏�',
+ '鞛�',
+ '頃�',
+ '瓴�',
+ '臁�',
+ '鞏�',
+ '毵�']
+ self.mid_numwords = [(1000, "觳�"), (100, "氚�")]
+ self.low_numwords = ["鞁�", "甑�", "韺�", "旃�", "鞙�", "鞓�", "靷�", "靷�", "鞚�",
+ "鞚�", "鞓�"]
+ self.ords = {"鞚�": "頃�",
+ "鞚�": "霊�",
+ "靷�": "靹�",
+ "靷�": "雱�",
+ "鞓�": "雼れ劘",
+ "鞙�": "鞐劘",
+ "旃�": "鞚缄潮",
+ "韺�": "鞐崯",
+ "甑�": "鞎勴檳",
+ "鞁�": "鞐�",
+ "鞚挫嫮": "鞀る",
+ "靷检嫮": "靹滊ジ",
+ "靷嫮": "毵堩潝",
+ "鞓れ嫮": "靿�",
+ "鞙§嫮": "鞓堨垳",
+ "旃犾嫮": "鞚柬潝",
+ "韺旍嫮": "鞐摖",
+ "甑嫮": "鞎勴潝"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum <= 10000:
+ return rpair
+ elif 10000 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 10000 and lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ else:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ if(value == 1):
+ return "觳� 氩堨Ц"
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("氚�")
+ if "鞁�" in lastwords[-1]:
+ ten_one = lastwords[-1].split("鞁�")
+ ten_one[0] = self.ords[ten_one[0] + "鞁�"]
+ try:
+ ten_one[1] = self.ords[ten_one[1]]
+ ten_one[0] = ten_one[0].replace("鞀る", "鞀る")
+ except KeyError:
+ pass
+ lastwords[-1] = ''.join(ten_one)
+ else:
+ lastwords[-1] = self.ords[lastwords[-1]]
+ outwords[-1] = "氚� ".join(lastwords)
+ return " ".join(outwords) + " 氩堨Ц"
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s 氩堨Ц" % (value)
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '旮办洂鞝�' if not suffix else suffix
+ valtext = self.to_cardinal(val)
+ return ("%s雲�" % valtext if not suffix
+ else "%s %s雲�" % (suffix, valtext))
+
+ def to_currency(self, val, currency="KRW", cents=False, separator="",
+ adjective=False):
+ left, right, is_negative = parse_currency_parts(
+ val, is_int_with_cents=cents)
+
+ try:
+ cr1, cr2 = self.CURRENCY_FORMS[currency]
+ if (cents or right) and not cr2:
+ raise ValueError('Decimals not supported for "%s"' % currency)
+ except KeyError:
+ raise NotImplementedError(
+ 'Currency code "%s" not implemented for "%s"' %
+ (currency, self.__class__.__name__))
+
+ minus_str = self.negword if is_negative else ""
+ return '%s%s%s%s%s' % (
+ minus_str,
+ ''.join(self.to_cardinal(left).split()),
+ cr1,
+ ' ' + self.to_cardinal(right)
+ if cr2 else '',
+ cr2 if cr2 else '',
+ )
diff --git a/fun_text_processing/num2words/num2words/lang_KZ.py b/fun_text_processing/num2words/num2words/lang_KZ.py
new file mode 100644
index 0000000..18f4a3b
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_KZ.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = '薪萤谢'
+
+ONES = {
+ 1: '斜褨褉',
+ 2: '械泻褨',
+ 3: '爷褕',
+ 4: '褌萤褉褌',
+ 5: '斜械褋',
+ 6: '邪谢褌褘',
+ 7: '卸械褌褨',
+ 8: '褋械谐褨蟹',
+ 9: '褌芯覔褘蟹',
+}
+
+TEN = '芯薪'
+
+TWENTIES = {
+ 2: '卸懈褘褉屑邪',
+ 3: '芯褌褘蟹',
+ 4: '覜褘褉褘覜',
+ 5: '械谢褍',
+ 6: '邪谢锌褘褋',
+ 7: '卸械褌锌褨褋',
+ 8: '褋械泻褋械薪',
+ 9: '褌芯覜褋邪薪',
+}
+
+HUNDRED = '卸爷蟹'
+
+THOUSANDS = {
+ 1: '屑褘遥',
+ 2: '屑懈谢谢懈芯薪',
+ 3: '屑懈谢谢懈邪褉写',
+ 4: '褌褉懈谢谢懈芯薪',
+ 5: '泻胁邪写褉懈谢谢懈芯薪',
+ 6: '泻胁懈薪褌懈谢谢懈芯薪',
+ 7: '褋械泻褋褌懈谢谢懈芯薪',
+ 8: '褋械锌褌懈谢谢懈芯薪',
+ 9: '芯泻褌懈谢谢懈芯薪',
+ 10: '薪芯薪懈谢谢懈芯薪',
+}
+
+
+class Num2Word_KZ(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'USD': ('写芯谢谢邪褉', '褑械薪褌'),
+ 'KZT': ('褌械遥谐械', '褌懈褘薪'),
+ }
+
+ def setup(self):
+ self.negword = "屑懈薪褍褋"
+ self.pointword = "斜爷褌褨薪"
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ return u'%s %s %s' % (
+ self._int2word(int(left)),
+ self.pointword,
+ (ZERO + ' ') * leading_zero_count + self._int2word(int(right))
+ )
+ else:
+ return self._int2word(int(n))
+
+ def pluralize(self, n, form):
+ return form
+
+ def _cents_verbose(self, number, currency):
+ return self._int2word(number, currency == 'KZT')
+
+ def _int2word(self, n, feminine=False):
+ if n < 0:
+ return ' '.join([self.negword, self._int2word(abs(n))])
+
+ if n == 0:
+ return ZERO
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ if n3 > 1:
+ words.append(ONES[n3])
+ words.append(HUNDRED)
+
+ if n2 == 1:
+ words.append(TEN)
+ elif n2 > 1:
+ words.append(TWENTIES[n2])
+
+ if n1 > 0:
+ words.append(ONES[n1])
+
+ if i > 0:
+ words.append(THOUSANDS[i])
+
+ return ' '.join(words)
+
+ def to_ordinal(self, number):
+ # TODO: Implement to_ordinal
+ raise NotImplementedError()
diff --git a/fun_text_processing/num2words/num2words/lang_LO.py b/fun_text_processing/num2words/num2words/lang_LO.py
new file mode 100644
index 0000000..a85c473
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_LO.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+# Lao number to words
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_LO(lang_EU.Num2Word_EU):
+ # GIGA_SUFFIX = "iljarder"
+ # MEGA_SUFFIX = "iljoner"
+
+ # def set_high_numwords(self, high):
+ # cap = 3 + 6 * len(high)
+
+ # for word, n in zip(high, range(cap, 3, -6)):
+ # if self.GIGA_SUFFIX:
+ # self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ # if self.MEGA_SUFFIX:
+ # self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 0, -3)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+
+ def setup(self):
+ super(Num2Word_LO, self).setup()
+
+ self.negword = "嗪ム夯嗪�"
+ self.pointword = "嗪堗焊嗪�"
+ self.exclude_title = ["嗷佮亥嗪�", "嗪堗焊嗪�", "嗪ム夯嗪�"]
+
+ self.high_numwords = [(10000000000, '嗪捍嗪氞簳嗪粪粔'),
+ (1000000000, '嗪簷嗪多粓嗪囙簳嗪粪粔'),
+ (100000000, '嗪簷嗪多粓嗪囙寒嗷夃涵嗪嵿亥嗷夃翰嗪�'),
+ (10000000,'嗪捍嗪氞亥嗷夃翰嗪�'),
+ (1000000, '嗪簷嗪多粓嗪囙亥嗷夃翰嗪�'),
+ (100000, '嗪簷嗪多粓嗪囙粊嗪簷'),
+ (1000, '嗪簷嗪多粓嗪囙簽嗪编簷') ]
+ self.mid_numwords = [(1000, "嗪簷嗪多粓嗪囙簽嗪编簷"), (100, "嗪簷嗪多粓嗪囙寒嗷夃涵嗪�"),
+ (90, "嗷�嗪佮夯嗷夃翰嗪捍嗪�"), (80, "嗷佮簺嗪斷邯嗪脆簹"), (70, "嗷�嗪堗罕嗪斷邯嗪脆簹"),
+ (60, "嗪夯嗪佮邯嗪脆簹"), (50, "嗪粔嗪侧邯嗪脆簹"), (40, "嗪旱嗷堗邯嗪脆簹"),
+ (30, "嗪翰嗪∴邯嗪脆簹")]
+
+ self.low_numwords = ['嗪娻翰嗪�', '嗪捍嗪氞粈嗪佮夯嗷夃翰', '嗪捍嗪氞粊嗪涏簲', '嗪捍嗪氞粈嗪堗罕嗪�', '嗪捍嗪氞韩嗪秽簛', '嗪捍嗪氞韩嗷夃翰', '嗪捍嗪氞邯嗪掂粓', '嗪捍嗪氞邯嗪侧骸', '嗪捍嗪氞邯嗪簢', '嗪捍嗪氞粈嗪罕嗪�', '嗪捍嗪歝', '嗷�嗪佮夯嗷夃翰', '嗷佮簺嗪�', '嗷�嗪堗罕嗪�', '嗪夯嗪�', '嗪粔嗪�', '嗪旱嗷�', '嗪翰嗪�', '嗪涵嗪�', '嗪簷嗪多粓嗪�', '嗪汗嗪�']
+
+
+ self.ords = {"嗪簷嗪多粓嗪�": "嗪椸粛嗪侧涵嗪脆簲",
+ "嗪涵嗪�": "嗪椸旱嗪涵嗪�",
+ "嗪翰嗪�": "嗪椸旱嗪翰嗪�",
+ "嗪旱嗷�": "嗪椸旱嗪旱嗷�",
+ "嗪粔嗪�": "嗪椸旱嗪粔嗪�",
+ "嗪夯嗪�":"嗪椸旱嗪夯嗪�",
+ "嗷�嗪堗罕嗪�": "嗪椸旱嗷�嗪堗罕嗪�",
+ "嗷佮簺嗪�": "嗪椸旱嗷佮簺嗪�",
+ "嗷�嗪佮夯嗷夃翰": "嗷�嗪佮夯嗷夃翰",
+ "嗪捍嗪歝": "嗪椸旱嗪捍嗪�"
+ }
+
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s %s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '嗪佮粓嗪簷嗪喊嗪曕喊嗪о罕嗪� ' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "嗪粔嗪簫"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_LT.py b/fun_text_processing/num2words/num2words/lang_LT.py
new file mode 100644
index 0000000..d04dedd
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_LT.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('nulis',)
+
+ONES_FEMININE = {
+ 1: ('viena',),
+ 2: ('dvi',),
+ 3: ('trys',),
+ 4: ('keturios',),
+ 5: ('penkios',),
+ 6: ('拧e拧ios',),
+ 7: ('septynios',),
+ 8: ('a拧tuonios',),
+ 9: ('devynios',),
+}
+
+ONES = {
+ 1: ('vienas',),
+ 2: ('du',),
+ 3: ('trys',),
+ 4: ('keturi',),
+ 5: ('penki',),
+ 6: ('拧e拧i',),
+ 7: ('septyni',),
+ 8: ('a拧tuoni',),
+ 9: ('devyni',),
+}
+
+TENS = {
+ 0: ('de拧imt',),
+ 1: ('vienuolika',),
+ 2: ('dvylika',),
+ 3: ('trylika',),
+ 4: ('keturiolika',),
+ 5: ('penkiolika',),
+ 6: ('拧e拧iolika',),
+ 7: ('septyniolika',),
+ 8: ('a拧tuoniolika',),
+ 9: ('devyniolika',),
+}
+
+TWENTIES = {
+ 2: ('dvide拧imt',),
+ 3: ('trisde拧imt',),
+ 4: ('keturiasde拧imt',),
+ 5: ('penkiasde拧imt',),
+ 6: ('拧e拧iasde拧imt',),
+ 7: ('septyniasde拧imt',),
+ 8: ('a拧tuoniasde拧imt',),
+ 9: ('devyniasde拧imt',),
+}
+
+HUNDRED = ('拧imtas', '拧imtai')
+
+THOUSANDS = {
+ 1: ('t奴kstantis', 't奴kstan膷iai', 't奴kstan膷i懦'),
+ 2: ('milijonas', 'milijonai', 'milijon懦'),
+ 3: ('milijardas', 'milijardai', 'milijard懦'),
+ 4: ('trilijonas', 'trilijonai', 'trilijon懦'),
+ 5: ('kvadrilijonas', 'kvadrilijonai', 'kvadrilijon懦'),
+ 6: ('kvintilijonas', 'kvintilijonai', 'kvintilijon懦'),
+ 7: ('sikstilijonas', 'sikstilijonai', 'sikstilijon懦'),
+ 8: ('septilijonas', 'septilijonai', 'septilijon懦'),
+ 9: ('oktilijonas', 'oktilijonai', 'oktilijon懦'),
+ 10: ('naintilijonas', 'naintilijonai', 'naintilijon懦'),
+}
+
+GENERIC_CENTS = ('centas', 'centai', 'cent懦')
+
+
+class Num2Word_LT(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'LTL': (('litas', 'litai', 'lit懦'), GENERIC_CENTS),
+ 'EUR': (('euras', 'eurai', 'eur懦'), GENERIC_CENTS),
+ 'USD': (('doleris', 'doleriai', 'doleri懦'), GENERIC_CENTS),
+ 'GBP': (
+ ('svaras sterling懦', 'svarai sterling懦', 'svar懦 sterling懦'),
+ ('pensas', 'pensai', 'pens懦')
+ ),
+ 'PLN': (
+ ('zlotas', 'zlotai', 'zlot懦'),
+ ('gra拧is', 'gra拧iai', 'gra拧i懦')),
+ 'RUB': (
+ ('rublis', 'rubliai', 'rubli懦'),
+ ('kapeika', 'kapeikos', 'kapeik懦')
+ ),
+ }
+
+ def setup(self):
+ self.negword = "minus"
+ self.pointword = "kablelis"
+
+ def pluralize(self, n, forms):
+ n1, n2, n3 = get_digits(n)
+ if n2 == 1 or n1 == 0 or n == 0:
+ return forms[2]
+ elif n1 == 1:
+ return forms[0]
+ else:
+ return forms[1]
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ base_str, n = self.parse_minus(n)
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return '%s%s %s %s' % (
+ base_str,
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return "%s%s" % (base_str, self._int2word(int(n)))
+
+ def to_ordinal(self, number):
+ raise NotImplementedError()
+
+ def _cents_verbose(self, number, currency):
+ return self._int2word(number, currency == 'RUB')
+
+ def _int2word(self, n, feminine=False):
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ words.append(ONES[n3][0])
+ if n3 > 1:
+ words.append(HUNDRED[1])
+ else:
+ words.append(HUNDRED[0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0:
+ if (i == 1 or feminine and i == 0) and n < 1000:
+ words.append(ONES_FEMININE[n1][0])
+ else:
+ words.append(ONES[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
diff --git a/fun_text_processing/num2words/num2words/lang_LV.py b/fun_text_processing/num2words/num2words/lang_LV.py
new file mode 100644
index 0000000..7a833d3
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_LV.py
@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('nulle',)
+
+ONES = {
+ 1: ('viens',),
+ 2: ('divi',),
+ 3: ('tr墨s',),
+ 4: ('膷etri',),
+ 5: ('pieci',),
+ 6: ('se拧i',),
+ 7: ('septi艈i',),
+ 8: ('asto艈i',),
+ 9: ('devi艈i',),
+}
+
+TENS = {
+ 0: ('desmit',),
+ 1: ('vienpadsmit',),
+ 2: ('divpadsmit',),
+ 3: ('tr墨spadsmit',),
+ 4: ('膷etrpadsmit',),
+ 5: ('piecpadsmit',),
+ 6: ('se拧padsmit',),
+ 7: ('septi艈padsmit',),
+ 8: ('asto艈padsmit',),
+ 9: ('devi艈padsmit',),
+}
+
+TWENTIES = {
+ 2: ('divdesmit',),
+ 3: ('tr墨sdesmit',),
+ 4: ('膷etrdesmit',),
+ 5: ('piecdesmit',),
+ 6: ('se拧desmit',),
+ 7: ('septi艈desmit',),
+ 8: ('asto艈desmit',),
+ 9: ('devi艈desmit',),
+}
+
+HUNDRED = ('simts', 'simti', 'simtu')
+
+THOUSANDS = {
+ 1: ('t奴kstotis', 't奴ksto拧i', 't奴ksto拧u'),
+ 2: ('miljons', 'miljoni', 'miljonu'),
+ 3: ('miljards', 'miljardi', 'miljardu'),
+ 4: ('triljons', 'triljoni', 'triljonu'),
+ 5: ('kvadriljons', 'kvadriljoni', 'kvadriljonu'),
+ 6: ('kvintiljons', 'kvintiljoni', 'kvintiljonu'),
+ 7: ('sikstiljons', 'sikstiljoni', 'sikstiljonu'),
+ 8: ('septiljons', 'septiljoni', 'septiljonu'),
+ 9: ('oktiljons', 'oktiljoni', 'oktiljonu'),
+ 10: ('nontiljons', 'nontiljoni', 'nontiljonu'),
+}
+
+GENERIC_DOLLARS = ('dol膩rs', 'dol膩ri', 'dol膩ru')
+GENERIC_CENTS = ('cents', 'centi', 'centu')
+
+GENERIC_KRONA = ('krona', 'kronas', 'kronu')
+GENERIC_ERA = ('膿re', '膿ras', '膿ru')
+
+
+class Num2Word_LV(Num2Word_Base):
+ """
+ Sadly we have a legal form (used in legal and finance documents):
+ http://www.eiro.lv/files/upload/files/Eiro_rakstiba-1.pdf
+ https://likumi.lv/doc.php?id=254741
+ http://eur-lex.europa.eu/legal-content/LV/TXT/HTML/?uri=CELEX:31998R0974&from=LV
+
+ Source: http://publications.europa.eu/code/lv/lv-5000500.htm
+ """
+ CURRENCY_FORMS = {
+ 'AUD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'CAD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ # repalced by EUR
+ 'EEK': (GENERIC_KRONA, GENERIC_CENTS),
+ 'EUR': (('eiro', 'eiro', 'eiro'), GENERIC_CENTS),
+ 'EUR_LEGAL': (('euro', 'euro', 'euro'), GENERIC_CENTS),
+ 'GBP': (
+ ('sterli艈u m膩rci艈a', 'sterli艈u m膩rci艈as', 'sterli艈u m膩rci艈u'),
+ ('penss', 'pensi', 'pensu')),
+ # replaced by EUR
+ 'LTL': (('lits', 'liti', 'litu'), GENERIC_CENTS),
+ # replaced by EUR
+ 'LVL': (('lats', 'lati', 'latu'),
+ ('sant墨ms', 'sant墨mi', 'sant墨mu')),
+ 'USD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'RUB': (('rublis', 'rub募i', 'rub募u'),
+ ('kapeika', 'kapeikas', 'kapeiku')),
+ 'SEK': (GENERIC_KRONA, GENERIC_ERA),
+ 'NOK': (GENERIC_KRONA, GENERIC_ERA),
+ 'PLN': (('zlots', 'zloti', 'zlotu'),
+ ('grasis', 'gra拧i', 'gra拧u')),
+ }
+
+ CURRENCY_ADJECTIVES = {
+ 'AUD': 'Austr膩lijas',
+ 'CAD': 'Kan膩das',
+ 'EEK': 'Igaunijas',
+ 'USD': 'ASV',
+ 'RUB': 'Kreivijas',
+ 'SEK': 'Zviedrijas',
+ 'NOK': 'Norv膿模ijas',
+ }
+
+ def setup(self):
+ self.negword = "m墨nus"
+ self.pointword = "komats"
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ base_str, n = self.parse_minus(n)
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return '%s%s %s %s' % (
+ base_str,
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return "%s%s" % (base_str, self._int2word(int(n)))
+
+ def pluralize(self, n, forms):
+ form = 0 if (n % 10 == 1 and n % 100 != 11) else 1 if n != 0 else 2
+ return forms[form]
+
+ def to_ordinal(self, number):
+ raise NotImplementedError()
+
+ def _int2word(self, n):
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ if n3 == 1 and n2 == 0 and n1 > 0:
+ words.append(HUNDRED[2])
+ elif n3 > 1:
+ words.append(ONES[n3][0])
+ words.append(HUNDRED[1])
+ else:
+ words.append(HUNDRED[0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0 and not (i > 0 and x == 1):
+ words.append(ONES[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
diff --git a/fun_text_processing/num2words/num2words/lang_MN.py b/fun_text_processing/num2words/num2words/lang_MN.py
new file mode 100644
index 0000000..ca2fdca
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_MN.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_MN(lang_EU.Num2Word_EU):
+ # def set_high_numwords(self, high):
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # self.cards[10 ** n] = word
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 3, -1)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+ # self.cards[10**n] = word + "釣涐灦釣�"
+ # try:
+ # ordinal_word = self.high_numwords[high]
+ # except KeyError:
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # print(word)
+ # print(n)
+ # self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_MN, self).setup()
+
+ self.negword = '褏邪褋邪褏'
+ self.pointword = '褑褝谐'
+
+ self.high_numwords = [(1000000,'薪褝谐 褋邪褟'), (100000, '薪褝谐 蟹褍褍薪 屑褟薪谐邪'), (10000, '邪褉胁邪薪 屑褟薪谐邪'), (1000, '薪褝谐 屑褟薪谐邪') ]
+
+ self.mid_numwords = [(100,'薪褝谐 蟹褍褍'), (90,'械褉褝薪'), (80,'薪邪褟薪'), (70,'写邪谢邪薪'), (60,'卸邪褉邪薪'), (50,'褌邪胁懈薪'), (40,'写萤褔懈薪'), (30,'谐褍褔懈薪')]
+
+ self.low_numwords = ['褏芯褉懈薪','邪褉胁邪薪 械褋萤薪','邪褉胁邪薪 薪邪泄屑邪薪','邪褉胁邪薪 写芯谢芯芯薪','邪褉胁邪薪 蟹褍褉谐邪邪','邪褉胁邪薪 褌邪胁','邪褉胁邪薪 写萤褉萤胁','邪褉胁邪薪 谐褍褉邪胁','邪褉胁邪薪 褏芯褢褉','邪褉胁邪薪 薪褝谐褝薪','邪褉邪胁','械褋萤薪','薪邪泄屑','袛芯谢芯芯','蟹褍褉谐邪邪','褌邪胁','写萤褉萤胁','谐褍褉邪胁','褏芯褢褉','薪褝谐','褌褝谐']
+
+ self.ords = {'薪褝谐': '褝褏谢褝褝写',
+ '褏芯褢褉': '褏芯褢褉写褍谐邪邪褉褌',
+ '谐褍褉邪胁': '谐褍褉邪胁 写邪褏褜',
+ '写萤褉萤胁': '褍褉邪谐褕',
+ '褌邪胁': '褌邪胁 写邪褏褜',
+ '蟹褍褉谐邪邪': '蟹褍褉谐邪邪 写邪褏褜',
+ '袛芯谢芯芯': '写芯谢芯芯 写邪褏褜',
+ '薪邪泄屑': '薪邪泄屑 写邪褏褜',
+ '械褋萤薪': '械褋 写褝褏',
+ '邪褉邪胁': '袗褉邪胁写褍谐邪邪褉'}
+
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s 斜芯谢芯薪 %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "蟹褍褍薪"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s%s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s%s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_MY.py b/fun_text_processing/num2words/num2words/lang_MY.py
new file mode 100644
index 0000000..b54f245
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_MY.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_MY(lang_EU.Num2Word_EU):
+ # def set_high_numwords(self, high):
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # self.cards[10 ** n] = word
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 3, -1)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+ # self.cards[10**n] = word + "釣涐灦釣�"
+ # try:
+ # ordinal_word = self.high_numwords[high]
+ # except KeyError:
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # print(word)
+ # print(n)
+ # self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_MY, self).setup()
+
+ self.negword = '醼♂�欋�踞�愥��'
+ self.pointword = '醼♂�欋�踞�愥��'
+
+ self.high_numwords = [(1000000,'醼炨�斸�横��'), (100000, '醼炨��斸�横��'), (10000, '醼炨�贬��勧�横��'), (1000, '醼戓�贬��勧��') ]
+
+ self.mid_numwords = [(100,'醼涐��'), (90,'醼�醼��羔�嗎�氠��'), (80,'醼涐�踞�呩�横�嗎�氠��'), (70,'醼佱��斸�踞�呩�横�嗎�氠��'), (60,'醼佱�坚�贬���醼横�嗎�氠�丰��'), (50,'醼勧��羔�嗎�氠��'), (40,'醼溼�贬�羔�嗎�氠��'), (30,'醼炨��夺�羔�嗎�氠��')]
+
+ # self.low_numwords = ['啶多啶ㄠ啶�', '啶忇', '啶︵啶�', '啶む啶�', '啶氞ぞ啶�', '啶ぞ啶佮', '啶�', '啶膏ぞ啶�', '啶嗋', '啶ㄠ', '啶︵じ', '啶忇啶距ぐ', '啶ぞ啶灌啶�', '啶む啶灌啶�', '啶氞啶�', '啶え啷嵿ぇ啷嵿ぐ', '啶膏啶灌啶�', '啶膏い啷嵿ぐ啶�', '啶呧啶距ぐ', '啶夃え啷嵿え啷�啶�']
+ self.low_numwords = ['醼斸�踞�呩�横�嗎�氠��','醼嗎�氠�丰�横��醼���','醼嗎�氠�丰�横�涐�踞�呩��','醼嗎�氠�丰�横�佱��斸�踞�呩��','醼嗎�氠�丰�横�佱�坚�贬���醼�','醼嗎�氠�丰�横�勧���','醼嗎�氠�丰�横�溼�贬��','醼嗎�氠�丰�横�炨��夺��','醼嗎�氠�丰�横�斸�踞�呩��','醼嗎�氠�丰�横�愥�呩��','醼嗎�氠��','醼�醼���','醼涐�踞�呩��','醼佱��斸�踞�呩��','醼佱�坚�贬���醼�','醼勧���','醼溼�贬��','醼炨��夺��','醼斸�踞�呩��','醼愥�呩��','醼炨���']
+
+ self.ords = {'醼愥�呩��':'醼曖�戓��',
+ '醼斸�踞�呩��':'醼掅��愥���',
+ '醼炨��夺��':'醼愥�愥���',
+ '醼溼�贬��':'醼呩�愥��愥�贯��',
+ '醼勧���':'醼曖�夅�贯�呩��',
+ '醼佱�坚�贬���醼�':'醼嗎�嬦�贯�屷��',
+ '醼佱��斸�踞�呩��':'醼炨�愥�贯�愥��',
+ '醼涐�踞�呩��':'醼♂�嬦�贯�屷��',
+ '醼�醼���':'醼斸�濁��',
+ '醼嗎�氠��':'醼掅�炨��'}
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s 醼斸�踞�勧�丰�� %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "醼涐���"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s%s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s%s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_NE.py b/fun_text_processing/num2words/num2words/lang_NE.py
new file mode 100644
index 0000000..f82c27a
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_NE.py
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_NE(lang_EU.Num2Word_EU):
+ # def set_high_numwords(self, high):
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # self.cards[10 ** n] = word
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 0, -3)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+ # self.cards[10**n] = word + "釣涐灦釣�"
+ # try:
+ # ordinal_word = self.high_numwords[high]
+ # except KeyError:
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # print(word)
+ # print(n)
+ # self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_NE, self).setup()
+
+ self.negword = '啶ぞ啶囙え啶�'
+ self.pointword = '啶呧啶�'
+
+ self.high_numwords = [(1000000000000000, '啶膏く 啶熰啶班た啶侧た啶え'), (1000000000000,'啶熰啶班た啶侧た啶え'), (1000000000, '啶た啶侧た啶え'), (1000000, '啶た啶侧た啶え'), (1000, '啶灌啶距ぐ') ]
+
+ self.mid_numwords = [(100, '啶忇 啶膏く'), (90, '啶ㄠが啷嵿が啷�'), (80, '啶呧じ啷嵿じ啷�'), (70, '啶膏い啷嵿い啶班'), (60, '啶膏ぞ啶�'), (50, '啶啶距じ'), (40, '啶氞ぞ啶侧啶�'), (30, '啶む啶�'), (20, '啶掂た啶�')]
+
+ # self.low_numwords = ['啶多啶ㄠ啶�', '啶忇', '啶︵啶�', '啶む啶�', '啶氞ぞ啶�', '啶ぞ啶佮', '啶�', '啶膏ぞ啶�', '啶嗋', '啶ㄠ', '啶︵じ', '啶忇啶距ぐ', '啶ぞ啶灌啶�', '啶む啶灌啶�', '啶氞啶�', '啶え啷嵿ぇ啷嵿ぐ', '啶膏啶灌啶�', '啶膏い啷嵿ぐ啶�', '啶呧啶距ぐ', '啶夃え啷嵿え啷�啶�']
+ self.low_numwords = ["啶夃え啷嵿え啶距啶�",
+ "啶呧啶距ぐ",
+ "啶膏い啷嵿ぐ",
+ "啶膏啶灌啶�",
+ "啶え啷嵿ぇ啷嵿ぐ",
+ "啶氞啶�",
+ "啶む啶灌啶�",
+ "啶ぞ啶灌啶�",
+ "啶忇啶距ぐ",
+ "啶︵ざ",
+ "啶ㄠ",
+ "啶嗋",
+ "啶膏ぞ啶�",
+ "啶�",
+ "啶ぞ啶佮",
+ "啶氞ぞ啶�",
+ "啶む啶�",
+ "啶︵啶�",
+ "啶忇",
+ "啶多啶ㄠ啶�"]
+
+ self.ords = {'啶忇': '啶す啶苦げ啷�',
+ '啶︵啶�': '啶︵啶︵啶膏啶班',
+ '啶む啶�': '啶む啶膏啶班',
+ '啶氞ぞ啶�': '啶氞啶ム啶班啶�',
+ '啶ぞ啶佮': '啶ぞ啶佮啷�',
+ '啶�': '啶涏啶熰',
+ '啶膏ぞ啶�': '啶膏ぞ啶む',
+ '啶嗋': '啶嗋啷�',
+ '啶ㄠ': '啶ㄠさ啷�',
+ '啶︵ざ': '啶︵じ啷�'}
+
+ self.labeled_numbers = {'0': '啶多啶ㄠ啶�',
+ '1': '啶忇',
+ '2': '啶︵啶�',
+ '3': '啶む啶�',
+ '4': '啶氞ぞ啶�',
+ '5': '啶ぞ啶佮',
+ '6': '啶�',
+ '7': '啶膏ぞ啶�',
+ '8': '啶嗋',
+ '9': '啶ㄠ',
+ '10': '啶︵ざ',
+ '11': '啶忇啶距ぐ',
+ '12': '啶ぞ啶灌啶�',
+ '13': '啶む啶灌啶�',
+ '14': '啶氞啶�',
+ '15': '啶え啷嵿ぇ啷嵿ぐ',
+ '16': '啶膏啶灌啶�',
+ '17': '啶膏い啷嵿ぐ',
+ '18': '啶呧啶距ぐ',
+ '19': '啶夃え啷嵿え啶距啶�',
+ '20': '啶掂た啶�',
+ '21': '啶忇啷嵿啶距啶�',
+ '22': '啶ぞ啶囙じ',
+ '23': '啶む啶堗じ',
+ '24': '啶氞啶掂た啶�',
+ '25': '啶啷嵿啶苦じ',
+ '26': '啶涏が啷嵿が啶苦じ',
+ '27': '啶膏い啷嵿い啶距啶�',
+ '28': '啶呧啷嵿啶距啶�',
+ '29': '啶夃え啶ㄠ啶むた啶�',
+ '30': '啶むた啶�',
+ '31': '啶忇啶む啶むた啶�',
+ '32': '啶い啷嵿い啶苦じ',
+ '33': '啶む啶む啶むた啶�',
+ '34': '啶氞啶佮い啶苦じ',
+ '35': '啶啶佮い啶苦じ',
+ '36': '啶涏い啷嵿い啶苦じ',
+ '37': '啶膏啶佮い啷�啶�',
+ '38': '啶呧啶む啶�',
+ '39': '啶夃え啶ㄠ啶氞ぞ啶侧啶�',
+ '40': '啶氞ぞ啶侧啶�',
+ '41': '啶忇啶氞ぞ啶侧啶�',
+ '42': '啶く啶距げ啷�啶�',
+ '43': '啶む啶班た啶ぞ啶侧啶�',
+ '44': '啶氞さ啶距げ啷�啶�',
+ '45': '啶啶佮い啶距げ啷�啶�',
+ '46': '啶涏く啶距げ啷�啶�',
+ '47': '啶膏啷嵿啶距げ啷�啶�',
+ '48': '啶呧啶氞ぞ啶侧啶�',
+ '49': '啶夃え啶ㄠ啶氞ぞ啶�',
+ '50': '啶啶距じ',
+ '51': '啶忇啶距啶ㄠ啶�',
+ '52': '啶ぞ啶夃え啷嵿え',
+ '53': '啶む啶班た啶え啷嵿え',
+ '54': '啶氞啶ㄠ啶�',
+ '55': '啶啶え啷嵿え',
+ '56': '啶涏お啶ㄠ啶�',
+ '57': '啶膏え啷嵿い啶距啶ㄠ啶�',
+ '58': '啶呧え啷嵿啶距啶ㄠ啶�',
+ '59': '啶夃え啶ㄠ啶膏ぞ啶犩',
+ '60': '啶膏ぞ啶犩',
+ '61': '啶忇啶膏啷嵿啷�',
+ '62': '啶く啶膏啷嵿啷�',
+ '63': '啶む啶班た啶膏啷嵿啷�',
+ '64': '啶氞啶傕じ啶熰啶犩',
+ '65': '啶啶傕じ啶熰啶犩',
+ '66': '啶涏く啶膏啷嵿啷�',
+ '67': '啶膏い啶膏啷嵿啷�',
+ '68': '啶呧啶膏啷嵿啷�',
+ '69': '啶夃え啶ㄠ啶膏い啷嵿い啶班',
+ '70': '啶膏い啷嵿い啶班',
+ '71': '啶忇啶灌い啷嵿い啶�',
+ '72': '啶す啶む啶むぐ',
+ '73': '啶む啶班た啶灌い啷嵿い啶�',
+ '74': '啶氞啶灌い啷嵿い啶�',
+ '75': '啶啶灌い啷嵿い啶�',
+ '76': '啶涏く啶灌い啷嵿い啶�',
+ '77': '啶膏い啶灌い啷嵿い啶�',
+ '78': '啶呧啶灌い啷嵿い啶�',
+ '79': '啶夃え啶距じ啷�',
+ '80': '啶呧じ啷�',
+ '81': '啶忇啶距じ啷�',
+ '82': '啶く啶距じ啷�',
+ '83': '啶む啶班た啶ぞ啶膏',
+ '84': '啶氞啶班ぞ啶膏',
+ '85': '啶啶距じ啷�',
+ '86': '啶涏く啶距じ啷�',
+ '87': '啶膏い啶距じ啷�',
+ '88': '啶呧啶距じ啷�',
+ '89': '啶夃え啶距え啷嵿え啶啶',
+ '90': '啶ㄠが啷嵿が啷�',
+ '91': '啶忇啶距え啷嵿え啶啶',
+ '92': '啶く啶距え啶啶',
+ '93': '啶む啶班た啶ぞ啶ㄠ啶ㄠが啷嵿が啷�',
+ '94': '啶氞啶班ぞ啶ㄠ啶ㄠが啷嵿が啷�',
+ '95': '啶え啷嵿啶距え啶啶',
+ '96': '啶涏く啶距え啷嵿え啶啶',
+ '97': '啶膏え啷嵿い啶距え啷嵿え啶啶',
+ '98': '啶呧え啷嵿啶距え啷嵿え啶啶',
+ '99': '啶夃え啶距え啷嵿じ啶�',
+ '100': '啶忇 啶膏く',
+ '200': '啶︵啶� 啶膏く',
+ '300': '啶む啶� 啶膏く',
+ '400': '啶氞ぞ啶� 啶膏く',
+ '500': '啶ぞ啶佮 啶膏く',
+ '600': '啶� 啶膏く',
+ '700': '啶膏ぞ啶� 啶膏く',
+ '800': '啶嗋 啶膏く',
+ '900': '啶ㄠ 啶膏く',
+ '1,000': '啶忇 啶灌啶距ぐ',
+ '2,000': '啶︵啶� 啶灌啶距ぐ',
+ '3,000': '啶む啶� 啶灌啶距ぐ',
+ '4,000': '啶氞ぞ啶� 啶灌啶距ぐ',
+ '5,000': '啶ぞ啶佮 啶灌啶距ぐ',
+ '6,000': '啶� 啶灌啶距ぐ',
+ '7,000': '啶膏ぞ啶� 啶灌啶距ぐ',
+ '8,000': '啶嗋 啶灌啶距ぐ',
+ '9,000': '啶ㄠ 啶灌啶距ぐ',
+ '10,000': '啶︵ざ 啶灌啶距ぐ',
+ '20,000': '啶た啶� 啶灌啶距ぐ',
+ '30,000': '啶む啶� 啶灌啶距ぐ',
+ '40,000': '啶氞ぞ啶侧た啶� 啶灌啶距ぐ',
+ '50,000': '啶啶距じ 啶灌啶距ぐ',
+ '60,000': '啶膏ぞ啶犩 啶灌啶距ぐ',
+ '70,000': '啶膏い啷嵿い啶班 啶灌啶距ぐ',
+ '80,000': '啶呧じ啷� 啶灌啶距ぐ',
+ '90,000': '啶ㄠが啷嵿が啷� 啶灌啶距ぐ',
+ '100,000': '啶忇 啶侧ぞ啶�',
+ '1,000,000': '啶︵ざ 啶侧ぞ啶�',
+ '10,000,000': '啶忇 啶曕ぐ啷嬥ぁ',
+ '1,000,000,000': '啶忇 啶呧ぐ啷嵿が',
+ '100,000,000,000': '啶忇 啶栢ぐ啷嵿が'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "啶膏く"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_NL.py b/fun_text_processing/num2words/num2words/lang_NL.py
new file mode 100644
index 0000000..97a5aa2
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_NL.py
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function, unicode_literals
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_NL(Num2Word_EU):
+ CURRENCY_FORMS = {
+ 'EUR': (('euro', 'euro'), ('cent', 'cent')),
+ 'GBP': (('pond', 'pond'), ('penny', 'pence')),
+ 'USD': (('dollar', 'dollar'), ('cent', 'cent')),
+ 'CNY': (('yuan', 'yuan'), ('jiao', 'fen')),
+ }
+
+ GIGA_SUFFIX = "iljard"
+ MEGA_SUFFIX = "iljoen"
+
+ def setup(self):
+ super(Num2Word_NL, self).setup()
+
+ self.negword = "min "
+ self.pointword = "komma"
+ # "Cannot treat float %s as ordinal."
+ self.errmsg_floatord = (
+ "Het zwevende puntnummer %s kan niet omgezet worden " +
+ "naar een ordernummer."
+ )
+ # "type(((type(%s)) ) not in [long, int, float]"
+ self.errmsg_nonnum = (
+ "Alleen nummers (type (%s)) kunnen naar " +
+ "woorden omgezet worden."
+ )
+ # "Cannot treat negative num %s as ordinal."
+ self.errmsg_negord = (
+ "Het negatieve getal %s kan niet omgezet " +
+ "worden naar een ordernummer."
+ )
+ # "abs(%s) must be less than %s."
+ self.errmsg_toobig = "Het getal %s moet minder zijn dan %s."
+ self.exclude_title = []
+
+ lows = ["non", "okt", "sept", "sext", "quint", "quadr", "tr", "b", "m"]
+ units = ["", "un", "duo", "tre", "quattuor", "quin", "sex", "sept",
+ "okto", "novem"]
+ tens = ["dez", "vigint", "trigint", "quadragint", "quinquagint",
+ "sexagint", "septuagint", "oktogint", "nonagint"]
+
+ self.high_numwords = (
+ ["zend"] + self.gen_high_numwords(units, tens, lows))
+
+ self.mid_numwords = [(1000, "duizend"), (100, "honderd"),
+ (90, "negentig"), (80, "tachtig"),
+ (70, "zeventig"), (60, "zestig"),
+ (50, "vijftig"), (40, "veertig"),
+ (30, "dertig")]
+
+ self.low_numwords = ["twintig", "negentien", "achttien", "zeventien",
+ "zestien", "vijftien", "veertien", "dertien",
+ "twaalf", "elf", "tien", "negen", "acht", "zeven",
+ "zes", "vijf", "vier", "drie", "twee", "茅茅n",
+ "nul"]
+
+ self.ords = {"茅茅n": "eerst",
+ "twee": "tweed",
+ "drie": "derd",
+ "vier": "vierd",
+ "vijf": "vijfd",
+ "zes": "zesd",
+ "zeven": "zevend",
+ "acht": "achtst",
+ "negen": "negend",
+ "tien": "tiend",
+ "elf": "elfd",
+ "twaalf": "twaalfd",
+
+ "ig": "igst",
+ "erd": "erdst",
+ "end": "endst",
+ "joen": "joenst",
+ "rd": "rdst"}
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 10 ** 6:
+ return next
+ ctext = "een"
+
+ if nnum > cnum:
+ if nnum >= 10 ** 6:
+ ctext += " "
+ val = cnum * nnum
+ else:
+ if nnum < 10 < cnum < 100:
+ if nnum == 1:
+ ntext = "een"
+
+ if ntext.endswith("e"):
+ ntext += "毛n" # "n"
+ else:
+ ntext += "en"
+ ntext, ctext = ctext, ntext # + "en"
+ elif cnum >= 10 ** 6:
+ ctext += " "
+ val = cnum + nnum
+
+ word = ctext + ntext
+ return word, val
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outword = self.to_cardinal(value)
+ for key in self.ords:
+ if outword.endswith(key):
+ outword = outword[:len(outword) - len(key)] + self.ords[key]
+ break
+ return outword + "e"
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return str(value) + "e"
+
+ def pluralize(self, n, forms):
+ """
+ :param n:
+ :param forms:
+ :return:
+
+ gettext form is nplurals=2; plural=(n != 1);
+ but this claims https://onzetaal.nl/taaladvies/euro-euros/
+ not sure if it's applied only to euro
+ """
+ return forms[0]
+
+ def to_year(self, val, longval=True):
+ if not (val // 100) % 10:
+ return self.to_cardinal(val)
+ return self.to_splitnum(val, hightxt="honderd", longval=longval)
diff --git a/fun_text_processing/num2words/num2words/lang_NO.py b/fun_text_processing/num2words/num2words/lang_NO.py
new file mode 100644
index 0000000..5852abb
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_NO.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_NO(lang_EU.Num2Word_EU):
+ GIGA_SUFFIX = "illard"
+ MEGA_SUFFIX = "illion"
+
+ def set_high_numwords(self, high):
+ cap = 3 + 6 * len(high)
+
+ for word, n in zip(high, range(cap, 3, -6)):
+ if self.GIGA_SUFFIX:
+ self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ if self.MEGA_SUFFIX:
+ self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+
+ def setup(self):
+ super(Num2Word_NO, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "komma"
+ self.exclude_title = ["og", "komma", "minus"]
+
+ self.mid_numwords = [(1000, "tusen"), (100, "hundre"),
+ (90, "nitti"), (80, "氓tti"), (70, "sytti"),
+ (60, "seksti"), (50, "femti"), (40, "f酶rti"),
+ (30, "tretti")]
+ self.low_numwords = ["tjue", "nitten", "atten", "sytten", "seksten", "femten", "fjorten", "tretten", "tolv", "elleve", "ti", "ni", "氓tte", "syv", "seks", "fem", "fire", "tre", "to", "en", "null"]
+ self.ords = {"en": "f酶rst",
+ "to": "andre",
+ "tre": "tredje",
+ "fire": "fjerde",
+ "fem": "femte",
+ "seks": "sjette",
+ "syv": "syvende",
+ "氓tte": "氓ttende",
+ "ni": "niende",
+ "ti": "tiende",
+ "elleve": "ellevte",
+ "tolv": "tolvte",
+ "tjue": "tjuende"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s og %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s, %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastwords = outwords[-1].split("-")
+ lastword = lastwords[-1].lower()
+ try:
+ lastword = self.ords[lastword]
+ except KeyError:
+ if lastword[-2:] == "ti":
+ lastword = lastword + "ende"
+ else:
+ lastword += "de"
+ lastwords[-1] = self.title(lastword)
+ outwords[-1] = "".join(lastwords)
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-2:])
+
+ def to_year(self, val, longval=True):
+ if not (val // 100) % 10:
+ return self.to_cardinal(val)
+ return self.to_splitnum(val, hightxt="hundre", jointxt="og",
+ longval=longval)
+
+ def to_currency(self, val, longval=True):
+ return self.to_splitnum(val, hightxt="krone/r", lowtxt="\xf8re/r",
+ jointxt="og", longval=longval, cents=True)
diff --git a/fun_text_processing/num2words/num2words/lang_PL.py b/fun_text_processing/num2words/num2words/lang_PL.py
new file mode 100644
index 0000000..f7760aa
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_PL.py
@@ -0,0 +1,245 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import itertools
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('zero',)
+
+ONES = {
+ 1: ('jeden',),
+ 2: ('dwa',),
+ 3: ('trzy',),
+ 4: ('cztery',),
+ 5: ('pi臋膰',),
+ 6: ('sze艣膰',),
+ 7: ('siedem',),
+ 8: ('osiem',),
+ 9: ('dziewi臋膰',),
+}
+
+ONES_ORDINALS = {
+ 1: ('pierwszy', "pierwszo"),
+ 2: ('drugi', "dwu"),
+ 3: ('trzeci', "trzy"),
+ 4: ('czwarty', "cztero"),
+ 5: ('pi膮ty', "pi臋cio"),
+ 6: ('sz贸sty', "sze艣cio"),
+ 7: ('si贸dmy', "siedmio"),
+ 8: ('贸smy', "o艣mio"),
+ 9: ('dziewi膮ty', "dziewi臋cio"),
+ 10: ('dziesi膮ty', "dziesi臋cio"),
+ 11: ('jedenasty', "jedenasto"),
+ 12: ('dwunasty', "dwunasto"),
+ 13: ('trzynasty', "trzynasto"),
+ 14: ('czternasty', "czternasto"),
+ 15: ('pi臋tnasty', "pi臋tnasto"),
+ 16: ('szesnasty', "szesnasto"),
+ 17: ('siedemnasty', "siedemnasto"),
+ 18: ('osiemnasty', "osiemnasto"),
+ 19: ('dziewi臋tnasty', "dziewi臋tnasto"),
+}
+
+TENS = {
+ 0: ('dziesi臋膰',),
+ 1: ('jedena艣cie',),
+ 2: ('dwana艣cie',),
+ 3: ('trzyna艣cie',),
+ 4: ('czterna艣cie',),
+ 5: ('pi臋tna艣cie',),
+ 6: ('szesna艣cie',),
+ 7: ('siedemna艣cie',),
+ 8: ('osiemna艣cie',),
+ 9: ('dziewi臋tna艣cie',),
+}
+
+
+TWENTIES = {
+ 2: ('dwadzie艣cia',),
+ 3: ('trzydzie艣ci',),
+ 4: ('czterdzie艣ci',),
+ 5: ('pi臋膰dziesi膮t',),
+ 6: ('sze艣膰dziesi膮t',),
+ 7: ('siedemdziesi膮t',),
+ 8: ('osiemdziesi膮t',),
+ 9: ('dziewi臋膰dzisi膮t',),
+}
+
+TWENTIES_ORDINALS = {
+ 2: ('dwudziesty', "dwudziesto"),
+ 3: ('trzydziesty', "trzydiesto"),
+ 4: ('czterdziesty', "czterdziesto"),
+ 5: ('pi臋膰dziesi膮ty', "pi臋膰dziesi臋cio"),
+ 6: ('sze艣膰dziesi膮ty', "sze艣膰dziesi臋cio"),
+ 7: ('siedemdziesi膮ty', "siedemdziesi臋cio"),
+ 8: ('osiemdziesi膮ty', "osiemdziesi臋cio"),
+ 9: ('dziewi臋膰dzisi膮ty', "dziewi臋膰dziesi臋cio"),
+}
+
+HUNDREDS = {
+ 1: ('sto',),
+ 2: ('dwie艣cie',),
+ 3: ('trzysta',),
+ 4: ('czterysta',),
+ 5: ('pi臋膰set',),
+ 6: ('sze艣膰set',),
+ 7: ('siedemset',),
+ 8: ('osiemset',),
+ 9: ('dziewi臋膰set',),
+}
+
+HUNDREDS_ORDINALS = {
+ 1: ('setny', "stu"),
+ 2: ('dwusetny', "dwustu"),
+ 3: ('trzysetny', "trzystu"),
+ 4: ('czterysetny', "czterystu"),
+ 5: ('pi臋膰setny', "pi臋cset"),
+ 6: ('sze艣膰setny', "sze艣膰set"),
+ 7: ('siedemsetny', "siedemset"),
+ 8: ('osiemsetny', "o艣iemset"),
+ 9: ('dziewi臋膰setny', "dziewi臋膰set"),
+}
+
+THOUSANDS = {
+ 1: ('tysi膮c', 'tysi膮ce', 'tysi臋cy'), # 10^3
+}
+
+prefixes_ordinal = {
+ 1: "tysi臋czny",
+ 2: "milionowy",
+ 3: "milairdowy"
+}
+
+prefixes = ( # 10^(6*x)
+ "mi", # 10^6
+ "bi", # 10^12
+ "try", # 10^18
+ "kwadry", # 10^24
+ "kwinty", # 10^30
+ "seksty", # 10^36
+ "septy", # 10^42
+ "okty", # 10^48
+ "nony", # 10^54
+ "decy" # 10^60
+)
+suffixes = ("lion", "liard") # 10^x or 10^(x+3)
+
+for idx, (p, s) in enumerate(itertools.product(prefixes, suffixes)):
+ name = p + s
+ THOUSANDS[idx+2] = (name, name + 'y', name + '贸w')
+
+
+class Num2Word_PL(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'PLN': (
+ ('z艂oty', 'z艂ote', 'z艂otych'), ('grosz', 'grosze', 'groszy')
+ ),
+ 'EUR': (
+ ('euro', 'euro', 'euro'), ('cent', 'centy', 'cent贸w')
+ ),
+ }
+
+ def setup(self):
+ self.negword = "minus"
+ self.pointword = "przecinek"
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return u'%s %s %s' % (
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return self._int2word(int(n))
+
+ def pluralize(self, n, forms):
+ if n == 1:
+ form = 0
+ elif 5 > n % 10 > 1 and (n % 100 < 10 or n % 100 > 20):
+ form = 1
+ else:
+ form = 2
+ return forms[form]
+
+ def last_fragment_to_ordinal(self, last, words, level):
+ n1, n2, n3 = get_digits(last)
+ last_two = n2*10+n1
+ if last_two == 0:
+ words.append(HUNDREDS_ORDINALS[n3][level])
+ elif level == 1 and last == 1:
+ return
+ elif last_two < 20:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][level])
+ words.append(ONES_ORDINALS[last_two][level])
+ elif last_two % 10 == 0:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][level])
+ words.append(TWENTIES_ORDINALS[n2][level])
+ else:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+ words.append(TWENTIES_ORDINALS[n2][0])
+ words.append(ONES_ORDINALS[n1][0])
+
+ def to_ordinal(self, number):
+ if number % 1 != 0:
+ raise NotImplementedError()
+ words = []
+ fragments = list(splitbyx(str(number), 3))
+ level = 0
+ last = fragments[-1]
+ while last == 0:
+ level = level+1
+ fragments.pop()
+ last = fragments[-1]
+ if len(fragments) > 1:
+ pre_part = self._int2word(number-(last*1000**level))
+ words.append(pre_part)
+ self.last_fragment_to_ordinal(last, words, 0 if level == 0 else 1)
+ output = " ".join(words)
+ if last == 1 and level > 0 and output != "":
+ output = output + " "
+ if level > 0:
+ output = output + prefixes_ordinal[level]
+ return output
+
+ def _int2word(self, n):
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0 and not (i > 0 and x == 1):
+ words.append(ONES[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
diff --git a/fun_text_processing/num2words/num2words/lang_PT.py b/fun_text_processing/num2words/num2words/lang_PT.py
new file mode 100644
index 0000000..72c306f
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_PT.py
@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, unicode_literals
+
+import re
+
+from .lang_EU import Num2Word_EU
+
+DOLLAR = ('d贸lar', 'd贸lares')
+CENTS = ('c锚ntimo', 'c锚ntimos')
+
+
+class Num2Word_PT(Num2Word_EU):
+
+ CURRENCY_FORMS = {
+ 'AUD': (DOLLAR, CENTS),
+ 'CAD': (DOLLAR, CENTS),
+ 'EUR': (('euro', 'euros'), CENTS),
+ 'GBP': (('libra', 'libras'), ('p茅ni', 'pence')),
+ 'USD': (DOLLAR, CENTS),
+ }
+
+ GIGA_SUFFIX = None
+ MEGA_SUFFIX = "ili茫o"
+
+ def setup(self):
+ super(Num2Word_PT, self).setup()
+ lows = ["quatr", "tr", "b", "m"]
+ self.high_numwords = self.gen_high_numwords([], [], lows)
+ self.negword = "menos "
+ self.pointword = "v铆rgula"
+ self.exclude_title = ["e", "v铆rgula", "menos"]
+
+ self.mid_numwords = [
+ (1000, "mil"), (100, "cem"), (90, "noventa"),
+ (80, "oitenta"), (70, "setenta"), (60, "sessenta"),
+ (50, "cinquenta"), (40, "quarenta"), (30, "trinta")
+ ]
+ self.low_numwords = [
+ "vinte", "dezanove", "dezoito", "dezassete", "dezasseis",
+ "quinze", "catorze", "treze", "doze", "onze", "dez",
+ "nove", "oito", "sete", "seis", "cinco", "quatro", "tr锚s", "dois",
+ "um", "zero"
+ ]
+ self.ords = [
+ {
+ 0: "",
+ 1: "primeiro",
+ 2: "segundo",
+ 3: "terceiro",
+ 4: "quarto",
+ 5: "quinto",
+ 6: "sexto",
+ 7: "s茅timo",
+ 8: "oitavo",
+ 9: "nono",
+ },
+ {
+ 0: "",
+ 1: "d茅cimo",
+ 2: "vig茅simo",
+ 3: "trig茅simo",
+ 4: "quadrag茅simo",
+ 5: "quinquag茅simo",
+ 6: "sexag茅simo",
+ 7: "septuag茅simo",
+ 8: "octog茅simo",
+ 9: "nonag茅simo",
+ },
+ {
+ 0: "",
+ 1: "cent茅simo",
+ 2: "ducent茅simo",
+ 3: "tricent茅simo",
+ 4: "quadrigent茅simo",
+ 5: "quingent茅simo",
+ 6: "seiscent茅simo",
+ 7: "septigent茅simo",
+ 8: "octigent茅simo",
+ 9: "nongent茅simo",
+ },
+ ]
+ self.thousand_separators = {
+ 3: "mil茅simo",
+ 6: "milion茅simo",
+ 9: "mil茅simo milion茅simo",
+ 12: "bilion茅simo",
+ 15: "mil茅simo bilion茅simo"
+ }
+ self.hundreds = {
+ 1: "cento",
+ 2: "duzentos",
+ 3: "trezentos",
+ 4: "quatrocentos",
+ 5: "quinhentos",
+ 6: "seiscentos",
+ 7: "setecentos",
+ 8: "oitocentos",
+ 9: "novecentos",
+ }
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+ ctext = "um"
+ elif cnum == 100 and not nnum % 1000 == 0:
+ ctext = "cento"
+
+ if nnum < cnum:
+ if cnum < 100:
+ return ("%s e %s" % (ctext, ntext), cnum + nnum)
+ return ("%s e %s" % (ctext, ntext), cnum + nnum)
+
+ elif (not nnum % 1000000000) and cnum > 1:
+ ntext = ntext[:-4] + "li玫es"
+ elif (not nnum % 1000000) and cnum > 1:
+ ntext = ntext[:-4] + "lh玫es"
+ # correct "mili茫o" to "milh茫o"
+ if ntext == 'mili茫o':
+ ntext = 'milh茫o'
+ if nnum == 100:
+ ctext = self.hundreds[cnum]
+ ntext = ""
+
+ else:
+ ntext = " " + ntext
+
+ return (ctext + ntext, cnum * nnum)
+
+ def to_cardinal(self, value):
+ result = super(Num2Word_PT, self).to_cardinal(value)
+
+ # Transforms "mil e cento e catorze" into "mil cento e catorze"
+ # Transforms "cem milh玫es e duzentos mil e duzentos e dez" em "cem
+ # milh玫es duzentos mil duzentos e dez" but "cem milh玫es e duzentos
+ # mil e duzentos" in "cem milh玫es duzentos mil e duzentos" and not in
+ # "cem milh玫es duzentos mil duzentos"
+ for ext in (
+ 'mil', 'milh茫o', 'milh玫es', 'mil milh玫es',
+ 'bili茫o', 'bili玫es', 'mil bili玫es'):
+ if re.match('.*{} e \\w*entos? (?=.*e)'.format(ext), result):
+ result = result.replace(
+ '{} e'.format(ext), '{}'.format(ext)
+ )
+
+ return result
+
+ # for the ordinal conversion the code is similar to pt_BR code,
+ # although there are other rules that are probably more correct in
+ # Portugal. Concerning numbers from 2000th on, saying "dois
+ # mil茅simos" instead of "segundo mil茅simo" (the first number
+ # would be used in the cardinal form instead of the ordinal) is better.
+ # This was not implemented.
+ # source:
+ # https://ciberduvidas.iscte-iul.pt/consultorio/perguntas/a-forma-por-extenso-de-2000-e-de-outros-ordinais/16428
+ def to_ordinal(self, value):
+ # Before changing this function remember this is used by pt-BR
+ # so act accordingly
+ self.verify_ordinal(value)
+
+ result = []
+ value = str(value)
+ thousand_separator = ''
+
+ for idx, char in enumerate(value[::-1]):
+ if idx and idx % 3 == 0:
+ thousand_separator = self.thousand_separators[idx]
+
+ if char != '0' and thousand_separator:
+ # avoiding "segundo milion茅simo mil茅simo" for 6000000,
+ # for instance
+ result.append(thousand_separator)
+ thousand_separator = ''
+
+ result.append(self.ords[idx % 3][int(char)])
+
+ result = ' '.join(result[::-1])
+ result = result.strip()
+ result = re.sub('\\s+', ' ', result)
+
+ if result.startswith('primeiro') and value != '1':
+ # avoiding "primeiro mil茅simo", "primeiro milion茅simo" and so on
+ result = result[9:]
+
+ return result
+
+ def to_ordinal_num(self, value):
+ # Before changing this function remember this is used by pt-BR
+ # so act accordingly
+ self.verify_ordinal(value)
+ return "%s潞" % (value)
+
+ def to_year(self, val, longval=True):
+ # Before changing this function remember this is used by pt-BR
+ # so act accordingly
+ if val < 0:
+ return self.to_cardinal(abs(val)) + ' antes de Cristo'
+ return self.to_cardinal(val)
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=' e',
+ adjective=False):
+ # change negword because base.to_currency() does not need space after
+ backup_negword = self.negword
+ self.negword = self.negword[:-1]
+ result = super(Num2Word_PT, self).to_currency(
+ val, currency=currency, cents=cents, separator=separator,
+ adjective=adjective)
+ # undo the change on negword
+ self.negword = backup_negword
+
+ # transforms "milh玫es euros" em "milh玫es de euros"
+ cr1, _ = self.CURRENCY_FORMS[currency]
+
+ for ext in (
+ 'milh茫o', 'milh玫es', 'bili茫o',
+ 'bili玫es', 'trili茫o', 'trili玫es'):
+ if re.match('.*{} (?={})'.format(ext, cr1[1]), result):
+ result = result.replace(
+ '{}'.format(ext), '{} de'.format(ext), 1
+ )
+ # do not print "e zero c锚ntimos"
+ result = result.replace(' e zero c锚ntimos', '')
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_PT_BR.py b/fun_text_processing/num2words/num2words/lang_PT_BR.py
new file mode 100644
index 0000000..834d11c
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_PT_BR.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+from __future__ import division, unicode_literals
+
+import re
+
+from . import lang_PT
+
+
+class Num2Word_PT_BR(lang_PT.Num2Word_PT):
+ def set_high_numwords(self, high):
+ max = 3 + 3*len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ self.cards[10**n] = word + "ilh茫o"
+
+ def setup(self):
+ super(Num2Word_PT_BR, self).setup()
+
+ self.low_numwords[1] = 'dezenove'
+ self.low_numwords[3] = 'dezessete'
+ self.low_numwords[4] = 'dezesseis'
+
+ self.thousand_separators = {
+ 3: "mil茅simo",
+ 6: "milion茅simo",
+ 9: "bilion茅simo",
+ 12: "trilion茅simo",
+ 15: "quadrilion茅simo"
+ }
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if cnum == 1:
+ if nnum < 1000000:
+ return next
+ ctext = "um"
+ elif cnum == 100 and not nnum == 1000:
+ ctext = "cento"
+
+ if nnum < cnum:
+ return ("%s e %s" % (ctext, ntext), cnum + nnum)
+
+ elif (not nnum % 1000000) and cnum > 1:
+ ntext = ntext[:-4] + "lh玫es"
+
+ if nnum == 100:
+ ctext = self.hundreds[cnum]
+ ntext = ""
+
+ else:
+ ntext = " " + ntext
+
+ return (ctext + ntext, cnum * nnum)
+
+ def to_cardinal(self, value):
+ result = lang_PT.Num2Word_EU.to_cardinal(self, value)
+
+ # Transforms "mil E cento e catorze reais" into "mil, cento e catorze
+ # reais"
+ for ext in (
+ 'mil', 'milh茫o', 'milh玫es', 'bilh茫o', 'bilh玫es',
+ 'trilh茫o', 'trilh玫es', 'quatrilh茫o', 'quatrilh玫es'):
+ if re.match('.*{} e \\w*ento'.format(ext), result):
+ result = result.replace(
+ '{} e'.format(ext), '{},'.format(ext), 1
+ )
+
+ return result
+
+ def to_currency(self, val, longval=True):
+ integer_part, decimal_part = ('%.2f' % val).split('.')
+
+ result = self.to_cardinal(int(integer_part))
+
+ appended_currency = False
+ for ext in (
+ 'milh茫o', 'milh玫es', 'bilh茫o', 'bilh玫es',
+ 'trilh茫o', 'trilh玫es', 'quatrilh茫o', 'quatrilh玫es'):
+ if result.endswith(ext):
+ result += ' de reais'
+ appended_currency = True
+
+ if result in ['um', 'menos um']:
+ result += ' real'
+ appended_currency = True
+ if not appended_currency:
+ result += ' reais'
+
+ if int(decimal_part):
+ cents = self.to_cardinal(int(decimal_part))
+ result += ' e ' + cents
+
+ if cents == 'um':
+ result += ' centavo'
+ else:
+ result += ' centavos'
+
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_RO.py b/fun_text_processing/num2words/num2words/lang_RO.py
new file mode 100644
index 0000000..8b539c3
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_RO.py
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_RO(lang_EU.Num2Word_EU):
+ GIGA_SUFFIX = "iliard/e"
+ MEGA_SUFFIX = "ilion"
+ # inflection for mi/billion follows different rule
+ MEGA_SUFFIX_I = "ilioane"
+ GIGA_SUFFIX_I = "iliarde"
+
+ def setup(self):
+ super(Num2Word_RO, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "virgul膬"
+ self.exclude_title = ["葯i", "virgul膬", "minus"]
+ self.errmsg_toobig = (
+ "Num膬rul e prea mare pentru a fi convertit 卯n cuvinte."
+ )
+ self.mid_numwords = [(1000, "mie/i"), (100, "sut膬/e"),
+ (90, "nou膬zeci"), (80, "optzeci"),
+ (70, "葯aptezeci"), (60, "葯aizeci"),
+ (50, "cincizeci"), (40, "patruzeci"),
+ (30, "treizeci")]
+ self.low_numwords = ["dou膬zeci", "nou膬sprezece", "optsprezece",
+ "葯aptesprezece", "葯aisprezece", "cincisprezece",
+ "paisprezece", "treisprezece", "doisprezece",
+ "unsprezece", "zece", "nou膬", "opt", "葯apte",
+ "葯ase", "cinci", "patru", "trei", "doi",
+ "unu", "zero"]
+ self.gen_numwords = ["", "o", "dou膬", "trei", "patru", "cinci",
+ "葯ase", "葯apte", "opt", "nou膬"]
+ self.gen_numwords_n = ["", "un", "dou膬", "trei", "patru", "cinci",
+ "葯ase", "葯apte", "opt", "nou膬"]
+ self.numwords_inflections = {
+ 100: self.gen_numwords,
+ 1000: self.gen_numwords,
+ 1000000: self.gen_numwords_n,
+ 1000000000: self.gen_numwords_n
+ }
+ self.ords = {"unu": "primul",
+ "doi": "al doilea",
+ "three": "al treilea",
+ "cinci": "al cincilea",
+ "opt": "al optulea",
+ "nou膬": "al nou膬lea",
+ "doisprezece": "al doisprezecelea"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ rtext_i = self.inflect(rnum, rtext, lnum)
+ if 1 <= lnum < 10:
+ if rnum not in self.numwords_inflections:
+ return (rtext, rnum)
+ else:
+ rtext_i = self.inflect(lnum * rnum, rtext, lnum)
+ lresult = (self.numwords_inflections[rnum][lnum], rtext_i)
+ return ("%s %s" % lresult, rnum)
+ elif 10 < lnum < 100:
+ if lnum % 10 == 0:
+ if rnum in self.numwords_inflections:
+ rtext_i = self.inflect(lnum * rnum, rtext, lnum)
+ return ("%s %s" % (ltext, rtext_i), lnum * rnum)
+ else:
+ return ("%s 葯i %s" % (ltext, rtext), lnum + rnum)
+ else:
+ rtext_i = self.inflect(lnum * rnum, rtext, lnum)
+ ltext_i = ltext if lnum % 10 != 2 \
+ else ltext.replace("doi", "dou膬")
+ return ("%s %s" % (ltext_i, rtext_i), lnum * rnum)
+ else:
+ if rnum in self.numwords_inflections:
+ rtext_i = self.inflect(lnum * rnum, rtext, lnum)
+ return ("%s %s" % (ltext, rtext_i), lnum * rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ if value == 1:
+ return "primul"
+ else:
+ value = self.to_cardinal(value)
+ return "al %slea" % (value)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ if value == 1:
+ return "1-ul"
+ return "al %s-lea" % (value)
+
+ def pluralize(self, n, forms):
+ if n == 1:
+ form = 0
+ elif n == 0 or (n % 100 > 0 and n % 100 < 20):
+ form = 1
+ else:
+ form = 2
+ return forms[form]
+
+ def inflect(self, value, text, side_effect=-1):
+ text = text.split("/")
+ result = text[0]
+ if len(text) > 1:
+ forms = [
+ text[0],
+ text[0][:-1] + text[1],
+ "de " + text[0][:-1] + text[1]
+ ]
+ result = self.pluralize(side_effect, forms)
+ # mega inflections are different
+ if side_effect > 1 and result.endswith(self.MEGA_SUFFIX):
+ result = result.replace(self.MEGA_SUFFIX, self.MEGA_SUFFIX_I)
+ elif side_effect > 1 and result.endswith("iliare"):
+ result = result.replace("iliare", self.GIGA_SUFFIX_I)
+ return result
+
+ def to_currency(self, val, currency="RON", cents=False, separator=" 葯i",
+ adjective=False):
+ # romanian currency has a particularity for numeral: one
+ self.gen_numwords[1] = "una"
+ result = super(Num2Word_RO, self).to_currency(
+ int(round(val*100)),
+ currency,
+ True,
+ separator,
+ adjective
+ )
+ self.gen_numwords[1] = "o" # revert numeral
+ return result.replace(
+ "unu leu", "un leu"
+ ).replace(
+ "unu ban", "un ban"
+ ).replace(
+ # if the romanian low text is 0, it is not usually printed
+ separator + " zero bani", ""
+ )
+
+ def to_year(self, val, suffix=None, longval=True):
+ result = super(Num2Word_RO, self).to_year(
+ val,
+ longval=longval
+ )
+ # for years we want the era negation e.g. B.C., in our case
+ # it's 卯.Hr. or 卯.e.n
+ if result.startswith(self.negword):
+ result = result.replace(self.negword, "")
+ suffix = "卯.Hr." if not suffix else suffix
+ if suffix:
+ result = "".join([
+ result,
+ " ",
+ suffix
+ ])
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_RU.py b/fun_text_processing/num2words/num2words/lang_RU.py
new file mode 100644
index 0000000..8cc9b15
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_RU.py
@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('薪芯谢褜',)
+
+ONES_FEMININE = {
+ 1: ('芯写薪邪',),
+ 2: ('写胁械',),
+ 3: ('褌褉懈',),
+ 4: ('褔械褌褘褉械',),
+ 5: ('锌褟褌褜',),
+ 6: ('褕械褋褌褜',),
+ 7: ('褋械屑褜',),
+ 8: ('胁芯褋械屑褜',),
+ 9: ('写械胁褟褌褜',),
+}
+
+ONES = {
+ 1: ('芯写懈薪',),
+ 2: ('写胁邪',),
+ 3: ('褌褉懈',),
+ 4: ('褔械褌褘褉械',),
+ 5: ('锌褟褌褜',),
+ 6: ('褕械褋褌褜',),
+ 7: ('褋械屑褜',),
+ 8: ('胁芯褋械屑褜',),
+ 9: ('写械胁褟褌褜',),
+}
+
+TENS = {
+ 0: ('写械褋褟褌褜',),
+ 1: ('芯写懈薪薪邪写褑邪褌褜',),
+ 2: ('写胁械薪邪写褑邪褌褜',),
+ 3: ('褌褉懈薪邪写褑邪褌褜',),
+ 4: ('褔械褌褘褉薪邪写褑邪褌褜',),
+ 5: ('锌褟褌薪邪写褑邪褌褜',),
+ 6: ('褕械褋褌薪邪写褑邪褌褜',),
+ 7: ('褋械屑薪邪写褑邪褌褜',),
+ 8: ('胁芯褋械屑薪邪写褑邪褌褜',),
+ 9: ('写械胁褟褌薪邪写褑邪褌褜',),
+}
+
+TWENTIES = {
+ 2: ('写胁邪写褑邪褌褜',),
+ 3: ('褌褉懈写褑邪褌褜',),
+ 4: ('褋芯褉芯泻',),
+ 5: ('锌褟褌褜写械褋褟褌',),
+ 6: ('褕械褋褌褜写械褋褟褌',),
+ 7: ('褋械屑褜写械褋褟褌',),
+ 8: ('胁芯褋械屑褜写械褋褟褌',),
+ 9: ('写械胁褟薪芯褋褌芯',),
+}
+
+HUNDREDS = {
+ 1: ('褋褌芯',),
+ 2: ('写胁械褋褌懈',),
+ 3: ('褌褉懈褋褌邪',),
+ 4: ('褔械褌褘褉械褋褌邪',),
+ 5: ('锌褟褌褜褋芯褌',),
+ 6: ('褕械褋褌褜褋芯褌',),
+ 7: ('褋械屑褜褋芯褌',),
+ 8: ('胁芯褋械屑褜褋芯褌',),
+ 9: ('写械胁褟褌褜褋芯褌',),
+}
+
+THOUSANDS = {
+ 1: ('褌褘褋褟褔邪', '褌褘褋褟褔懈', '褌褘褋褟褔'), # 10^3
+ 2: ('屑懈谢谢懈芯薪', '屑懈谢谢懈芯薪邪', '屑懈谢谢懈芯薪芯胁'), # 10^6
+ 3: ('屑懈谢谢懈邪褉写', '屑懈谢谢懈邪褉写邪', '屑懈谢谢懈邪褉写芯胁'), # 10^9
+ 4: ('褌褉懈谢谢懈芯薪', '褌褉懈谢谢懈芯薪邪', '褌褉懈谢谢懈芯薪芯胁'), # 10^12
+ 5: ('泻胁邪写褉懈谢谢懈芯薪', '泻胁邪写褉懈谢谢懈芯薪邪', '泻胁邪写褉懈谢谢懈芯薪芯胁'), # 10^15
+ 6: ('泻胁懈薪褌懈谢谢懈芯薪', '泻胁懈薪褌懈谢谢懈芯薪邪', '泻胁懈薪褌懈谢谢懈芯薪芯胁'), # 10^18
+ 7: ('褋械泻褋褌懈谢谢懈芯薪', '褋械泻褋褌懈谢谢懈芯薪邪', '褋械泻褋褌懈谢谢懈芯薪芯胁'), # 10^21
+ 8: ('褋械锌褌懈谢谢懈芯薪', '褋械锌褌懈谢谢懈芯薪邪', '褋械锌褌懈谢谢懈芯薪芯胁'), # 10^24
+ 9: ('芯泻褌懈谢谢懈芯薪', '芯泻褌懈谢谢懈芯薪邪', '芯泻褌懈谢谢懈芯薪芯胁'), # 10^27
+ 10: ('薪芯薪懈谢谢懈芯薪', '薪芯薪懈谢谢懈芯薪邪', '薪芯薪懈谢谢懈芯薪芯胁'), # 10^30
+}
+
+
+class Num2Word_RU(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'RUB': (
+ ('褉褍斜谢褜', '褉褍斜谢褟', '褉褍斜谢械泄'), ('泻芯锌械泄泻邪', '泻芯锌械泄泻懈', '泻芯锌械械泻')
+ ),
+ 'EUR': (
+ ('械胁褉芯', '械胁褉芯', '械胁褉芯'), ('褑械薪褌', '褑械薪褌邪', '褑械薪褌芯胁')
+ ),
+ 'USD': (
+ ('写芯谢谢邪褉', '写芯谢谢邪褉邪', '写芯谢谢邪褉芯胁'), ('褑械薪褌', '褑械薪褌邪', '褑械薪褌芯胁')
+ ),
+ 'UAH': (
+ ('谐褉懈胁薪邪', '谐褉懈胁薪褘', '谐褉懈胁械薪'), ('泻芯锌械泄泻邪', '泻芯锌械泄泻懈', '泻芯锌械械泻')
+ ),
+ 'KZT': (
+ ('褌械薪谐械', '褌械薪谐械', '褌械薪谐械'), ('褌懈褘薪', '褌懈褘薪邪', '褌懈褘薪芯胁')
+ ),
+ }
+
+ def setup(self):
+ self.negword = "屑懈薪褍褋"
+ self.pointword = "蟹邪锌褟褌邪褟"
+ self.ords = {"薪芯谢褜": "薪褍谢械胁芯泄",
+ "芯写懈薪": "锌械褉胁褘泄",
+ "写胁邪": "胁褌芯褉芯泄",
+ "褌褉懈": "褌褉械褌懈泄",
+ "褔械褌褘褉械": "褔械褌胁械褉褌褘泄",
+ "锌褟褌褜": "锌褟褌褘泄",
+ "褕械褋褌褜": "褕械褋褌芯泄",
+ "褋械屑褜": "褋械写褜屑芯泄",
+ "胁芯褋械屑褜": "胁芯褋褜屑芯泄",
+ "写械胁褟褌褜": "写械胁褟褌褘泄",
+ "褋褌芯": "褋芯褌褘泄"}
+ self.ords_feminine = {"芯写懈薪": "",
+ "芯写薪邪": "",
+ "写胁械": "写胁褍褏",
+ "褌褉懈": "褌褉褢褏",
+ "褔械褌褘褉械": "褔械褌褘褉褢褏",
+ "锌褟褌褜": "锌褟褌懈",
+ "褕械褋褌褜": "褕械褋褌懈",
+ "褋械屑褜": "褋械屑懈",
+ "胁芯褋械屑褜": "胁芯褋褜屑懈",
+ "写械胁褟褌褜": "写械胁褟褌懈"}
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return u'%s %s %s' % (
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return self._int2word(int(n))
+
+ def pluralize(self, n, forms):
+ if n % 100 < 10 or n % 100 > 20:
+ if n % 10 == 1:
+ form = 0
+ elif 5 > n % 10 > 1:
+ form = 1
+ else:
+ form = 2
+ else:
+ form = 2
+ return forms[form]
+
+ def to_ordinal(self, number):
+ self.verify_ordinal(number)
+ outwords = self.to_cardinal(number).split(" ")
+ lastword = outwords[-1].lower()
+ try:
+ if len(outwords) > 1:
+ if outwords[-2] in self.ords_feminine:
+ outwords[-2] = self.ords_feminine.get(
+ outwords[-2], outwords[-2])
+ elif outwords[-2] == '写械褋褟褌褜':
+ outwords[-2] = outwords[-2][:-1] + '懈'
+ if len(outwords) == 3:
+ if outwords[-3] in ['芯写懈薪', '芯写薪邪']:
+ outwords[-3] = ''
+ lastword = self.ords[lastword]
+ except KeyError:
+ if lastword[:-3] in self.ords_feminine:
+ lastword = self.ords_feminine.get(
+ lastword[:-3], lastword) + "褋芯褌褘泄"
+ elif lastword[-1] == "褜" or lastword[-2] == "褌":
+ lastword = lastword[:-1] + "褘泄"
+ elif lastword[-1] == "泻":
+ lastword = lastword + "芯胁芯泄"
+ elif lastword[-5:] == "写械褋褟褌":
+ lastword = lastword.replace('褜', '懈') + '褘泄'
+ elif lastword[-2] == "褔" or lastword[-1] == "褔":
+ if lastword[-2] == "褔":
+ lastword = lastword[:-1] + "薪褘泄"
+ if lastword[-1] == "褔":
+ lastword = lastword + "薪褘泄"
+ elif lastword[-1] == "薪" or lastword[-2] == "薪":
+ lastword = lastword[:lastword.rfind('薪') + 1] + "薪褘泄"
+ elif lastword[-1] == "写" or lastword[-2] == "写":
+ lastword = lastword[:lastword.rfind('写') + 1] + "薪褘泄"
+ outwords[-1] = self.title(lastword)
+ return " ".join(outwords).strip()
+
+ def _money_verbose(self, number, currency):
+ return self._int2word(number, currency == 'UAH')
+
+ def _cents_verbose(self, number, currency):
+ return self._int2word(number, currency in ('UAH', 'RUB'))
+
+ def _int2word(self, n, feminine=False):
+ if n < 0:
+ return ' '.join([self.negword, self._int2word(abs(n))])
+
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ elif n1 > 0:
+ ones = ONES_FEMININE if i == 1 or feminine and i == 0 else ONES
+ words.append(ones[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
diff --git a/fun_text_processing/num2words/num2words/lang_SI.py b/fun_text_processing/num2words/num2words/lang_SI.py
new file mode 100644
index 0000000..a18c608
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_SI.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+# Sinhala number to words
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_SI(lang_EU.Num2Word_EU):
+ # GIGA_SUFFIX = "iljarder"
+ # MEGA_SUFFIX = "iljoner"
+
+ # def set_high_numwords(self, high):
+ # cap = 3 + 6 * len(high)
+
+ # for word, n in zip(high, range(cap, 3, -6)):
+ # if self.GIGA_SUFFIX:
+ # self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ # if self.MEGA_SUFFIX:
+ # self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 0, -3)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+
+ def setup(self):
+ super(Num2Word_SI, self).setup()
+
+ self.negword = "喽呧订喾� "
+ self.pointword = "喽洁稓喾娻穫喾娻逗喽�"
+ self.exclude_title = ["喾勦窂", "喽洁稓喾娻穫喾娻逗喽�", "喽呧订喾�"]
+
+ self.high_numwords = [(10000000000, '喽多窉喽洁窉喽亨侗 喽穭喽亨稓喾�'),
+ (1000000000, '喽多窉喽洁窉喽亨侗喽亨稓喾�'),
+ (100000000, '喽膏窉喽洁窉喽亨侗 喾冟窉喽亨逗喽氞穵'),
+ (10000000,'喽穬 喽膏窉喽洁窉喽亨侗'),
+ (1000000, '喽膏窉喽洁窉喽亨侗喽亨稓喾�'),
+ (100000, '喽戉稓喾� 喽洁稓喾娻穫喽亨稓喾�'),
+ (1000, '喽穭喾冟稓喾�') ]
+ self.mid_numwords = [(1000, "喽穭喾冟稓喾�"), (100, "喾冟窉喽亨逗"),
+ (90, "喽呧侗喾栢穩喽氞穵"), (80, "喽呧穬喾栢穩"), (70, "喾勦窅喽穵喽窇"),
+ (60, "喾勦窅喽�"), (50, "喽脆侗喾冟穵"), (40, "喾勦董喾呧窉喾勦稓喾�"),
+ (30, "喽窉喾冟穵")]
+
+ self.low_numwords = ['喾�喾掄穬喾娻穬喽氞穵','喽穭喽编穩喽�','喽穭喽呧锭','喽窂喾勦董','喽穭喾冟逗','喽脆穭喾呧窚喾�','喽穭喾勦董喽�','喽穭喽窋喽�','喽窚喾呧窚喾冟穵','喽戉稓喾溹穮喾溹穬喾�','喽穬','喽编穩喽�','喽呧锭','喾勦董','喾勦逗','喽脆穭','喾勦董喽�','喽窋喽编穵','喽窓喽�','喽戉稓','喾佮窋喽编穵喽�']
+
+ self.ords = {'喽戉稓': '喽脆穮喽膏窋喾�喽�',
+ '喽窓喽�': '喽窓喾�喾愢侗喾�',
+ '喽窋喽编穵': '喽窓喾�喽�',
+ '喾勦董喽�': '喾勦董喽秽穩喽�',
+ '喽脆穭': '喽脆穬喾娻穩喽�',
+ '喾勦逗': '喾勦逗 喾�喽�',
+ '喾勦董': '喾勦董喾娻穩喽�',
+ '喽呧锭': '喽呧锭喾�喾愢侗喾�',
+ '喽编穩喽�': '喽编穩喾�喾愢侗喾�',
+ '喽穬': '喽穭喾�喽�'}
+
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s %s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = '喾冟窉喽亨穩喾冟锭 喽脆窓喽� ' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "喾冟窉喽亨逗喽氞穵" #hundred
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_SL.py b/fun_text_processing/num2words/num2words/lang_SL.py
new file mode 100644
index 0000000..fb0e287
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_SL.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2003, Taro Ogawa. All Rights Reserved.
+# Copyright (c) 2013, Savoir-faire Linux inc. All Rights Reserved.
+# Copyright (c) 2015, Blaz Bregar. All Rights Reserved.
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301 USA
+
+from __future__ import unicode_literals
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_SL(Num2Word_EU):
+ GIGA_SUFFIX = "ilijard"
+ MEGA_SUFFIX = "ilijon"
+
+ def setup(self):
+ super(Num2Word_SL, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "celih"
+ self.errmsg_nonnum = "Only numbers may be converted to words."
+ self.errmsg_toobig = "Number is too large to convert to words."
+ self.exclude_title = []
+
+ self.mid_numwords = [(1000, "tiso膷"), (900, "devetsto"),
+ (800, "osemsto"), (700, "sedemsto"),
+ (600, "拧eststo"), (500, "petsto"),
+ (400, "拧tiristo"), (300, "tristo"),
+ (200, "dvesto"), (100, "sto"),
+ (90, "devetdeset"), (80, "osemdeset"),
+ (70, "sedemdeset"), (60, "拧estdeset"),
+ (50, "petdeset"), (40, "拧tirideset"),
+ (30, "trideset")]
+ self.low_numwords = ["dvajset", "devetnajst", "osemnajst",
+ "sedemnajst", "拧estnajst", "petnajst",
+ "拧tirinajst", "trinajst", "dvanajst",
+ "enajst", "deset", "devet", "osem", "sedem",
+ "拧est", "pet", "拧tiri", "tri", "dve", "ena",
+ "ni膷"]
+ self.ords = {"ena": "prv",
+ "dve": "drug",
+ "tri": "tretj",
+ "拧tiri": "膷etrt",
+ "sedem": "sedm",
+ "osem": "osm",
+ "sto": "stot",
+ "tiso膷": "tiso膷",
+ "milijon": "milijont"
+ }
+ self.ordflag = False
+
+ def merge(self, curr, next):
+ ctext, cnum, ntext, nnum = curr + next
+
+ if ctext.endswith("dve") and self.ordflag and nnum <= 1000000:
+ ctext = ctext[:len(ctext)-1] + "a"
+
+ if ctext == "dve" and not self.ordflag and nnum < 1000000000:
+ ctext = "dva"
+
+ if (ctext.endswith("tri") or ctext.endswith("拧tiri")) and\
+ nnum == 1000000 and not self.ordflag:
+ if ctext.endswith("拧tiri"):
+ ctext = ctext[:-1]
+ ctext = ctext + "je"
+
+ if cnum >= 20 and cnum < 100 and nnum == 2:
+ ntext = "dva"
+
+ if ctext.endswith("ena") and nnum >= 1000:
+ ctext = ctext[0:-1]
+
+ if cnum == 1:
+ if nnum < 10**6 or self.ordflag:
+ return next
+ ctext = ""
+
+ if nnum > cnum:
+ if nnum >= 10**6:
+ if self.ordflag:
+ ntext += "t"
+
+ elif cnum == 2:
+ if ntext.endswith("d"):
+ ntext += "i"
+ else:
+ ntext += "a"
+
+ elif 2 < cnum < 5:
+ if ntext.endswith("d"):
+ ntext += "e"
+ elif not ntext.endswith("d"):
+ ntext += "i"
+
+ elif ctext.endswith("en"):
+ if ntext.endswith("d") or ntext.endswith("n"):
+ ntext += ""
+
+ elif ctext.endswith("dve") and ntext.endswith("n"):
+ ctext = ctext[:-1] + "a"
+ ntext += "a"
+
+ elif ctext.endswith("je") and ntext.endswith("n"):
+ ntext += "i"
+
+ else:
+ if ntext.endswith("d"):
+ ntext += "a"
+ elif ntext.endswith("n"):
+ ntext += ""
+ elif ntext.endswith("d"):
+ ntext += "e"
+ else:
+ ntext += "ov"
+
+ if nnum >= 10**2 and self.ordflag is False and ctext:
+ ctext += " "
+
+ val = cnum * nnum
+ else:
+ if nnum < 10 < cnum < 100:
+ ntext, ctext = ctext, ntext + "in"
+ elif cnum >= 10**2 and self.ordflag is False:
+ ctext += " "
+ val = cnum + nnum
+
+ word = ctext + ntext
+ return (word, val)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ self.ordflag = True
+ outword = self.to_cardinal(value)
+ self.ordflag = False
+ for key in self.ords:
+ if outword.endswith(key):
+ outword = outword[:len(outword) - len(key)] + self.ords[key]
+ break
+ return outword + "i"
+
+ # Is this correct??
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return str(value) + "."
+
+ def to_currency(self, val, longval=True, old=False):
+ if old:
+ return self.to_splitnum(val, hightxt="evro/a/v",
+ lowtxt="stotin/a/i/ov",
+ jointxt="in", longval=longval)
+ return super(Num2Word_SL, self).to_currency(val, jointxt="in",
+ longval=longval)
+
+ def to_year(self, val, longval=True):
+ if not (val//100) % 10:
+ return self.to_cardinal(val)
+ return self.to_splitnum(val, hightxt="hundert", longval=longval)
diff --git a/fun_text_processing/num2words/num2words/lang_SR.py b/fun_text_processing/num2words/num2words/lang_SR.py
new file mode 100644
index 0000000..cd29489
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_SR.py
@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .currency import parse_currency_parts, prefix_currency
+from .utils import get_digits, splitbyx
+
+ZERO = ('nula',)
+
+ONES = {
+ 1: ('jedan', 'jedna'),
+ 2: ('dva', 'dve'),
+ 3: ('tri', 'tri'),
+ 4: ('膷etiri', '膷etiri'),
+ 5: ('pet', 'pet'),
+ 6: ('拧est', '拧est'),
+ 7: ('sedam', 'sedam'),
+ 8: ('osam', 'osam'),
+ 9: ('devet', 'devet'),
+}
+
+TENS = {
+ 0: ('deset',),
+ 1: ('jedanaest',),
+ 2: ('dvanaest',),
+ 3: ('trinaest',),
+ 4: ('膷etrnaest',),
+ 5: ('petnaest',),
+ 6: ('拧esnaest',),
+ 7: ('sedamnaest',),
+ 8: ('osamnaest',),
+ 9: ('devetnaest',),
+}
+
+TWENTIES = {
+ 2: ('dvadeset',),
+ 3: ('trideset',),
+ 4: ('膷etrdeset',),
+ 5: ('pedeset',),
+ 6: ('拧ezdeset',),
+ 7: ('sedamdeset',),
+ 8: ('osamdeset',),
+ 9: ('devedeset',),
+}
+
+HUNDREDS = {
+ 1: ('sto',),
+ 2: ('dvesta',),
+ 3: ('trista',),
+ 4: ('膷etristo',),
+ 5: ('petsto',),
+ 6: ('拧esto',),
+ 7: ('sedamsto',),
+ 8: ('osamsto',),
+ 9: ('devetsto',),
+}
+
+SCALE = {
+ 0: ('', '', '', False),
+ 1: ('hiljada', 'hiljade', 'hiljada', True), # 10^3
+ 2: ('milion', 'miliona', 'miliona', False), # 10^6
+ 3: ('bilion', 'biliona', 'biliona', False), # 10^9
+ 4: ('trilion', 'triliona', 'triliona', False), # 10^12
+ 5: ('kvadrilion', 'kvadriliona', 'kvadriliona', False), # 10^15
+ 6: ('kvintilion', 'kvintiliona', 'kvintiliona', False), # 10^18
+ 7: ('sekstilion', 'sekstiliona', 'sekstiliona', False), # 10^21
+ 8: ('septilion', 'septiliona', 'septiliona', False), # 10^24
+ 9: ('oktilion', 'oktiliona', 'oktiliona', False), # 10^27
+ 10: ('nonilion', 'noniliona', 'noniliona', False), # 10^30
+}
+
+
+class Num2Word_SR(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'RUB': (
+ ('rublja', 'rublje', 'rublji', True),
+ ('kopejka', 'kopejke', 'kopejki', True)
+ ),
+ 'EUR': (
+ ('evro', 'evra', 'evra', False),
+ ('cent', 'centa', 'centi', False)
+ ),
+ 'RSD': (
+ ('dinar', 'dinara', 'dinara', False),
+ ('para', 'pare', 'para', True)
+ ),
+ }
+
+ def setup(self):
+ self.negword = "minus"
+ self.pointword = "zapeta"
+
+ def to_cardinal(self, number, feminine=False):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right), feminine))
+ return u'%s %s %s' % (
+ self._int2word(int(left), feminine),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return self._int2word(int(n), feminine)
+
+ def pluralize(self, number, forms):
+ if number % 100 < 10 or number % 100 > 20:
+ if number % 10 == 1:
+ form = 0
+ elif 1 < number % 10 < 5:
+ form = 1
+ else:
+ form = 2
+ else:
+ form = 2
+ return forms[form]
+
+ def to_ordinal(self, number):
+ raise NotImplementedError()
+
+ def _cents_verbose(self, number, currency):
+ return self._int2word(
+ number,
+ self.CURRENCY_FORMS[currency][1][-1]
+ )
+
+ def _int2word(self, number, feminine=False):
+ if number < 0:
+ return ' '.join([self.negword, self._int2word(abs(number))])
+
+ if number == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(number), 3))
+ chunk_len = len(chunks)
+ for chunk in chunks:
+ chunk_len -= 1
+ digit_right, digit_mid, digit_left = get_digits(chunk)
+
+ if digit_left > 0:
+ words.append(HUNDREDS[digit_left][0])
+
+ if digit_mid > 1:
+ words.append(TWENTIES[digit_mid][0])
+
+ if digit_mid == 1:
+ words.append(TENS[digit_right][0])
+ elif digit_right > 0:
+ is_feminine = feminine or SCALE[chunk_len][-1]
+ gender_idx = int(is_feminine)
+ words.append(
+ ONES[digit_right][gender_idx]
+ )
+
+ if chunk_len > 0 and chunk != 0:
+ words.append(self.pluralize(chunk, SCALE[chunk_len]))
+
+ return ' '.join(words)
+
+ def to_currency(self, val, currency='EUR', cents=True, separator=',',
+ adjective=False):
+ """
+ Args:
+ val: Numeric value
+ currency (str): Currency code
+ cents (bool): Verbose cents
+ separator (str): Cent separator
+ adjective (bool): Prefix currency name with adjective
+ Returns:
+ str: Formatted string
+
+ """
+ left, right, is_negative = parse_currency_parts(val)
+
+ try:
+ cr1, cr2 = self.CURRENCY_FORMS[currency]
+
+ except KeyError:
+ raise NotImplementedError(
+ 'Currency code "%s" not implemented for "%s"' %
+ (currency, self.__class__.__name__))
+
+ if adjective and currency in self.CURRENCY_ADJECTIVES:
+ cr1 = prefix_currency(
+ self.CURRENCY_ADJECTIVES[currency],
+ cr1
+ )
+
+ minus_str = "%s " % self.negword if is_negative else ""
+ cents_str = self._cents_verbose(right, currency) \
+ if cents else self._cents_terse(right, currency)
+
+ return u'%s%s %s%s %s %s' % (
+ minus_str,
+ self.to_cardinal(left, feminine=cr1[-1]),
+ self.pluralize(left, cr1),
+ separator,
+ cents_str,
+ self.pluralize(right, cr2)
+ )
diff --git a/fun_text_processing/num2words/num2words/lang_SV.py b/fun_text_processing/num2words/num2words/lang_SV.py
new file mode 100644
index 0000000..f070933
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_SV.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_SV(lang_EU.Num2Word_EU):
+ GIGA_SUFFIX = "iljarder"
+ MEGA_SUFFIX = "iljoner"
+
+ def set_high_numwords(self, high):
+ cap = 3 + 6 * len(high)
+
+ for word, n in zip(high, range(cap, 3, -6)):
+ if self.GIGA_SUFFIX:
+ self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ if self.MEGA_SUFFIX:
+ self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+
+ def setup(self):
+ super(Num2Word_SV, self).setup()
+
+ self.negword = "minus "
+ self.pointword = "komma"
+ self.exclude_title = ["och", "komma", "minus"]
+
+ self.mid_numwords = [(1000, "tusen"), (100, "hundra"),
+ (90, "nittio"), (80, "氓ttio"), (70, "sjuttio"),
+ (60, "sextio"), (50, "femtio"), (40, "f枚rtio"),
+ (30, "trettio")]
+ self.low_numwords = ["tjugo", "nitton", "arton", "sjutton",
+ "sexton", "femton", "fjorton", "tretton",
+ "tolv", "elva", "tio", "nio", "氓tta",
+ "sju", "sex", "fem", "fyra", "tre", "tv氓",
+ "ett", "noll"]
+ self.ords = {"noll": "nollte",
+ "ett": "f枚rsta",
+ "tv氓": "andra",
+ "tre": "tredje",
+ "fyra": "fj盲rde",
+ "fem": "femte",
+ "sex": "sj盲tte",
+ "sju": "sjunde",
+ "氓tta": "氓ttonde",
+ "nio": "nionde",
+ "tio": "tionde",
+ "elva": "elfte",
+ "tolv": "tolfte",
+ "tjugo": "tjugonde"}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif rnum >= 1000000 and lnum == 1:
+ return ("%s %s" % ('en', rtext[:-2]), lnum + rnum)
+ elif rnum >= 1000000 and lnum > 1:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value).split(" ")
+ lastword = outwords[-1]
+ ending_length = 0
+ try:
+ lastword_ending = self.ords[lastword[-4:]]
+ ending_length = 4
+ except KeyError:
+ try:
+ lastword_ending = self.ords[lastword[-3:]]
+ ending_length = 3
+ except KeyError:
+ lastword_ending = "de"
+ if lastword_ending == 'de':
+ lastword_first_part = self.title(lastword)[:]
+ else:
+ lastword_first_part = self.title(lastword)[:-ending_length]
+ lastword_correct = lastword_first_part + lastword_ending
+ outwords[-1] = lastword_correct
+ return " ".join(outwords)
+
+ def to_ordinal_num(self, value):
+ raise NotImplementedError(
+ "'ordinal_num' is not implemented for swedish language")
+
+ def to_year(self, val, longval=True):
+ raise NotImplementedError(
+ "'year' is not implemented for swedish language")
+
+ def to_currency(self, val, longval=True):
+ raise NotImplementedError(
+ "'currency' is not implemented for swedish language")
diff --git a/fun_text_processing/num2words/num2words/lang_SW.py b/fun_text_processing/num2words/num2words/lang_SW.py
new file mode 100644
index 0000000..22c58fa
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_SW.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+# Swahili number to words
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_SW(lang_EU.Num2Word_EU):
+ # GIGA_SUFFIX = "iljarder"
+ # MEGA_SUFFIX = "iljoner"
+
+ # def set_high_numwords(self, high):
+ # cap = 3 + 6 * len(high)
+
+ # for word, n in zip(high, range(cap, 3, -6)):
+ # if self.GIGA_SUFFIX:
+ # self.cards[10 ** n] = word + self.GIGA_SUFFIX
+
+ # if self.MEGA_SUFFIX:
+ # self.cards[10 ** (n - 3)] = word + self.MEGA_SUFFIX
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 0, -3)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+
+ def setup(self):
+ super(Num2Word_SW, self).setup()
+
+ self.negword = "kuondoa "
+ self.pointword = "hatua"
+ self.exclude_title = ["na", "hatua", "kuondoa"]
+
+ self.high_numwords = [(10000000000, 'bilioni kumi'),
+ (1000000000, 'bilioni moja'),
+ (100000000, 'milioni mia moja'),
+ (10000000,'milioni kumi'),
+ (1000000, 'milioni moja'),
+ (100000, 'laki moja'),
+ (1000, 'elfu moja') ]
+ self.mid_numwords = [(1000, "elfu moja"), (100, "mia moja"),
+ (90, "tisini"), (80, "themanini"), (70, "sabini"),
+ (60, "sitini"), (50, "hamsini"), (40, "arobaini"),
+ (30, "thelathini")]
+
+ self.low_numwords = ['sufuri', 'moja', 'mbili', 'tatu', 'nne', 'tano', 'sita', 'saba', 'nane', 'tisa', 'kumi', 'kumi na moja', 'kumi na mbili', 'kumi na tatu', 'kumi na nne', 'kumi na tano', 'kumi na sita', 'kumi na saba', 'kumi na nane', 'kumi na tisa', 'ishirini']
+
+ self.ords = {"moja":"kwanza",
+ "mbili":"pili",
+ "tatu":"cha tatu",
+ "nne":"nne",
+ "tano":"tano",
+ "sita":"ya sita",
+ "saba":"ya saba",
+ "nane":"ya nane",
+ "tisa":"ya tisa",
+ "kumi":"ya kumi"}
+
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s %s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'kabla ya karne ' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "mia"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s %s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s %s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_TE.py b/fun_text_processing/num2words/num2words/lang_TE.py
new file mode 100644
index 0000000..274d312
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_TE.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .lang_EU import Num2Word_EU
+
+
+class Num2Word_TE(Num2Word_EU):
+ def set_high_numwords(self, high):
+ for n, word in self.high_numwords:
+ self.cards[10**n] = word
+
+ def setup(self):
+ self.low_numwords = [
+ "喟む眾喟傕碍啾嗋睎 喟む眾喟睄喟翱喟︵翱",
+ "喟む眾喟傕碍啾嗋睎 喟庎皑喟苦爱喟苦唉喟�",
+ "喟む眾喟傕碍啾嗋睎 喟忇啊啾�",
+ "喟む眾喟傕碍啾嗋睎 喟嗋鞍啾�",
+ "喟む眾喟傕碍啾嗋睎 喟呧隘喟苦唉啾�",
+ "喟む眾喟傕碍啾嗋睎 喟ㄠ熬喟侧眮喟椸眮",
+ "喟む眾喟傕碍啾嗋睎 喟眰喟∴眮",
+ "喟む眾喟傕碍啾嗋睎 喟班眴喟傕啊啾�",
+ "喟む眾喟傕碍啾嗋睎 喟掄皶喟熰翱",
+ "喟む眾喟傕碍啾嗋睎",
+ "喟庎皑喟眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟庎皑喟眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟庎皑喟眴啾� 喟忇啊啾�",
+ "喟庎皑喟眴啾� 喟嗋鞍啾�",
+ "喟庎皑喟眴啾� 喟呧隘喟苦唉啾�",
+ "喟庎皑喟眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟庎皑喟眴啾� 喟眰喟∴眮",
+ "喟庎皑喟眴啾� 喟班眴喟傕啊啾�",
+ "喟庎皑喟眴啾� 喟掄皶喟熰翱",
+ "喟庎皑喟眴啾�",
+ "喟∴眴喟睄喟眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟∴眴喟睄喟眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟∴眴喟睄喟眴啾� 喟忇啊啾�",
+ "喟∴眴喟睄喟眴啾� 喟嗋鞍啾�",
+ "喟∴眴喟睄喟眴啾� 喟呧隘喟苦唉啾�",
+ "喟∴眴喟睄喟眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟∴眴喟睄喟眴啾� 喟眰喟∴眮",
+ "喟∴眴喟睄喟眴啾� 喟班眴喟傕啊啾�",
+ "喟∴眴喟睄喟眴啾� 喟掄皶喟熰翱",
+ "喟∴眴喟睄喟眴啾�",
+ "喟呧鞍喟掂眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟呧鞍喟掂眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟呧鞍喟掂眴啾� 喟忇啊啾�",
+ "喟呧鞍喟掂眴啾� 喟嗋鞍啾�",
+ "喟呧鞍喟掂眴啾� 喟呧隘喟苦唉啾�",
+ "喟呧鞍喟掂眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟呧鞍喟掂眴啾� 喟眰喟∴眮",
+ "喟呧鞍喟掂眴啾� 喟班眴喟傕啊啾�",
+ "喟呧鞍喟掂眴啾� 喟掄皶喟熰翱",
+ "喟呧鞍喟掂眴啾�",
+ "喟熬喟眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟熬喟眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟熬喟眴啾� 喟忇啊啾�",
+ "喟熬喟眴啾� 喟嗋鞍啾�",
+ "喟熬喟眴啾� 喟呧隘喟苦唉啾�",
+ "喟熬喟眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟熬喟眴啾� 喟眰喟∴眮",
+ "喟熬喟眴啾� 喟班眴喟傕啊啾�",
+ "喟熬喟眴啾� 喟掄皶喟熰翱",
+ "喟熬喟眴啾� ",
+ "喟ㄠ安喟眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟ㄠ安喟眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟ㄠ安喟眴啾� 喟忇啊啾�",
+ "喟ㄠ安喟眴啾� 喟嗋鞍啾�",
+ "喟ㄠ安喟眴啾� 喟呧隘喟苦唉啾�",
+ "喟ㄠ安喟眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟ㄠ安喟眴啾� 喟眰喟∴眮",
+ "喟ㄠ安喟眴啾� 喟班眴喟傕啊啾�",
+ "喟ㄠ安喟眴啾� 喟掄皶喟熰翱",
+ "喟ㄠ安喟眴啾�",
+ "喟眮喟睄喟眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟眮喟睄喟眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟眮喟睄喟眴啾� 喟忇啊啾�",
+ "喟眮喟睄喟眴啾� 喟嗋鞍啾�",
+ "喟眮喟睄喟眴啾� 喟愢唉啾�",
+ "喟眮喟睄喟眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟眮喟睄喟眴啾� 喟眰喟∴眮",
+ "喟眮喟睄喟眴啾� 喟班眴喟傕啊啾�",
+ "喟眮喟睄喟眻 喟掄皶喟熰翱",
+ "喟眮喟睄喟眴啾�",
+ "喟囙鞍喟掂眴啾� 喟む眾喟睄喟翱喟︵翱",
+ "喟囙鞍喟掂眴啾� 喟庎皑喟苦爱喟苦唉喟�",
+ "喟囙鞍喟掂眴啾� 喟忇啊啾�",
+ "喟囙鞍喟掂眴啾� 喟嗋鞍啾�",
+ "喟囙鞍喟掂眻 喟呧隘喟苦唉啾�",
+ "喟囙鞍喟掂眴啾� 喟ㄠ熬喟侧眮喟椸眮",
+ "喟囙鞍喟掂眴啾� 喟眰喟∴眮",
+ "喟囙鞍喟掂眴啾� 喟班眴喟傕啊啾�",
+ "喟囙鞍喟掂眴啾� 喟掄皶喟熰翱",
+ "喟囙鞍喟掂眴啾�",
+ "喟皞喟︵眾喟睄喟翱喟︵翱",
+ "喟哀啾嵿哀啾嗋皑喟苦爱喟苦唉喟�",
+ "喟唉喟苦肮啾囙啊啾�",
+ "喟唉喟灌熬喟班眮",
+ "喟唉啾佮皑喟翱喟︵眮",
+ "喟哀啾嵿皑喟距安啾佮皸啾�",
+ "喟唉喟眰喟∴眮",
+ "喟皑啾嵿皑啾嗋皞喟∴眮",
+ "喟唉喟曕眾喟傕啊啾�",
+ "喟唉喟�",
+ "喟む眾喟睄喟翱喟︵翱",
+ "喟庎皑喟苦爱喟苦唉喟�",
+ "喟忇啊啾�",
+ "喟嗋鞍啾�",
+ "喟呧隘喟苦唉啾�",
+ "喟ㄠ熬喟侧眮喟椸眮",
+ "喟眰喟∴眮",
+ "喟班眴喟傕啊啾�",
+ "喟掄皶喟熰翱",
+ "喟膏眮喟ㄠ睄喟�",
+ ]
+
+ self.mid_numwords = [(100, "喟掂皞喟�")]
+
+ self.high_numwords = [(7, "喟曕眿喟熰睄喟�"), (5, "喟侧皶啾嵿胺"), (3, "喟掂眹喟翱")]
+
+ self.pointword = "喟翱喟傕唉啾佮暗啾� "
+
+ self.modifiers = [
+ " 啾� ",
+ "喟� ",
+ " 喟� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾� ",
+ " 啾�",
+ " 啾� ",
+ " 啾� ",
+ " 喟� ",
+ " 喟� ",
+ " 喟� ",
+ ]
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s-%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ if ltext[-1] in self.modifiers:
+ return ("%s %s" % (ltext[:-1], rtext), lnum + rnum)
+ else:
+ return ("%s %s" % (ltext+"喟�", rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s %s" % (ltext, rtext), lnum * rnum)
+ return ("%s %s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value)[-1:])
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ outwords = self.to_cardinal(value)
+ if outwords[-1] in self.modifiers:
+ outwords = outwords[:-1]
+ ordinal_num = outwords + "喟�"
+ return ordinal_num
diff --git a/fun_text_processing/num2words/num2words/lang_TH.py b/fun_text_processing/num2words/num2words/lang_TH.py
new file mode 100644
index 0000000..cd60ee9
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_TH.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from num2words.base import Num2Word_Base
+from num2words.currency import parse_currency_parts
+from num2words.utils import splitbyx
+
+
+class Num2Word_TH(Num2Word_Base):
+
+ def setup(self):
+ self.negword = '喔曕复喔斷弗喔�'
+ self.pointword = '喔堗父喔�'
+
+ self.CURRENCY_FORMS = {
+ 'THB': (('喔氞覆喔�', '喔氞覆喔�'), ('喔笗喔侧竾喔勦箤', '喔笗喔侧竾喔勦箤')),
+ 'USD': (('喔斷腑喔ム弗喔侧福喙�', '喔斷腑喔ム弗喔侧福喙�'), ('喙�喔嬥笝喔曕箤', '喙�喔嬥笝喔曕箤')),
+ 'EUR': (('喔⑧腹喙傕福', '喔⑧腹喙傕福'), ('喙�喔嬥笝喔曕箤', '喙�喔嬥笝喔曕箤')),
+ }
+
+ self.high_numwords = []
+
+ self.mid_numwords = ['', '喔复喔�', '喔｀箟喔涪', '喔炧副喔�', '喔浮喔粪箞喔�', '喙佮釜喔�', '喔ム箟喔侧笝']
+
+ self.low_numwords = [
+ '喔ㄠ腹喔權涪喙�', '喔笝喔多箞喔�', '喔腑喔�', '喔覆喔�', '喔傅喙�',
+ '喔箟喔�', '喔竵', '喙�喔堗箛喔�', '喙佮笡喔�', '喙�喔佮箟喔�'
+ ]
+
+ def set_high_numwords(self, high_numwords):
+ pass
+
+ def set_mid_numwords(self, mid_numwords):
+ pass
+
+ def splitnum(self, six_num):
+ length = len(six_num) > 1
+ word_num = ''
+
+ for index, num in enumerate(map(int, six_num)):
+ if num:
+ if index:
+ word_num = self.mid_numwords[index] + word_num
+
+ if length and num == 1 and index == 0:
+ word_num += '喙�喔箛喔�'
+ elif index == 1 and num == 2:
+ word_num = '喔⑧傅喙�' + word_num
+ elif index != 1 or num != 1:
+ word_num = self.low_numwords[num] + word_num
+
+ elif num == 0 and index == 0 and length == 0:
+ word_num = self.low_numwords[0]
+
+ return word_num
+
+ def split_six(self, num_txt):
+ result = splitbyx(num_txt, 6, format_int=False)
+ result = list(result)[::-1]
+ number_list = []
+ for i in result:
+ number_list.append(i[::-1])
+ return number_list
+
+ def add_text_million(self, word_num):
+ result = ''
+
+ for index, t in enumerate(reversed(word_num)):
+ if index == 0:
+ result = t
+ else:
+ result = result + '喔ム箟喔侧笝' + t
+
+ return result
+
+ def round_2_decimal(self, number):
+ integer, cents, negative = parse_currency_parts(
+ number, is_int_with_cents=False
+ )
+ integer = '{}'.format(integer)
+ cents = '{}'.format(cents)
+
+ if len(cents) < 2:
+ add_zero = 2 - len(cents)
+ cents = ('0' * add_zero) + cents
+
+ text_num = integer + '.' + cents
+
+ return text_num, negative
+
+ def left_num_to_text(self, number):
+
+ left_num_list = self.split_six(number)
+
+ left_text_list = []
+ for i in left_num_list:
+ left_text_list.append(self.splitnum(i))
+
+ left_text = self.add_text_million(left_text_list)
+ return left_text
+
+ def to_cardinal(self, number):
+ negative = number < 0
+
+ pre, post = self.float2tuple(number)
+ precision = self.precision
+ pre = '{}'.format(pre)
+ post = '{}'.format(post)
+
+ if negative:
+ pre = pre.lstrip('-')
+
+ if len(post) < precision:
+ add_zero = precision - len(post)
+ post = ('0' * add_zero) + post
+
+ result = self.left_num_to_text(pre)
+
+ right_text = ''
+ if not post == '0':
+ for i in map(int, post):
+ right_text = right_text + self.low_numwords[i]
+ result = result + '喔堗父喔�' + right_text
+
+ if negative:
+ result = '喔曕复喔斷弗喔�' + result
+
+ return result
+
+ def to_ordinal(self, number):
+ return self.to_cardinal(number)
+
+ def to_currency(self, number, currency='THB'):
+
+ number, negative = self.round_2_decimal(number)
+
+ split_num = number.split('.')
+
+ left_num = split_num[0]
+ left_text = self.left_num_to_text(left_num)
+
+ right_num = split_num[1]
+ right_text = self.splitnum(right_num[::-1].rstrip('0'))
+
+ try:
+ cr1, cr2 = self.CURRENCY_FORMS[currency]
+
+ except KeyError:
+ raise NotImplementedError(
+ 'Currency code "%s" not implemented for "%s"' %
+ (currency, self.__class__.__name__))
+
+ if right_num == '00':
+ if currency == 'THB':
+ result = left_text + cr1[0] + '喔栢箟喔о笝'
+ else:
+ result = left_text + cr1[0]
+ else:
+ if left_num == '0':
+ result = right_text + cr2[0]
+ else:
+ result = left_text + cr1[0] + right_text + cr2[0]
+
+ if negative:
+ result = self.negword + result
+
+ return result
diff --git a/fun_text_processing/num2words/num2words/lang_TR.py b/fun_text_processing/num2words/num2words/lang_TR.py
new file mode 100644
index 0000000..7fba871
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_TR.py
@@ -0,0 +1,846 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2003, Taro Ogawa. All Rights Reserved.
+# Copyright (c) 2013, Savoir-faire Linux inc. All Rights Reserved.
+# Copyright (c) 2017, Tufan Kaynak, Framras. All Rights Reserved.
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301 USA
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+
+
+class Num2Word_TR(Num2Word_Base):
+ def __init__(self):
+ self.precision = 2
+ self.negword = u"eksi"
+ self.pointword = u"virg眉l"
+ self.CURRENCY_UNIT = u"lira"
+ self.CURRENCY_SUBUNIT = u"kuru艧"
+ self.errmsg_nonnum = u"Sadece say谋lar yaz谋ya 莽evrilebilir."
+ self.errmsg_floatord = u"Tam say谋 olmayan {} s谋ralamada kullan谋lamaz."
+ self.errmsg_negord = u"Pozitif olmayan {} s谋ralamada kullan谋lamaz."
+ self.errmsg_toobig = u"abs({}) say谋 yaz谋ya 莽evirmek i莽in 莽ok b眉y眉k. " \
+ u"Yaz谋ya 莽evrilebilecek en b眉y眉k rakam {}."
+ self.exclude_title = []
+ self.DECIMAL_SIGN = ","
+ self.ORDINAL_SIGN = "."
+ self.ZERO = u"s谋f谋r"
+ self.CARDINAL_ONES = {
+ "1": u"bir",
+ "2": u"iki",
+ "3": u"眉莽",
+ "4": u"d枚rt",
+ "5": u"be艧",
+ "6": u"alt谋",
+ "7": u"yedi",
+ "8": u"sekiz",
+ "9": u"dokuz"
+ }
+ self.ORDINAL_ONES = {
+ "1": u"birinci",
+ "2": u"ikinci",
+ "3": u"眉莽眉nc眉",
+ "4": u"d枚rd眉nc眉",
+ "5": u"be艧inci",
+ "6": u"alt谋nc谋",
+ "7": u"yedinci",
+ "8": u"sekizinci",
+ "9": u"dokuzuncu"
+ }
+ self.CARDINAL_TENS = {
+ "1": u"on",
+ "2": u"yirmi",
+ "3": u"otuz",
+ "4": u"k谋rk",
+ "5": u"elli",
+ "6": u"altm谋艧",
+ "7": u"yetmi艧",
+ "8": u"seksen",
+ "9": u"doksan"
+ }
+ self.ORDINAL_TENS = {
+ "1": u"onuncu",
+ "2": u"yirminci",
+ "3": u"otuzuncu",
+ "4": u"k谋rk谋nc谋",
+ "5": u"ellinci",
+ "6": u"altm谋艧谋nc谋",
+ "7": u"yetmi艧inci",
+ "8": u"sekseninci",
+ "9": u"doksan谋nc谋"
+ }
+ self.HUNDREDS = {
+ "2": u"iki",
+ "3": u"眉莽",
+ "4": u"d枚rt",
+ "5": u"be艧",
+ "6": u"alt谋",
+ "7": u"yedi",
+ "8": u"sekiz",
+ "9": u"dokuz"
+ }
+ self.CARDINAL_HUNDRED = (u"y眉z",)
+ self.ORDINAL_HUNDRED = (u"y眉z眉nc眉",)
+ self.CARDINAL_TRIPLETS = {
+ 1: u"bin",
+ 2: u"milyon",
+ 3: u"milyar",
+ 4: u"trilyon",
+ 5: u"katrilyon",
+ 6: u"kentilyon"
+ }
+ self.ORDINAL_TRIPLETS = {
+ 1: u"bininci",
+ 2: u"milyonuncu",
+ 3: u"milyar谋nc谋",
+ 4: u"trilyonuncu",
+ 5: u"katrilyonuncu",
+ 6: u"kentilyon"
+ }
+ self.MAXVAL = (10 ** ((len(self.CARDINAL_TRIPLETS) + 1) * 3)) - 1
+
+ self.integers_to_read = []
+ self.total_triplets_to_read = 0
+ self.total_digits_outside_triplets = 0
+ self.order_of_last_zero_digit = 0
+
+ def to_cardinal(self, value):
+ wrd = ""
+ is_cardinal = self.verify_cardinal(value)
+ if not is_cardinal:
+ return wrd
+
+ if not int(value) == value:
+ return self.to_cardinal_float(value)
+ self.to_splitnum(value)
+
+ if self.order_of_last_zero_digit >= len(self.integers_to_read[0]):
+ # number like 00 and all 0s and even more, raise error
+ return wrd
+
+ if self.total_triplets_to_read == 1:
+ if self.total_digits_outside_triplets == 2:
+ if self.order_of_last_zero_digit == 1:
+ # number like x0, read cardinal x0 and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ return wrd
+ if self.order_of_last_zero_digit == 0:
+ # number like xy, read cardinal xy and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ return wrd
+
+ if self.total_digits_outside_triplets == 1:
+ if self.order_of_last_zero_digit == 0:
+ # number like x, read cardinal x and return
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ if self.integers_to_read[0][0] == "0":
+ return self.ZERO
+ return wrd
+
+ if self.total_digits_outside_triplets == 0:
+ if self.order_of_last_zero_digit == 2:
+ # number like x00, read cardinal x00 and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ return wrd
+ if self.order_of_last_zero_digit == 1:
+ # number like xy0, read cardinal xy0 and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ return wrd
+ if self.order_of_last_zero_digit == 0:
+ # number like xyz, read cardinal xyz and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ return wrd
+
+ if self.total_triplets_to_read >= 2:
+ if self.total_digits_outside_triplets == 2:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x0 and all 0s, read cardinal x0 0..0
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 2:
+ # number like xy and all 0s, read cardinal xy 0..0
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 2:
+ # number like xy and others, read cardinal xy n..n
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ if self.total_digits_outside_triplets == 1:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x and all 0s, read cardinal x 0..0
+ # and return
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][0] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 1:
+ # number like x and others, read cardinal x n..n
+ # and return
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][0] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ if self.total_digits_outside_triplets == 0:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x00 and all 0s, read cardinal x00 0..0
+ # and return
+ wrd += self.HUNDREDS.get(self.integers_to_read[0][0], "")
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 2:
+ # number like xy0 and all 0s, read cardinal xy0 0..0
+ # and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 3:
+ # number like xyz and all 0s, read cardinal xyz 0..0
+ # and return
+ wrd += self.HUNDREDS.get(self.integers_to_read[0][0], "")
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 3:
+ # number like xyz and all others, read cardinal xyz n..n
+ wrd += self.HUNDREDS.get(self.integers_to_read[0][0], "")
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][2] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ for i in list(range(self.total_triplets_to_read - 1, 0, -1)):
+ reading_triplet_order = self.total_triplets_to_read - i
+ if self.total_digits_outside_triplets == 0:
+ last_read_digit_order = reading_triplet_order * 3
+ else:
+ last_read_digit_order = (reading_triplet_order - 1) * 3 +\
+ self.total_digits_outside_triplets
+
+ if not self.integers_to_read[0][
+ last_read_digit_order: last_read_digit_order + 3
+ ] == "000":
+ if not self.integers_to_read[0][
+ last_read_digit_order
+ ] == "0":
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][last_read_digit_order], ""
+ )
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 1:
+ if i == 1:
+ wrd += self.CARDINAL_HUNDRED[0]
+ return wrd
+ elif i > 1:
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ wrd += self.CARDINAL_HUNDRED[0]
+
+ if not self.integers_to_read[0][
+ last_read_digit_order + 1] == "0":
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 2:
+ if i == 1:
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+ return wrd
+ elif i > 1:
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+
+ if not self.integers_to_read[0][
+ last_read_digit_order + 2] == "0":
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 3:
+ if i == 1:
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ return wrd
+ if i == 2:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2
+ ] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ elif not self.integers_to_read[0][
+ last_read_digit_order + 2] == "1":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+ return wrd
+ if i > 2:
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2
+ ] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ else:
+ if i == 2:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2
+ ] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ elif not self.integers_to_read[0][
+ last_read_digit_order + 2
+ ] == "1":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+
+ return wrd
+
+ def to_cardinal_float(self, value):
+ self.to_splitnum(value)
+ wrd = ""
+ wrd += self.pointword
+ if len(self.integers_to_read[1]) >= 1:
+ wrd += self.CARDINAL_TENS.get(self.integers_to_read[1][0], "")
+
+ if len(self.integers_to_read[1]) == 2:
+ wrd += self.CARDINAL_ONES.get(self.integers_to_read[1][1], "")
+
+ if self.integers_to_read[0] == "0":
+ wrd = self.ZERO + wrd
+ else:
+ wrd = self.to_cardinal(int(self.integers_to_read[0])) + wrd
+ return wrd
+
+ def verify_cardinal(self, value):
+ iscardinal = True
+ try:
+ if not float(value) == value:
+ iscardinal = False
+ except (ValueError, TypeError):
+ raise TypeError(self.errmsg_nonnum)
+ if abs(value) >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig.format(value, self.MAXVAL))
+ return iscardinal
+
+ def verify_ordinal(self, value):
+ isordinal = True
+ try:
+ if not int(value) == value:
+ isordinal = False
+ if not abs(value) == value:
+ raise TypeError(self.errmsg_negord.format(value))
+ except (ValueError, TypeError):
+ raise TypeError(self.errmsg_nonnum)
+ if abs(value) >= self.MAXVAL:
+ raise OverflowError(self.errmsg_toobig.format(value, self.MAXVAL))
+ return isordinal
+
+ def to_ordinal(self, value):
+ wrd = ""
+ isordinal = self.verify_ordinal(value)
+ if isordinal:
+ self.to_splitnum(value)
+
+ if self.order_of_last_zero_digit >= len(self.integers_to_read[0]):
+ # number like 00 and all 0s and even more, raise error
+ return wrd
+
+ if self.total_triplets_to_read == 1:
+ if self.total_digits_outside_triplets == 2:
+ if self.order_of_last_zero_digit == 1:
+ # number like x0, read ordinal x0 and return
+ wrd += self.ORDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ return wrd
+ if self.order_of_last_zero_digit == 0:
+ # number like xy, read ordinal xy and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.ORDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ return wrd
+
+ if self.total_digits_outside_triplets == 1:
+ if self.order_of_last_zero_digit == 0:
+ # number like x, read ordinal x and return
+ wrd += self.ORDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ if self.integers_to_read[0][0] == "0":
+ return u"s谋f谋r谋nc谋"
+ return wrd
+
+ if self.total_digits_outside_triplets == 0:
+ if self.order_of_last_zero_digit == 2:
+ # number like x00, read ordinal x00 and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.ORDINAL_HUNDRED[0]
+ return wrd
+ if self.order_of_last_zero_digit == 1:
+ # number like xy0, read ordinal xy0 and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.ORDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ return wrd
+ if self.order_of_last_zero_digit == 0:
+ # number like xyz, read ordinal xyz and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ if not self.integers_to_read[0][2] == "0":
+ wrd += self.ORDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ return wrd
+
+ if self.total_triplets_to_read >= 2:
+ if self.total_digits_outside_triplets == 2:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x0 and all 0s, read ordinal x0 0..0
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 2:
+ # number like xy and all 0s, read ordinal xy 0..0
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 2:
+ # number like xy and others, read cardinal xy n..n
+ # and return
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ if self.total_digits_outside_triplets == 1:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x and all 0s, read ordinal x 0..0
+ # and return
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][0] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 1:
+ # number like x and others, read cardinal x n..n
+ # and return
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][0] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ if self.total_digits_outside_triplets == 0:
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 1:
+ # number like x00 and all 0s, read ordinal x00 0..0
+ # and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 2:
+ # number like xy0 and all 0s, read ordinal xy0 0..0
+ # and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - 3:
+ # number like xyz and all 0s, read ordinal xyz 0..0
+ # and return
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+ return wrd
+ if self.order_of_last_zero_digit < len(
+ self.integers_to_read[0]) - 3:
+ # number like xyz and all others, read cardinal
+ # xyz n..n
+ wrd += self.HUNDREDS.get(
+ self.integers_to_read[0][0], ""
+ )
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][1], ""
+ )
+ if not (self.total_triplets_to_read == 2 and
+ self.integers_to_read[0][2] == "1"):
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][2], ""
+ )
+ wrd += self.CARDINAL_TRIPLETS[
+ self.total_triplets_to_read - 1
+ ]
+
+ for i in list(range(self.total_triplets_to_read - 1, 0, -1)):
+ reading_triplet_order = self.total_triplets_to_read - i
+ if self.total_digits_outside_triplets == 0:
+ last_read_digit_order = reading_triplet_order * 3
+ else:
+ last_read_digit_order = \
+ (reading_triplet_order - 1) * 3 + \
+ self.total_digits_outside_triplets
+
+ if not self.integers_to_read[0][
+ last_read_digit_order: last_read_digit_order + 3
+ ] == "000":
+ if not self.integers_to_read[0][
+ last_read_digit_order
+ ] == "0":
+ if not self.integers_to_read[0][
+ last_read_digit_order
+ ] == "1":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order
+ ], ""
+ )
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 1:
+ if i == 1:
+ wrd += self.ORDINAL_HUNDRED[0]
+ return wrd
+ elif i > 1:
+ wrd += self.CARDINAL_HUNDRED[0]
+ wrd += self.ORDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ wrd += self.CARDINAL_HUNDRED[0]
+
+ if not self.integers_to_read[0][
+ last_read_digit_order + 1
+ ] == "0":
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 2:
+ if i == 1:
+ wrd += self.ORDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+ return wrd
+ elif i > 1:
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ wrd += self.CARDINAL_TENS.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 1], ""
+ )
+
+ if not self.integers_to_read[0][
+ last_read_digit_order + 2
+ ] == "0":
+ if self.order_of_last_zero_digit == len(
+ self.integers_to_read[0]) - (
+ last_read_digit_order) - 3:
+ if i == 1:
+ wrd += self.ORDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ return wrd
+ if i == 2:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ elif not self.integers_to_read[0][
+ last_read_digit_order + 2
+ ] == "1":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[i - 1]
+ return wrd
+ if i > 2:
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ wrd += self.ORDINAL_TRIPLETS[i - 1]
+ return wrd
+ else:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ else:
+ if not self.integers_to_read[0][
+ last_read_digit_order:
+ last_read_digit_order + 2] == "00":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+ elif not self.integers_to_read[0][
+ last_read_digit_order + 2] == "1":
+ wrd += self.CARDINAL_ONES.get(
+ self.integers_to_read[0][
+ last_read_digit_order + 2], ""
+ )
+
+ wrd += self.CARDINAL_TRIPLETS[i - 1]
+
+ return wrd
+
+ def to_splitnum(self, val):
+ float_digits = str(int(val * 10 ** self.precision))
+ if not int(val) == 0:
+ self.integers_to_read = [
+ str(int(val)),
+ float_digits[len(float_digits) - self.precision:]
+ ]
+ else:
+ self.integers_to_read = [
+ "0",
+ "0" * (self.precision - len(float_digits)) +
+ float_digits[len(float_digits) - self.precision:]
+ ]
+ if len(self.integers_to_read[0]) % 3 > 0:
+ self.total_triplets_to_read = (len(self.integers_to_read[0]) // 3)\
+ + 1
+ elif len(self.integers_to_read[0]) % 3 == 0:
+ self.total_triplets_to_read = len(self.integers_to_read[0]) // 3
+ self.total_digits_outside_triplets = len(self.integers_to_read[0]) % 3
+
+ okunacak = list(self.integers_to_read[0][::-1])
+ self.order_of_last_zero_digit = 0
+ found = 0
+ for i in range(len(okunacak) - 1):
+ if int(okunacak[i]) == 0 and found == 0:
+ self.order_of_last_zero_digit = i + 1
+ else:
+ found = 1
+
+ def to_currency(self, value):
+ if int(value) == 0:
+ return u"bedelsiz"
+ valueparts = self.to_cardinal(value).split(self.pointword)
+ if len(valueparts) == 1:
+ return valueparts[0] + self.CURRENCY_UNIT
+ if len(valueparts) == 2:
+ return self.CURRENCY_UNIT.join(valueparts) + \
+ self.CURRENCY_SUBUNIT
diff --git a/fun_text_processing/num2words/num2words/lang_UK.py b/fun_text_processing/num2words/num2words/lang_UK.py
new file mode 100644
index 0000000..835e728
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_UK.py
@@ -0,0 +1,837 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .base import Num2Word_Base
+from .utils import get_digits, splitbyx
+
+ZERO = ('薪褍谢褜',)
+
+ONES_FEMININE = {
+ 1: ('芯写薪邪',),
+ 2: ('写胁褨',),
+ 3: ('褌褉懈',),
+ 4: ('褔芯褌懈褉懈',),
+ 5: ('锌\'褟褌褜',),
+ 6: ('褕褨褋褌褜',),
+ 7: ('褋褨屑',),
+ 8: ('胁褨褋褨屑',),
+ 9: ('写械胁\'褟褌褜',),
+}
+
+ONES = {
+ 1: ('芯写懈薪',),
+ 2: ('写胁邪',),
+ 3: ('褌褉懈',),
+ 4: ('褔芯褌懈褉懈',),
+ 5: ('锌\'褟褌褜',),
+ 6: ('褕褨褋褌褜',),
+ 7: ('褋褨屑',),
+ 8: ('胁褨褋褨屑',),
+ 9: ('写械胁\'褟褌褜',),
+}
+
+ONES_ORDINALS = {
+ 1: ("锌械褉褕懈泄", "芯写薪芯"),
+ 2: ("写褉褍谐懈泄", "写胁芯褏"),
+ 3: ("褌褉械褌褨泄", "褌褉褜芯褏"),
+ 4: ("褔械褌胁械褉褌懈泄", "褔芯褌懈褉褜芯褏"),
+ 5: ("锌'褟褌懈泄", "锌'褟褌懈"),
+ 6: ("褕芯褋褌懈泄", "褕械褋褌懈"),
+ 7: ("褋褜芯屑懈泄", "褋械屑懈"),
+ 8: ("胁芯褋褜屑懈泄", "胁芯褋褜屑懈"),
+ 9: ("写械胁'褟褌懈泄", "写械胁'褟褌懈"),
+ 10: ("写械褋褟褌懈泄", "写械褋褟褌懈"),
+ 11: ("芯写懈薪邪写褑褟褌懈泄", "芯写懈薪邪写褑褟褌懈"),
+ 12: ("写胁邪薪邪写褑褟褌懈泄", "写胁邪薪邪写褑褟褌懈"),
+ 13: ("褌褉懈薪邪写褑褟褌懈泄", "褌褉懈薪邪写褑褟褌懈"),
+ 14: ("褔芯褌懈褉薪邪写褑褟褌懈泄", "褔芯褌懈褉薪邪写褑褟褌懈"),
+ 15: ("锌'褟褌薪邪写褑褟褌懈泄", "锌'褟褌薪邪写褑褟褌懈"),
+ 16: ("褕褨褋褌薪邪写褑褟褌懈泄", "褕褨褋褌薪邪写褑褟褌懈"),
+ 17: ("褋褨屑薪邪写褑褟褌懈泄", "褋褨屑薪邪写褑褟褌懈"),
+ 18: ("胁褨褋褨屑薪邪写褑褟褌懈泄", "胁褨褋褨屑薪邪写褑褟褌懈"),
+ 19: ("写械胁'褟褌薪邪写褑褟褌懈泄", "写械胁'褟褌薪邪写褑褟褌懈"),
+}
+
+TENS = {
+ 0: ('写械褋褟褌褜',),
+ 1: ('芯写懈薪邪写褑褟褌褜',),
+ 2: ('写胁邪薪邪写褑褟褌褜',),
+ 3: ('褌褉懈薪邪写褑褟褌褜',),
+ 4: ('褔芯褌懈褉薪邪写褑褟褌褜',),
+ 5: ('锌\'褟褌薪邪写褑褟褌褜',),
+ 6: ('褕褨褋褌薪邪写褑褟褌褜',),
+ 7: ('褋褨屑薪邪写褑褟褌褜',),
+ 8: ('胁褨褋褨屑薪邪写褑褟褌褜',),
+ 9: ('写械胁\'褟褌薪邪写褑褟褌褜',),
+}
+
+TWENTIES = {
+ 2: ('写胁邪写褑褟褌褜',),
+ 3: ('褌褉懈写褑褟褌褜',),
+ 4: ('褋芯褉芯泻',),
+ 5: ('锌\'褟褌写械褋褟褌',),
+ 6: ('褕褨褋褌写械褋褟褌',),
+ 7: ('褋褨屑写械褋褟褌',),
+ 8: ('胁褨褋褨屑写械褋褟褌',),
+ 9: ('写械胁\'褟薪芯褋褌芯',),
+}
+
+TWENTIES_ORDINALS = {
+ 2: ("写胁邪写褑褟褌懈泄", "写胁邪写褑褟褌懈"),
+ 3: ("褌褉懈写褑褟褌懈泄", "褌褉懈写褑褟褌懈"),
+ 4: ("褋芯褉芯泻芯胁懈泄", "褋芯褉芯泻邪"),
+ 5: ("锌'褟褌写械褋褟褌懈泄", "锌'褟褌写械褋褟褌懈"),
+ 6: ("褕褨褋褌写械褋褟褌懈泄", "褕褨褋褌写械褋褟褌懈"),
+ 7: ("褋褨屑写械褋褟褌懈泄", "褋褨屑写械褋褟褌懈"),
+ 8: ("胁褨褋褨屑写械褋褟褌懈泄", "胁褨褋褨屑写械褋褟褌懈"),
+ 9: ("写械胁'褟薪芯褋褌懈泄", "写械胁'褟薪芯褋褌懈"),
+}
+
+HUNDREDS = {
+ 1: ('褋褌芯',),
+ 2: ('写胁褨褋褌褨',),
+ 3: ('褌褉懈褋褌邪',),
+ 4: ('褔芯褌懈褉懈褋褌邪',),
+ 5: ('锌\'褟褌褋芯褌',),
+ 6: ('褕褨褋褌褋芯褌',),
+ 7: ('褋褨屑褋芯褌',),
+ 8: ('胁褨褋褨屑褋芯褌',),
+ 9: ('写械胁\'褟褌褋芯褌',),
+}
+
+HUNDREDS_ORDINALS = {
+ 1: ("褋芯褌懈泄", "褋褌芯"),
+ 2: ("写胁芯褏褋芯褌懈泄", "写胁芯褏褋芯褌"),
+ 3: ("褌褉褜芯褏褋芯褌懈泄", "褌褉褜芯褏褋芯褌"),
+ 4: ("褔芯褌懈褉褜芯褏褋芯褌懈泄", "褔芯褌懈褉褜芯褏褋芯褌"),
+ 5: ("锌'褟褌懈褋芯褌懈泄", "锌'褟褌褋芯褌"),
+ 6: ("褕械褋褌懈褋芯褌懈泄", "褕褨褋褌褋芯褌"),
+ 7: ("褋械屑懈褋芯褌懈泄", "褋褨屑褋芯褌"),
+ 8: ("胁芯褋褜屑懈褋芯褌懈泄", "胁褨褋褨屑褋芯褌"),
+ 9: ("写械胁'褟褌懈褋芯褌懈泄", "写械胁'褟褌褋芯褌"),
+}
+
+THOUSANDS = {
+ 1: ('褌懈褋褟褔邪', '褌懈褋褟褔褨', '褌懈褋褟褔'), # 10^3
+ 2: ('屑褨谢褜泄芯薪', '屑褨谢褜泄芯薪懈', '屑褨谢褜泄芯薪褨胁'), # 10^6
+ 3: ('屑褨谢褜褟褉写', '屑褨谢褜褟褉写懈', '屑褨谢褜褟褉写褨胁'), # 10^9
+ 4: ('褌褉懈谢褜泄芯薪', '褌褉懈谢褜泄芯薪懈', '褌褉懈谢褜泄芯薪褨胁'), # 10^12
+ 5: ('泻胁邪写褉懈谢褜泄芯薪', '泻胁邪写褉懈谢褜泄芯薪懈', '泻胁邪写褉懈谢褜泄芯薪褨胁'), # 10^15
+ 6: ('泻胁褨薪褌懈谢褜泄芯薪', '泻胁褨薪褌懈谢褜泄芯薪懈', '泻胁褨薪褌懈谢褜泄芯薪褨胁'), # 10^18
+ 7: ('褋械泻褋褌懈谢褜泄芯薪', '褋械泻褋褌懈谢褜泄芯薪懈', '褋械泻褋褌懈谢褜泄芯薪褨胁'), # 10^21
+ 8: ('褋械锌褌懈谢褜泄芯薪', '褋械锌褌懈谢褜泄芯薪懈', '褋械锌褌懈谢褜泄芯薪褨胁'), # 10^24
+ 9: ('芯泻褌懈谢褜泄芯薪', '芯泻褌懈谢褜泄芯薪懈', '芯泻褌懈谢褜泄芯薪褨胁'), # 10^27
+ 10: ('薪芯薪褨谢褜泄芯薪', '薪芯薪褨谢褜泄芯薪懈', '薪芯薪褨谢褜泄芯薪褨胁'), # 10^30
+}
+
+prefixes_ordinal = {
+ 1: "褌懈褋褟褔薪懈泄",
+ 2: "屑褨谢褜泄芯薪薪懈泄",
+ 3: "屑褨谢褜褟褉写薪懈泄",
+ 4: "褌褉懈谢褜泄芯薪薪懈泄",
+ 5: "泻胁邪写褉懈谢褜泄芯薪薪懈泄",
+ 6: "泻胁褨薪褌懈谢褜泄芯薪薪懈泄",
+ 7: "褋械泻褋褌懈谢褜泄芯薪薪懈泄",
+ 8: "褋械锌褌懈谢褜泄芯薪薪懈泄",
+ 9: "芯泻褌懈谢褜泄芯薪薪懈泄",
+ 10: "薪芯薪褨谢褜泄芯薪薪懈泄",
+}
+
+FEMININE_MONEY = ('AOA', 'BAM', 'BDT', 'BWP', 'CZK', 'DKK',
+ 'ERN', 'HNL', 'HRK', 'IDR', 'INR', 'ISK',
+ 'JPY', 'KPW', 'KRW', 'LKR', 'MOP', 'MRU',
+ 'MUR', 'MVR', 'MWK', 'NGN', 'NIO', 'NOK',
+ 'NPR', 'PKR', 'SCR', 'SEK', 'STN', 'TRY',
+ 'WST', 'UAH', 'ZMW')
+FEMININE_CENTS = ('ALL', 'BDT', 'BGN', 'BYN', 'GHS', 'HRK',
+ 'ILS', 'INR', 'NPR', 'OMR', 'OMR', 'PKR',
+ 'RSD', 'RUB', 'UAH')
+
+GENERIC_DOLLARS = ('写芯谢邪褉', '写芯谢邪褉懈', '写芯谢邪褉褨胁')
+GENERIC_CENTS = ('褑械薪褌', '褑械薪褌懈', '褑械薪褌褨胁')
+
+
+class Num2Word_UK(Num2Word_Base):
+ CURRENCY_FORMS = {
+ 'AED': (
+ ('写懈褉褏邪屑', '写懈褉褏邪屑懈', '写懈褉褏邪屑褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'AFN': (
+ ('邪褎谐邪薪褨', '邪褎谐邪薪褨', '邪褎谐邪薪褨'),
+ ('锌褍谢', '锌褍谢懈', '锌褍谢褨胁')
+ ),
+ 'ALL': (
+ ('谢械泻', '谢械泻懈', '谢械泻褨胁'),
+ ('泻褨薪写邪褉泻邪', '泻褨薪写邪褉泻懈', '泻褨薪写邪褉芯泻')
+ ),
+ 'AMD': (
+ ('写褉邪屑', '写褉邪屑懈', '写褉邪屑褨胁'),
+ ('谢褍屑', '谢褍屑', '谢褍屑')
+ ),
+ 'ANG': (
+ ('谐褍谢褜写械薪', '谐褍谢褜写械薪懈', '谐褍谢褜写械薪褨胁'),
+ GENERIC_CENTS
+ ),
+ 'AOA': (
+ ('泻胁邪薪蟹邪', '泻胁邪薪蟹懈', '泻胁邪薪蟹'),
+ ('褋械薪褌懈屑芯', '褋械薪褌懈屑芯', '褋械薪褌懈屑芯')
+ ),
+ 'ARS': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'AUD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'AWG': (
+ ('褎谢芯褉懈薪', '褎谢芯褉懈薪懈', '褎谢芯褉懈薪褨胁'),
+ GENERIC_CENTS
+ ),
+ 'AZN': (
+ ('屑邪薪邪褌', '屑邪薪邪褌懈', '屑邪薪邪褌褨胁'),
+ ('谐褟锌褨泻', '谐褟锌褨泻懈', '谐褟锌褨泻褨胁')
+ ),
+ 'BAM': (
+ ('屑邪褉泻邪', '屑邪褉泻懈', '屑邪褉芯泻'),
+ ('褎械薪褨薪谐', '褎械薪褨薪谐懈', '褎械薪褨薪谐褨胁')
+ ),
+ 'BBD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BDT': (
+ ('褌邪泻邪', '褌邪泻懈', '褌邪泻'),
+ ('锌芯泄褕邪', '锌芯泄褕褨', '锌芯泄褕')
+ ),
+ 'BGN': (
+ ('谢械胁', '谢械胁懈', '谢械胁褨胁'),
+ ('褋褌芯褌懈薪泻邪', '褋褌芯褌懈薪泻懈', '褋褌芯褌懈薪芯泻')
+ ),
+ 'BHD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'BIF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'BMD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BND': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BOB': (
+ ('斜芯谢褨胁褨邪薪芯', '斜芯谢褨胁褨邪薪芯', '斜芯谢褨胁褨邪薪芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'BRL': (
+ ('褉械邪谢', '褉械邪谢懈', '褉械邪谢褨胁'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'BSD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'BTN': (
+ ('薪谐褍谢褌褉褍屑', '薪谐褍谢褌褉褍屑懈', '薪谐褍谢褌褉褍屑褨胁'),
+ ('褔械褌褉褍屑', '褔械褌褉褍屑懈', '褔械褌褉褍屑褨胁')
+ ),
+ 'BWP': (
+ ('锌褍谢邪', '锌褍谢懈', '锌褍谢'),
+ ('褌褏械斜械', '褌褏械斜械', '褌褏械斜械')
+ ),
+ 'BYN': (
+ ('褉褍斜械谢褜', '褉褍斜谢褨', '褉褍斜谢褨胁'),
+ ('泻芯锌褨泄泻邪', '泻芯锌褨泄泻懈', '泻芯锌褨泄芯泻')
+ ),
+ 'BZD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'CAD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'CDF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'CHF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'CLP': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'CNY': (
+ ('褞邪薪褜', '褞邪薪褨', '褞邪薪褨胁'),
+ ('褎懈薪褜', '褎懈薪褨', '褎懈薪褨胁')
+ ),
+ 'COP': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'CRC': (
+ ('泻芯谢芯薪', '泻芯谢芯薪懈', '泻芯谢芯薪褨胁'),
+ ('褋械薪褌褨屑芯', '褋械薪褌褨屑芯', '褋械薪褌褨屑芯')
+ ),
+ 'CUC': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'CUP': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'CVE': (
+ ('械褋泻褍写芯', '械褋泻褍写芯', '械褋泻褍写芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'CZK': (
+ ('泻褉芯薪邪', '泻褉芯薪懈', '泻褉芯薪'),
+ ('谐械谢械褉', '谐械谢械褉懈', '谐械谢械褉褨胁')
+ ),
+ 'DJF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'DKK': (
+ ('泻褉芯薪邪', '泻褉芯薪懈', '泻褉芯薪'),
+ ('械褉械', '械褉械', '械褉械')
+ ),
+ 'DOP': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'DZD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'EGP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌褨邪褋褌褉', '锌褨邪褋褌褉懈', '锌褨邪褋褌褉褨胁')
+ ),
+ 'ERN': (
+ ('薪邪泻褎邪', '薪邪泻褎懈', '薪邪泻褎'),
+ GENERIC_CENTS
+ ),
+ 'ETB': (
+ ('斜懈褉', '斜懈褉懈', '斜懈褉褨胁'),
+ GENERIC_CENTS
+ ),
+ 'EUR': (
+ ('褦胁褉芯', '褦胁褉芯', '褦胁褉芯'),
+ GENERIC_CENTS
+ ),
+ 'FJD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'FKP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌械薪褋', '锌械薪褋懈', '锌械薪褋褨胁')
+ ),
+ 'GBP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌械薪褋', '锌械薪褋懈', '锌械薪褋褨胁')
+ ),
+ 'GEL': (
+ ('谢邪褉褨', '谢邪褉褨', '谢邪褉褨'),
+ ('褌械褌褉褨', '褌械褌褉褨', '褌械褌褉褨')
+ ),
+ 'GHS': (
+ ('褋械写褨', '褋械写褨', '褋械写褨'),
+ ('锌械褋械胁邪', '锌械褋械胁懈', '锌械褋械胁')
+ ),
+ 'GIP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌械薪褋', '锌械薪褋懈', '锌械薪褋褨胁')
+ ),
+ 'GMD': (
+ ('写邪谢邪褋褨', '写邪谢邪褋褨', '写邪谢邪褋褨'),
+ ('斜褍褌褍褌', '斜褍褌褍褌懈', '斜褍褌褍褌褨胁')
+ ),
+ 'GNF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'GTQ': (
+ ('泻械褌褋邪谢褜', '泻械褌褋邪谢褨', '泻械褌褋邪谢褨胁'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'GYD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'HKD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'HNL': (
+ ('谢械屑锌褨褉邪', '谢械屑锌褨褉懈', '谢械屑锌褨褉'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'HRK': (
+ ('泻褍薪邪', '泻褍薪懈', '泻褍薪'),
+ ('谢褨锌邪', '谢褨锌懈', '谢褨锌')
+ ),
+ 'HTG': (
+ ('谐褍褉写', '谐褍褉写懈', '谐褍褉写褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'HUF': (
+ ('褎芯褉懈薪褌', '褎芯褉懈薪褌懈', '褎芯褉懈薪褌褨胁'),
+ ('褎褨谢谢械褉', '褎褨谢谢械褉懈', '褎褨谢谢械褉褨胁')
+ ),
+ 'IDR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ GENERIC_CENTS
+ ),
+ 'ILS': (
+ ('褕械泻械谢褜', '褕械泻械谢褨', '褕械泻械谢褨胁'),
+ ('邪谐芯褉邪', '邪谐芯褉懈', '邪谐芯褉')
+ ),
+ 'INR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ ('锌邪泄褋邪', '锌邪泄褋懈', '锌邪泄褋')
+ ),
+ 'IQD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'IRR': (
+ ('褉褨邪谢', '褉褨邪谢懈', '褉褨邪谢褨胁'),
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁')
+ ),
+ 'ISK': (
+ ('泻褉芯薪邪', '泻褉芯薪懈', '泻褉芯薪'),
+ ('械泄褉械', '械泄褉械', '械泄褉械')
+ ),
+ 'JMD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'JOD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'JPY': (
+ ('褦薪邪', '褦薪懈', '褦薪'),
+ ('褋械薪', '褋械薪', '褋械薪')
+ ),
+ 'KES': (
+ ('褕懈谢褨薪谐', '褕懈谢褨薪谐懈', '褕懈谢褨薪谐褨胁'),
+ GENERIC_CENTS
+ ),
+ 'KGS': (
+ ('褋芯屑', '褋芯屑懈', '褋芯屑褨胁'),
+ ('褌懈褩薪', '褌懈褩薪懈', '褌懈褩薪褨胁')
+ ),
+ 'KHR': (
+ ('褉褨褦谢褜', '褉褨褦谢褨', '褉褨褦谢褨胁'),
+ ('褋褍', '褋褍', '褋褍')
+ ),
+ 'KMF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'KPW': (
+ ('胁芯薪邪', '胁芯薪懈', '胁芯薪'),
+ ('褔芯薪', '褔芯薪懈', '褔芯薪褨胁')
+ ),
+ 'KRW': (
+ ('胁芯薪邪', '胁芯薪懈', '胁芯薪'),
+ ('写卸械芯薪', '写卸械芯薪懈', '写卸械芯薪褨胁')
+ ),
+ 'KWD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'KYD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'KZT': (
+ ('褌械薪褜谐械', '褌械薪褜谐械', '褌械薪褜谐械'),
+ ('褌懈褩薪', '褌懈褩薪懈', '褌懈褩薪褨胁')),
+ 'LAK': (
+ ('泻褨锌', '泻褨锌懈', '泻褨锌褨胁'),
+ ('邪褌', '邪褌懈', '邪褌褨胁')
+ ),
+ 'LBP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌褨邪褋褌褉', '锌褨邪褋褌褉懈', '锌褨邪褋褌褉褨胁')
+ ),
+ 'LKR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ GENERIC_CENTS
+ ),
+ 'LRD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'LSL': (
+ ('谢芯褌褨', '屑邪谢芯褌褨', '屑邪谢芯褌褨'),
+ ('褋械薪褌械', '谢褨褋械薪褌械', '谢褨褋械薪褌械')
+ ),
+ 'LYD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('写懈褉褏邪屑', '写懈褉褏邪屑懈', '写懈褉褏邪屑褨胁')
+ ),
+ 'MAD': (
+ ('写懈褉褏邪屑', '写懈褉褏邪屑懈', '写懈褉褏邪屑褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'MDL': (
+ ('谢械泄', '谢械褩', '谢械褩'),
+ ('斜邪薪', '斜邪薪褨', '斜邪薪褨')
+ ),
+ 'MGA': (
+ ('邪褉褨邪褉褨', '邪褉褨邪褉褨', '邪褉褨邪褉褨'),
+ ('褨褉邪泄屑斜褨谢邪薪褜褟', '褨褉邪泄屑斜褨谢邪薪褜褟', '褨褉邪泄屑斜褨谢邪薪褜褟')
+ ),
+ 'MKD': (
+ ('写械薪邪褉', '写械薪邪褉懈', '写械薪邪褉褨胁'),
+ ('写械薪褨', '写械薪褨', '写械薪褨')
+ ),
+ 'MMK': (
+ ('泻\'褟褌', '泻\'褟褌', '泻\'褟褌'),
+ ('锌\'褟', '锌\'褟', '锌\'褟')
+ ),
+ 'MNT': (
+ ('褌褍谐褉懈泻', '褌褍谐褉懈泻懈', '褌褍谐褉懈泻褨胁'),
+ ('屑褍薪谐褍', '屑褍薪谐褍', '屑褍薪谐褍')
+ ),
+ 'MOP': (
+ ('锌邪褌邪泻邪', '锌邪褌邪泻褨', '锌邪褌邪泻'),
+ ('邪胁芯', '邪胁芯', '邪胁芯')
+ ),
+ 'MRU': (
+ ('褍谐褨褟', '褍谐褨褩', '褍谐褨泄'),
+ ('褏褍屑褋', '褏褍屑褋', '褏褍屑褋')
+ ),
+ 'MUR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ GENERIC_CENTS
+ ),
+ 'MVR': (
+ ('褉褍褎褨褟', '褉褍褎褨褩', '褉褍褎褨泄'),
+ ('谢邪褉褨', '谢邪褉褨', '谢邪褉褨')
+ ),
+ 'MWK': (
+ ('泻胁邪褔邪', '泻胁邪褔褨', '泻胁邪褔'),
+ ('褌邪屑斜邪谢邪', '褌邪屑斜邪谢邪', '褌邪屑斜邪谢邪')
+ ),
+ 'MXN': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'MYR': (
+ ('褉懈薪谐褨褌', '褉懈薪谐褨褌懈', '褉懈薪谐褨褌褨胁'),
+ GENERIC_CENTS
+ ),
+ 'MZN': (
+ ('屑械褌褨泻邪谢', '屑械褌褨泻邪谢懈', '屑械褌褨泻邪谢褨胁'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'NAD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'NGN': (
+ ('薪邪泄褉邪', '薪邪泄褉懈', '薪邪泄褉'),
+ ('泻芯斜芯', '泻芯斜芯', '泻芯斜芯')
+ ),
+ 'NIO': (
+ ('泻芯褉写芯斜邪', '泻芯褉写芯斜懈', '泻芯褉写芯斜'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'NOK': (
+ ('泻褉芯薪邪', '泻褉芯薪懈', '泻褉芯薪'),
+ ('械褉械', '械褉械', '械褉械')
+ ),
+ 'NPR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ ('锌邪泄褋邪', '锌邪泄褋懈', '锌邪泄褋')
+ ),
+ 'NZD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'OMR': (
+ ('褉褨邪谢', '褉褨邪谢懈', '褉褨邪谢褨胁'),
+ ('斜邪泄蟹邪', '斜邪泄蟹懈', '斜邪泄蟹')
+ ),
+ 'PAB': (
+ ('斜邪谢褜斜芯邪', '斜邪谢褜斜芯邪', '斜邪谢褜斜芯邪'),
+ ('褋械薪褌械褋褨屑芯', '褋械薪褌械褋褨屑芯', '褋械薪褌械褋褨屑芯')
+ ),
+ 'PEN': (
+ ('褋芯谢褜', '褋芯谢褜', '褋芯谢褜'),
+ ('褋械薪褌褨屑芯', '褋械薪褌褨屑芯', '褋械薪褌褨屑芯')
+ ),
+ 'PGK': (
+ ('泻褨薪邪', '泻褨薪邪', '泻褨薪邪'),
+ ('褌芯泄褟', '褌芯泄褟', '褌芯泄褟')
+ ),
+ 'PHP': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌邪胁芯', '褋械薪褌邪胁芯', '褋械薪褌邪胁芯')
+ ),
+ 'PKR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ ('锌邪泄褋邪', '锌邪泄褋懈', '锌邪泄褋')
+ ),
+ 'PLN': (
+ ('蟹谢芯褌懈泄', '蟹谢芯褌褨', '蟹谢芯褌懈褏'),
+ ('谐褉芯褕', '谐褉芯褕褨', '谐褉芯褕褨胁')
+ ),
+ 'PYG': (
+ ('谐褍邪褉邪薪褨', '谐褍邪褉邪薪褨', '谐褍邪褉邪薪褨'),
+ ('褋械薪褌褨屑芯', '褋械薪褌褨屑芯', '褋械薪褌褨屑芯')
+ ),
+ 'QAR': (
+ ('褉褨邪谢', '褉褨邪谢懈', '褉褨邪谢褨胁'),
+ ('写懈褉褏邪屑', '写懈褉褏邪屑懈', '写懈褉褏邪屑褨胁')
+ ),
+ 'RON': (
+ ('谢械泄', '谢械褩', '谢械褩'),
+ ('斜邪薪', '斜邪薪褨', '斜邪薪褨')
+ ),
+ 'RSD': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('锌邪褉邪', '锌邪褉懈', '锌邪褉')
+ ),
+ 'RUB': (
+ ('褉褍斜谢褜', '褉褍斜谢褨', '褉褍斜谢褨胁'),
+ ('泻芯锌褨泄泻邪', '泻芯锌褨泄泻懈', '泻芯锌褨泄芯泻')
+ ),
+ 'RWF': (
+ ('褎褉邪薪泻', '褎褉邪薪泻懈', '褎褉邪薪泻褨胁'),
+ ('褋邪薪褌懈屑', '褋邪薪褌懈屑懈', '褋邪薪褌懈屑褨胁')
+ ),
+ 'SAR': (
+ ('褉褨邪谢', '褉褨邪谢懈', '褉褨邪谢褨胁'),
+ ('褏邪谢邪谢', '褏邪谢邪谢懈', '褏邪谢邪谢褨胁')
+ ),
+ 'SBD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'SCR': (
+ ('褉褍锌褨褟', '褉褍锌褨褩', '褉褍锌褨泄'),
+ GENERIC_CENTS
+ ),
+ 'SDG': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌褨邪褋褌褉', '锌褨邪褋褌褉懈', '锌褨邪褋褌褉褨胁')
+ ),
+ 'SEK': (
+ ('泻褉芯薪邪', '泻褉芯薪懈', '泻褉芯薪'),
+ ('械褉械', '械褉械', '械褉械')
+ ),
+ 'SGD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'SHP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌械薪褋', '锌械薪褋懈', '锌械薪褋褨胁')
+ ),
+ 'SLL': (
+ ('谢械芯薪械', '谢械芯薪械', '谢械芯薪械'),
+ GENERIC_CENTS
+ ),
+ 'SOS': (
+ ('褕懈谢褨薪谐', '褕懈谢褨薪谐懈', '褕懈谢褨薪谐褨胁'),
+ GENERIC_CENTS
+ ),
+ 'SRD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'SSP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌褨邪褋褌褉', '锌褨邪褋褌褉懈', '锌褨邪褋褌褉褨胁')
+ ),
+ 'STN': (
+ ('写芯斜褉邪', '写芯斜褉懈', '写芯斜褉'),
+ ('褋械薪褌褨屑芯', '褋械薪褌褨屑芯', '褋械薪褌褨屑芯')
+ ),
+ 'SYP': (
+ ('褎褍薪褌', '褎褍薪褌懈', '褎褍薪褌褨胁'),
+ ('锌褨邪褋褌褉', '锌褨邪褋褌褉懈', '锌褨邪褋褌褉褨胁')
+ ),
+ 'SZL': (
+ ('谢褨谢邪薪覒械薪褨', '谢褨谢邪薪覒械薪褨', '谢褨谢邪薪覒械薪褨'),
+ GENERIC_CENTS
+ ),
+ 'THB': (
+ ('斜邪褌', '斜邪褌懈', '斜邪褌褨胁'),
+ ('褋邪褌邪薪谐', '褋邪褌邪薪谐懈', '褋邪褌邪薪谐褨胁')
+ ),
+ 'TJS': (
+ ('褋芯屑芯薪褨', '褋芯屑芯薪褨', '褋芯屑芯薪褨'),
+ ('写褨褉邪屑', '写褨褉邪屑懈', '写褨褉邪屑褨胁')
+ ),
+ 'TMT': (
+ ('屑邪薪邪褌', '屑邪薪邪褌懈', '屑邪薪邪褌褨胁'),
+ ('褌械薪谐械', '褌械薪谐械', '褌械薪谐械')
+ ),
+ 'TND': (
+ ('写懈薪邪褉', '写懈薪邪褉懈', '写懈薪邪褉褨胁'),
+ ('屑褨谢谢褨屑', '屑褨谢谢褨屑懈', '屑褨谢谢褨屑褨胁')
+ ),
+ 'TOP': (
+ ('锌邪邪薪谐邪', '锌邪邪薪谐邪', '锌邪邪薪谐邪'),
+ ('褋械薪褨褌褨', '褋械薪褨褌褨', '褋械薪褨褌褨')
+ ),
+ 'TRY': (
+ ('谢褨褉邪', '谢褨褉懈', '谢褨褉'),
+ ('泻褍褉褍褕', '泻褍褉褍褕褨', '泻褍褉褍褕褨胁')
+ ),
+ 'TTD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'TWD': (
+ ('薪芯胁懈泄 写芯谢邪褉', '薪芯胁褨 写芯谢邪褉懈', '薪芯胁懈褏 写芯谢邪褉褨胁'),
+ GENERIC_CENTS
+ ),
+ 'TZS': (
+ ('褕懈谢褨薪谐', '褕懈谢褨薪谐懈', '褕懈谢褨薪谐褨胁'),
+ GENERIC_CENTS
+ ),
+ 'UAH': (
+ ('谐褉懈胁薪褟', '谐褉懈胁薪褨', '谐褉懈胁械薪褜'),
+ ('泻芯锌褨泄泻邪', '泻芯锌褨泄泻懈', '泻芯锌褨泄芯泻')
+ ),
+ 'UGX': (
+ ('褕懈谢褨薪谐', '褕懈谢褨薪谐懈', '褕懈谢褨薪谐褨胁'),
+ GENERIC_CENTS
+ ),
+ 'USD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'UYU': (
+ ('锌械褋芯', '锌械褋芯', '锌械褋芯'),
+ ('褋械薪褌械褋褨屑芯', '褋械薪褌械褋褨屑芯', '褋械薪褌械褋褨屑芯')
+ ),
+ 'UZS': (
+ ('褋褍屑', '褋褍屑懈', '褋褍屑褨胁'),
+ ('褌懈褩薪', '褌懈褩薪懈', '褌懈褩薪褨胁')
+ ),
+ 'VND': (
+ ('写芯薪谐', '写芯薪谐懈', '写芯薪谐褨胁'),
+ ('褋褍', '褋褍', '褋褍')
+ ),
+ 'WST': (
+ ('褌邪谢邪', '褌邪谢懈', '褌邪谢'),
+ ('褋械薪械', '褋械薪械', '褋械薪械')
+ ),
+ 'XCD': (GENERIC_DOLLARS, GENERIC_CENTS),
+ 'YER': (
+ ('褉褨邪谢', '褉褨邪谢懈', '褉褨邪谢褨胁'),
+ ('褎褨谢褋', '褎褨谢褋懈', '褎褨谢褋褨胁')
+ ),
+ 'ZAR': (
+ ('褉邪薪写', '褉邪薪写懈', '褉邪薪写褨胁'),
+ GENERIC_CENTS
+ ),
+ 'ZMW': (
+ ('泻胁邪褔邪', '泻胁邪褔褨', '泻胁邪褔'),
+ ('薪谐胁械', '薪谐胁械', '薪谐胁械')
+ ),
+ }
+
+ def setup(self):
+ self.negword = "屑褨薪褍褋"
+ self.pointword = "泻芯屑邪"
+
+ def to_cardinal(self, number):
+ n = str(number).replace(',', '.')
+ if '.' in n:
+ left, right = n.split('.')
+ leading_zero_count = len(right) - len(right.lstrip('0'))
+ decimal_part = ((ZERO[0] + ' ') * leading_zero_count +
+ self._int2word(int(right)))
+ return u'%s %s %s' % (
+ self._int2word(int(left)),
+ self.pointword,
+ decimal_part
+ )
+ else:
+ return self._int2word(int(n))
+
+ def pluralize(self, n, forms):
+ if n % 100 < 10 or n % 100 > 20:
+ if n % 10 == 1:
+ form = 0
+ elif 5 > n % 10 > 1:
+ form = 1
+ else:
+ form = 2
+ else:
+ form = 2
+
+ return forms[form]
+
+ def _int2word(self, n, feminine=False):
+ if n < 0:
+ return ' '.join([self.negword, self._int2word(abs(n))])
+
+ if n == 0:
+ return ZERO[0]
+
+ words = []
+ chunks = list(splitbyx(str(n), 3))
+ i = len(chunks)
+ for x in chunks:
+ i -= 1
+
+ if x == 0:
+ continue
+
+ n1, n2, n3 = get_digits(x)
+
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+
+ if n2 > 1:
+ words.append(TWENTIES[n2][0])
+
+ if n2 == 1:
+ words.append(TENS[n1][0])
+ # elif n1 > 0 and not (i > 0 and x == 1):
+ elif n1 > 0:
+ ones = ONES_FEMININE if i == 1 or feminine and i == 0 else ONES
+ words.append(ones[n1][0])
+
+ if i > 0:
+ words.append(self.pluralize(x, THOUSANDS[i]))
+
+ return ' '.join(words)
+
+ def _money_verbose(self, number, currency):
+ return self._int2word(number, currency in FEMININE_MONEY)
+
+ def _cents_verbose(self, number, currency):
+ return self._int2word(number, currency in FEMININE_CENTS)
+
+ @staticmethod
+ def last_fragment_to_ordinal(last, words, level):
+ n1, n2, n3 = get_digits(last)
+ last_two = n2*10+n1
+ if last_two == 0:
+ words.append(HUNDREDS_ORDINALS[n3][level])
+ elif level == 1 and last == 1:
+ return
+ elif last_two < 20:
+ if level == 0:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+ words.append(ONES_ORDINALS[last_two][0])
+ else:
+ last_fragment_string = ''
+ if n3 > 0:
+ last_fragment_string += HUNDREDS_ORDINALS[n3][1]
+ last_fragment_string += ONES_ORDINALS[last_two][1]
+ words.append(last_fragment_string)
+ elif last_two % 10 == 0:
+ if level == 0:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+ words.append(TWENTIES_ORDINALS[n2][0])
+ else:
+ last_fragment_string = ''
+ if n3 > 0:
+ last_fragment_string += HUNDREDS_ORDINALS[n3][1]
+ last_fragment_string += TWENTIES_ORDINALS[n2][1]
+ words.append(last_fragment_string)
+ else:
+ if level == 0:
+ if n3 > 0:
+ words.append(HUNDREDS[n3][0])
+ words.append(TWENTIES[n2][0])
+ words.append(ONES_ORDINALS[n1][0])
+ else:
+ last_fragment_string = ''
+ if n3 > 0:
+ last_fragment_string += HUNDREDS_ORDINALS[n3][1]
+ last_fragment_string += TWENTIES_ORDINALS[n2][1]
+ last_fragment_string += ONES_ORDINALS[n1][1]
+ words.append(last_fragment_string)
+
+ def to_ordinal(self, number):
+ self.verify_ordinal(number)
+
+ words = []
+ fragments = list(splitbyx(str(number), 3))
+ level = 0
+ last = fragments[-1]
+ while last == 0:
+ level = level + 1
+ fragments.pop()
+ last = fragments[-1]
+ if len(fragments) > 1:
+ pre_part = self._int2word(number - (last * 1000 ** level))
+ words.append(pre_part)
+ Num2Word_UK.last_fragment_to_ordinal(
+ last,
+ words,
+ 0 if level == 0 else 1
+ )
+ output = " ".join(words)
+ if last == 1 and level > 0 and output != "":
+ output = output + " "
+ if level > 0:
+ output = output + prefixes_ordinal[level]
+ return output
diff --git a/fun_text_processing/num2words/num2words/lang_UR.py b/fun_text_processing/num2words/num2words/lang_UR.py
new file mode 100644
index 0000000..dab8dbf
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_UR.py
@@ -0,0 +1,232 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from . import lang_EU
+
+
+class Num2Word_UR(lang_EU.Num2Word_EU):
+ # def set_high_numwords(self, high):
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # self.cards[10 ** n] = word
+ def reverse_text(text):
+ return ''.join(reversed(text))
+
+ def set_high_numwords(self, high):
+ max = 3 * len(high)
+ for word, n in zip(high, range(max, 3, -3)):
+ # print(word[0],word[1],n)
+ self.cards[10 ** n] = word[1]
+ # self.cards[10**n] = word + "釣涐灦釣�"
+ # try:
+ # ordinal_word = self.high_numwords[high]
+ # except KeyError:
+ # max = 3 + 3 * len(high)
+ # for word, n in zip(high, range(max, 3, -3)):
+ # print(word)
+ # print(n)
+ # self.cards[10 ** n] = word
+
+ def setup(self):
+ super(Num2Word_UR, self).setup()
+
+ self.negword = '賲丕卅賳爻'
+ self.pointword = '倬賵丕卅賳俟'
+
+ self.high_numwords = [(1000000000000,'俟乇蹖賱蹖賳'), (1000000000, '丕蹖讴 丕乇亘'), (1000000, '丿爻 賱丕讴诰') ]
+
+ self.mid_numwords = [(1000, '蹃夭丕乇'), (100,'爻賵'),(90, '賳賵蹝'), (80, '丕爻蹖'), (70, '爻鬲乇'), (60, '爻丕俟诰'), (50, '倬趩丕爻'), (40, '趩丕賱蹖爻'), (30, '鬲蹖爻')]
+
+ self.low_numwords = ["亘蹖爻",
+ "丕賳蹖爻",
+ "丕俟诰丕乇蹃",
+ "爻鬲乇蹃",
+ "爻賵賱蹃",
+ "倬賳丿乇蹃",
+ "趩賵丿蹃",
+ "鬲蹖乇蹃",
+ "亘丕乇蹃",
+ "诏蹖丕乇蹃",
+ "丿爻",
+ "賳賵",
+ "丌俟诰",
+ "爻丕鬲",
+ "趩诰",
+ "倬丕賳趩",
+ "趩丕乇",
+ "鬲蹖賳",
+ "丿賵",
+ "丕蹖讴",
+ "氐賮乇"]
+
+ self.ords = {'丕蹖讴':'倬蹃賱丕',
+ '丿賵':'丿賵爻乇丕',
+ '鬲蹖賳':'鬲蹖爻乇蹝',
+ '趩丕乇':'趩賵鬲诰丕',
+ '倬丕賳趩':'倬丕賳趩賵蹖诤',
+ '趩诰':'趩诰俟丕',
+ '爻丕鬲':'爻丕鬲賵蹖诤',
+ '丌俟诰':'丌俟诰賵蹖诤',
+ '賳賵':'賳賵蹖诤',
+ '丿爻':'丿爻賵蹖诤'}
+
+ self.labeled_numbers = {'0':'氐賮乇',
+ '1':'丕蹖讴',
+ '2':'丿賵',
+ '3':'鬲蹖賳',
+ '4':'趩丕乇',
+ '5':'倬丕賳趩',
+ '6':'趩诰',
+ '7':'爻丕鬲',
+ '8':'丌俟诰',
+ '9':'賳賵',
+ '10':'丿爻',
+ '11':'诏蹖丕乇蹃',
+ '12':'亘丕乇蹃',
+ '13':'鬲蹖乇蹃',
+ '14':'趩賵丿蹃',
+ '15':'倬賳丿乇蹃',
+ '16':'爻賵賱蹃',
+ '17':'爻鬲乇蹃',
+ '18':'丕俟诰丕乇蹃',
+ '19':'丕賳蹖爻',
+ '20':'亘蹖爻',
+ '21':'丕讴蹖爻',
+ '22':'亘丕卅蹖爻',
+ '23':'鬲卅蹖爻',
+ '24':'趩賵亘蹖爻',
+ '25':'倬趩蹖爻',
+ '26':'趩诰亘亘蹖爻',
+ '27':'爻鬲丕卅蹖爻',
+ '28':'丕俟诰丕卅蹖爻',
+ '29':'丕賳鬲蹖爻',
+ '30':'鬲蹖爻',
+ '31':'丕讴鬲蹖爻',
+ '32':'亘鬲蹖爻',
+ '33':'鬲蹖賳鬲蹖爻',
+ '34':'趩賵賳鬲蹖爻',
+ '35':'倬蹖賳鬲蹖爻',
+ '36':'趩诰鬲蹖爻',
+ '37':'爻蹖賳鬲蹖爻',
+ '38':'丕趹鬲蹖爻',
+ '39':'丕賳鬲丕賱蹖爻',
+ '40':'趩丕賱蹖爻',
+ '41':'丕讴鬲丕賱蹖爻',
+ '42':'亘蹖丕賱蹖爻',
+ '43':'鬲蹖賳鬲丕賱蹖爻',
+ '44':'趩賵丕賱蹖爻',
+ '45':'倬蹖賳鬲丕賱蹖爻',
+ '46':'趩诰蹖丕賱蹖爻',
+ '47':'爻蹖賳鬲丕賱蹖爻',
+ '48':'丕趹鬲丕賱蹖爻',
+ '49':'丕賳趩丕爻',
+ '50':'倬趩丕爻',
+ '51':'丕讴蹖丕賵賳',
+ '52':'亘丕賵賳',
+ '53':'鬲乇倬賳',
+ '54':'趩賵賳',
+ '55':'倬趩倬賳',
+ '56':'趩诰倬賳',
+ '57':'爻鬲丕賵賳',
+ '58':'丕俟诰丕賵賳',
+ '59':'丕賳爻俟诰',
+ '60':'爻丕俟诰',
+ '61':'丕讴爻俟诰',
+ '62':'亘丕爻俟诰',
+ '63':'鬲乇爻俟诰',
+ '64':'趩賵爻俟诰',
+ '65':'倬蹖賳爻俟诰',
+ '66':'趩诰蹖丕爻俟诰',
+ '67':'爻趹爻俟诰',
+ '68':'丕俟诰爻俟诰',
+ '69':'丕賳蹃鬲乇',
+ '70':'爻鬲乇',
+ '71':'丕讴蹃鬲乇',
+ '72':'亘蹃鬲乇',
+ '73':'鬲蹃鬲乇',
+ '74':'趩賵蹃鬲乇',
+ '75':'倬趩蹃鬲乇',
+ '76':'趩诰蹃鬲乇',
+ '77':'爻鬲鬲乇',
+ '78':'丕俟诰蹃鬲乇',
+ '79':'丕賳丕爻蹖',
+ '80':'丕爻蹖',
+ '81':'丕讴蹖丕爻蹖',
+ '82':'亘蹖丕爻蹖',
+ '83':'鬲乇丕爻蹖',
+ '84':'趩賵乇丕爻蹖',
+ '85':'倬趩丕爻蹖',
+ '86':'趩诰蹖丕爻蹖',
+ '87':'爻鬲丕爻蹖',
+ '88':'丕俟诰丕爻蹖',
+ '89':'賳賵丕爻蹖',
+ '90':'賳賵蹝',
+ '91':'丕讴丕賳賵蹝',
+ '92':'亘丕賳賵蹝',
+ '93':'鬲乇丕賳賵蹝',
+ '94':'趩賵乇丕賳賵蹝',
+ '95':'倬趩丕賳賵蹝',
+ '96':'趩诰蹖丕賳賵蹝',
+ '97':'爻鬲丕賳賵蹝',
+ '98':'丕俟诰丕賳賵蹝',
+ '99':'賳賳丕賳賵蹝',
+ '100':'爻賵',
+ '200':'丿賵 爻賵',
+ '1000':'丕蹖讴 蹃夭丕乇',
+ '2000':'丿賵 蹃夭丕乇',
+ '100000':'丕蹖讴 賱丕讴诰',
+ '1000000':'丿爻 賱丕讴诰',
+ '2000000':'亘蹖爻 賱丕讴诰',
+ '10000000':'丕蹖讴 讴乇賵趹',
+ '100000000':'丿爻 讴乇賵趹',
+ '1000000000':'丕蹖讴 丕乇亘',
+ '10000000000':'丿爻 丕乇亘'}
+
+ def merge(self, lpair, rpair):
+ ltext, lnum = lpair
+ rtext, rnum = rpair
+ if lnum == 1 and rnum < 100:
+ return (rtext, rnum)
+ elif 100 > lnum > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif lnum >= 100 > rnum:
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+ elif rnum > lnum:
+ return ("%s%s" % (ltext, rtext), lnum * rnum)
+ return ("%s%s" % (ltext, rtext), lnum + rnum)
+
+ def to_ordinal_num(self, value):
+ self.verify_ordinal(value)
+ return "%s%s" % (value, self.to_ordinal(value))
+
+ def to_ordinal(self, value):
+ self.verify_ordinal(value)
+ try:
+ ordinal_word = self.ords[value]
+ except KeyError:
+ ordinal_word = value #TODO: check nepali ordinal word suffix
+ return ordinal_word
+
+
+ def to_year(self, val, suffix=None, longval=True):
+ if val < 0:
+ val = abs(val)
+ suffix = 'BC' if not suffix else suffix
+ high, low = (val // 100, val % 100)
+ # If year is 00XX, X00X, or beyond 9999, go cardinal.
+ if (high == 0
+ or (high % 10 == 0 and low < 10)
+ or high >= 100):
+ valtext = self.to_cardinal(val)
+ else:
+ hightext = self.to_cardinal(high)
+ if low == 0:
+ lowtext = "啶膏く"
+ elif low < 10:
+ lowtext = "%s" % self.to_cardinal(low)
+ else:
+ lowtext = self.to_cardinal(low)
+ valtext = "%s%s" % (hightext, lowtext)
+ return (valtext if not suffix
+ else "%s%s" % (valtext, suffix))
diff --git a/fun_text_processing/num2words/num2words/lang_VI.py b/fun_text_processing/num2words/num2words/lang_VI.py
new file mode 100644
index 0000000..1999949
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/lang_VI.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+to_19 = (u'kh么ng', u'm峄檛', u'hai', u'ba', u'b峄憂', u'n膬m', u's谩u',
+ u'b岷', u't谩m', u'ch铆n', u'm瓢峄漣', u'm瓢峄漣 m峄檛', u'm瓢峄漣 hai',
+ u'm瓢峄漣 ba', u'm瓢峄漣 b峄憂', u'm瓢峄漣 l膬m', u'm瓢峄漣 s谩u', u'm瓢峄漣 b岷',
+ u'm瓢峄漣 t谩m', u'm瓢峄漣 ch铆n')
+tens = (u'hai m瓢啤i', u'ba m瓢啤i', u'b峄憂 m瓢啤i', u'n膬m m瓢啤i',
+ u's谩u m瓢啤i', u'b岷 m瓢啤i', u't谩m m瓢啤i', u'ch铆n m瓢啤i')
+denom = ('',
+ u'ngh矛n', u'tri峄噓', u't峄�', u'ngh矛n t峄�', u'tr膬m ngh矛n t峄�',
+ 'Quintillion', 'Sextillion', 'Septillion', 'Octillion', 'Nonillion',
+ 'Decillion', 'Undecillion', 'Duodecillion', 'Tredecillion',
+ 'Quattuordecillion', 'Sexdecillion', 'Septendecillion',
+ 'Octodecillion', 'Novemdecillion', 'Vigintillion')
+
+
+class Num2Word_VI(object):
+
+ def _convert_nn(self, val):
+ if val < 20:
+ return to_19[val]
+ for (dcap, dval) in ((k, 20 + (10 * v)) for (v, k) in enumerate(tens)):
+ if dval + 10 > val:
+ if val % 10:
+ a = u'l膬m'
+ if to_19[val % 10] == u'm峄檛':
+ a = u'm峄憈'
+ else:
+ a = to_19[val % 10]
+ if to_19[val % 10] == u'n膬m':
+ a = u'l膬m'
+ return dcap + ' ' + a
+ return dcap
+
+ def _convert_nnn(self, val):
+ word = ''
+ (mod, rem) = (val % 100, val // 100)
+ if rem > 0:
+ word = to_19[rem] + u' tr膬m'
+ if mod > 0:
+ word = word + ' '
+ if mod > 0 and mod < 10:
+ if mod == 5:
+ word = word != '' and word + u'l岷� n膬m' or word + u'n膬m'
+ else:
+ word = word != '' and word + u'l岷� ' \
+ + self._convert_nn(mod) or word + self._convert_nn(mod)
+ if mod >= 10:
+ word = word + self._convert_nn(mod)
+ return word
+
+ def vietnam_number(self, val):
+ if val < 100:
+ return self._convert_nn(val)
+ if val < 1000:
+ return self._convert_nnn(val)
+ for (didx, dval) in ((v - 1, 1000 ** v) for v in range(len(denom))):
+ if dval > val:
+ mod = 1000 ** didx
+ lval = val // mod
+ r = val - (lval * mod)
+
+ ret = self._convert_nnn(lval) + u' ' + denom[didx]
+ if 99 >= r > 0:
+ ret = self._convert_nnn(lval) + u' ' + denom[didx] + u' l岷�'
+ if r > 0:
+ ret = ret + ' ' + self.vietnam_number(r)
+ return ret
+
+ def number_to_text(self, number):
+ number = '%.2f' % number
+ the_list = str(number).split('.')
+ start_word = self.vietnam_number(int(the_list[0]))
+ final_result = start_word
+ if len(the_list) > 1 and int(the_list[1]) > 0:
+ end_word = self.vietnam_number(int(the_list[1]))
+ final_result = final_result + ' ph岷﹜ ' + end_word
+ return final_result
+
+ def to_cardinal(self, number):
+ return self.number_to_text(number)
+
+ def to_ordinal(self, number):
+ return self.to_cardinal(number)
diff --git a/fun_text_processing/num2words/num2words/utils.py b/fun_text_processing/num2words/num2words/utils.py
new file mode 100644
index 0000000..5b7cd8c
--- /dev/null
+++ b/fun_text_processing/num2words/num2words/utils.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+
+def splitbyx(n, x, format_int=True):
+ length = len(n)
+ if length > x:
+ start = length % x
+ if start > 0:
+ result = n[:start]
+ yield int(result) if format_int else result
+ for i in range(start, length, x):
+ result = n[i:i+x]
+ yield int(result) if format_int else result
+ else:
+ yield int(n) if format_int else n
+
+
+def get_digits(n):
+ a = [int(x) for x in reversed(list(('%03d' % n)[-3:]))]
+ return a
diff --git a/fun_text_processing/num2words/requirements-test.txt b/fun_text_processing/num2words/requirements-test.txt
new file mode 100644
index 0000000..7165ac7
--- /dev/null
+++ b/fun_text_processing/num2words/requirements-test.txt
@@ -0,0 +1,6 @@
+flake8
+flake8-copyright
+isort
+pep8<1.6
+coverage
+delegator.py
diff --git a/fun_text_processing/num2words/setup.py b/fun_text_processing/num2words/setup.py
new file mode 100644
index 0000000..a764d1d
--- /dev/null
+++ b/fun_text_processing/num2words/setup.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+import re
+from io import open
+
+from setuptools import find_packages, setup
+
+PACKAGE_NAME = "num2words"
+
+CLASSIFIERS = [
+ 'Development Status :: 5 - Production/Stable',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: GNU Library or Lesser General Public License '
+ '(LGPL)',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Topic :: Software Development :: Internationalization',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'Topic :: Software Development :: Localization',
+ 'Topic :: Text Processing :: Linguistic',
+]
+
+LONG_DESC = open('README.rst', 'rt', encoding="utf-8").read() + '\n\n' + \
+ open('CHANGES.rst', 'rt', encoding="utf-8").read()
+
+
+def find_version(fname):
+ """Parse file & return version number matching 0.0.1 regex
+ Returns str or raises RuntimeError
+ """
+ version = ''
+ with open(fname, 'r', encoding="utf-8") as fp:
+ reg = re.compile(r'__version__ = [\'"]([^\'"]*)[\'"]')
+ for line in fp:
+ m = reg.match(line)
+ if m:
+ version = m.group(1)
+ break
+ if not version:
+ raise RuntimeError('Cannot find version information')
+ return version
+
+
+setup(
+ name=PACKAGE_NAME,
+ version=find_version("bin/num2words"),
+ description='Modules to convert numbers to multilingual words.',
+ long_description=LONG_DESC,
+ license='Alibaba Group',
+ author='Zhang Chong, Alibaba DAMO Academy',
+ author_email='',
+ maintainer='Zhang Chong',
+ maintainer_email='',
+ keywords=' number word numbers words convert conversion '
+ 'localisation localization internationalisation '
+ 'internationalization',
+ url='https://github.com/num2words',
+ packages=find_packages(exclude=['tests']),
+ test_suite='tests',
+ classifiers=CLASSIFIERS,
+ scripts=['bin/num2words'],
+ install_requires=["docopt>=0.6.2"],
+ tests_require=['delegator.py'],
+)
diff --git a/fun_text_processing/num2words/tox.ini b/fun_text_processing/num2words/tox.ini
new file mode 100644
index 0000000..11c8f61
--- /dev/null
+++ b/fun_text_processing/num2words/tox.ini
@@ -0,0 +1,32 @@
+[tox]
+envlist = flake8,isort,py27,py34,py35,py36,py37
+
+[testenv]
+passenv = TRAVIS TRAVIS_*
+deps =
+ coverage
+ delegator.py
+commands =
+ coverage run -m unittest discover
+ coverage report --fail-under=75 --omit=.tox/*,tests/*,/usr/*
+ coverage report --fail-under=100 --include=tests/* --skip-covered
+
+[testenv:flake8]
+changedir = {toxinidir}
+deps =
+ flake8
+ flake8-copyright
+commands =
+ flake8
+
+[testenv:isort]
+changedir = {toxinidir}
+deps =
+ isort
+ delegator.py
+commands =
+ isort --check-only --recursive --diff num2words tests
+
+[testenv:py27]
+setenv =
+ PYTHONIOENCODING = UTF-8
diff --git a/fun_text_processing/text_normalization/data_loader_utils.py b/fun_text_processing/text_normalization/data_loader_utils.py
index abd89a9..0b9de52 100644
--- a/fun_text_processing/text_normalization/data_loader_utils.py
+++ b/fun_text_processing/text_normalization/data_loader_utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import json
diff --git a/fun_text_processing/text_normalization/de/taggers/cardinal.py b/fun_text_processing/text_normalization/de/taggers/cardinal.py
index 738a180..cf53ba6 100644
--- a/fun_text_processing/text_normalization/de/taggers/cardinal.py
+++ b/fun_text_processing/text_normalization/de/taggers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from collections import defaultdict
diff --git a/fun_text_processing/text_normalization/de/taggers/date.py b/fun_text_processing/text_normalization/de/taggers/date.py
index 0539d2d..cec6fd1 100644
--- a/fun_text_processing/text_normalization/de/taggers/date.py
+++ b/fun_text_processing/text_normalization/de/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/taggers/decimal.py b/fun_text_processing/text_normalization/de/taggers/decimal.py
index bcc4309..9237753 100644
--- a/fun_text_processing/text_normalization/de/taggers/decimal.py
+++ b/fun_text_processing/text_normalization/de/taggers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/taggers/electronic.py b/fun_text_processing/text_normalization/de/taggers/electronic.py
index 2e6f9dc..187ac92 100644
--- a/fun_text_processing/text_normalization/de/taggers/electronic.py
+++ b/fun_text_processing/text_normalization/de/taggers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/taggers/fraction.py b/fun_text_processing/text_normalization/de/taggers/fraction.py
index 89594a4..920a82d 100644
--- a/fun_text_processing/text_normalization/de/taggers/fraction.py
+++ b/fun_text_processing/text_normalization/de/taggers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/de/taggers/measure.py b/fun_text_processing/text_normalization/de/taggers/measure.py
index 4f0e169..9ad177e 100644
--- a/fun_text_processing/text_normalization/de/taggers/measure.py
+++ b/fun_text_processing/text_normalization/de/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/taggers/money.py b/fun_text_processing/text_normalization/de/taggers/money.py
index 8d1152d..18b07f9 100644
--- a/fun_text_processing/text_normalization/de/taggers/money.py
+++ b/fun_text_processing/text_normalization/de/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/taggers/ordinal.py b/fun_text_processing/text_normalization/de/taggers/ordinal.py
index 00b8427..9a26a9c 100644
--- a/fun_text_processing/text_normalization/de/taggers/ordinal.py
+++ b/fun_text_processing/text_normalization/de/taggers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
# Russian minimally supervised number grammar.
diff --git a/fun_text_processing/text_normalization/de/taggers/telephone.py b/fun_text_processing/text_normalization/de/taggers/telephone.py
index 4d8704e..bf769c9 100644
--- a/fun_text_processing/text_normalization/de/taggers/telephone.py
+++ b/fun_text_processing/text_normalization/de/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/taggers/time.py b/fun_text_processing/text_normalization/de/taggers/time.py
index 82b4034..8880654 100644
--- a/fun_text_processing/text_normalization/de/taggers/time.py
+++ b/fun_text_processing/text_normalization/de/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py b/fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py
index 9a6c059..04c4e01 100644
--- a/fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/text_normalization/de/taggers/whitelist.py b/fun_text_processing/text_normalization/de/taggers/whitelist.py
index 3f81a43..58d4f7b 100644
--- a/fun_text_processing/text_normalization/de/taggers/whitelist.py
+++ b/fun_text_processing/text_normalization/de/taggers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/taggers/word.py b/fun_text_processing/text_normalization/de/taggers/word.py
index 73a5dbb..362517d 100644
--- a/fun_text_processing/text_normalization/de/taggers/word.py
+++ b/fun_text_processing/text_normalization/de/taggers/word.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/text_normalization/de/utils.py b/fun_text_processing/text_normalization/de/utils.py
index 56cd5bf..e233075 100644
--- a/fun_text_processing/text_normalization/de/utils.py
+++ b/fun_text_processing/text_normalization/de/utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import csv
import os
diff --git a/fun_text_processing/text_normalization/de/verbalizers/cardinal.py b/fun_text_processing/text_normalization/de/verbalizers/cardinal.py
index 0ddfd60..aef0c56 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/cardinal.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/de/verbalizers/date.py b/fun_text_processing/text_normalization/de/verbalizers/date.py
index 2ca01f3..c3413a9 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/date.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/verbalizers/decimal.py b/fun_text_processing/text_normalization/de/verbalizers/decimal.py
index a0f8038..ee330db 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/decimal.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/decimal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.taggers.decimal import quantities
diff --git a/fun_text_processing/text_normalization/de/verbalizers/electronic.py b/fun_text_processing/text_normalization/de/verbalizers/electronic.py
index 12f0a8d..ca0faac 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/electronic.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/verbalizers/fraction.py b/fun_text_processing/text_normalization/de/verbalizers/fraction.py
index 432b469..a77e26d 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/fraction.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/fraction.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/de/verbalizers/measure.py b/fun_text_processing/text_normalization/de/verbalizers/measure.py
index 01a8657..e802ce8 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/measure.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/de/verbalizers/money.py b/fun_text_processing/text_normalization/de/verbalizers/money.py
index c1b4226..a562b2c 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/money.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_preserve_order
diff --git a/fun_text_processing/text_normalization/de/verbalizers/ordinal.py b/fun_text_processing/text_normalization/de/verbalizers/ordinal.py
index 028fad0..4c18c66 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/ordinal.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/ordinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path
diff --git a/fun_text_processing/text_normalization/de/verbalizers/telephone.py b/fun_text_processing/text_normalization/de/verbalizers/telephone.py
index a42cfec..417dbb0 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/telephone.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_preserve_order
diff --git a/fun_text_processing/text_normalization/de/verbalizers/time.py b/fun_text_processing/text_normalization/de/verbalizers/time.py
index cc8550c..482a4c2 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/time.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/time.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
diff --git a/fun_text_processing/text_normalization/de/verbalizers/verbalize.py b/fun_text_processing/text_normalization/de/verbalizers/verbalize.py
index b8baad7..1533327 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/verbalize.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/verbalize.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst as CardinalTagger
from fun_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst
diff --git a/fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py b/fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py
index e4d2adc..6587412 100644
--- a/fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py
+++ b/fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/text_normalization/en/__init__.py b/fun_text_processing/text_normalization/en/__init__.py
index 248ebfa..40ce4aa 100644
--- a/fun_text_processing/text_normalization/en/__init__.py
+++ b/fun_text_processing/text_normalization/en/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
from fun_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst
diff --git a/fun_text_processing/text_normalization/en/clean_eval_data.py b/fun_text_processing/text_normalization/en/clean_eval_data.py
index 5b81253..da9d7a9 100644
--- a/fun_text_processing/text_normalization/en/clean_eval_data.py
+++ b/fun_text_processing/text_normalization/en/clean_eval_data.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from argparse import ArgumentParser
diff --git a/fun_text_processing/text_normalization/en/data/number/cardinal_number_name.far b/fun_text_processing/text_normalization/en/data/number/cardinal_number_name.far
new file mode 100755
index 0000000..2ec6825
--- /dev/null
+++ b/fun_text_processing/text_normalization/en/data/number/cardinal_number_name.far
Binary files differ
diff --git a/fun_text_processing/text_normalization/en/graph_utils.py b/fun_text_processing/text_normalization/en/graph_utils.py
index 2749f84..2ee3536 100644
--- a/fun_text_processing/text_normalization/en/graph_utils.py
+++ b/fun_text_processing/text_normalization/en/graph_utils.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
import string
diff --git a/fun_text_processing/text_normalization/en/taggers/abbreviation.py b/fun_text_processing/text_normalization/en/taggers/abbreviation.py
index 25574ba..a0dee24 100644
--- a/fun_text_processing/text_normalization/en/taggers/abbreviation.py
+++ b/fun_text_processing/text_normalization/en/taggers/abbreviation.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/cardinal.py b/fun_text_processing/text_normalization/en/taggers/cardinal.py
index 4882867..c406fe4 100644
--- a/fun_text_processing/text_normalization/en/taggers/cardinal.py
+++ b/fun_text_processing/text_normalization/en/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/date.py b/fun_text_processing/text_normalization/en/taggers/date.py
index 70af3aa..897d05d 100644
--- a/fun_text_processing/text_normalization/en/taggers/date.py
+++ b/fun_text_processing/text_normalization/en/taggers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/decimal.py b/fun_text_processing/text_normalization/en/taggers/decimal.py
index 11cb60c..19f0ccb 100644
--- a/fun_text_processing/text_normalization/en/taggers/decimal.py
+++ b/fun_text_processing/text_normalization/en/taggers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, TO_UPPER, GraphFst, get_abs_path
diff --git a/fun_text_processing/text_normalization/en/taggers/electronic.py b/fun_text_processing/text_normalization/en/taggers/electronic.py
index ad4095d..269cdb1 100644
--- a/fun_text_processing/text_normalization/en/taggers/electronic.py
+++ b/fun_text_processing/text_normalization/en/taggers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/fraction.py b/fun_text_processing/text_normalization/en/taggers/fraction.py
index 3788fb8..d7ee091 100644
--- a/fun_text_processing/text_normalization/en/taggers/fraction.py
+++ b/fun_text_processing/text_normalization/en/taggers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, get_abs_path
diff --git a/fun_text_processing/text_normalization/en/taggers/measure.py b/fun_text_processing/text_normalization/en/taggers/measure.py
index a469f9e..30f6cb9 100644
--- a/fun_text_processing/text_normalization/en/taggers/measure.py
+++ b/fun_text_processing/text_normalization/en/taggers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/money.py b/fun_text_processing/text_normalization/en/taggers/money.py
index 50317bd..e2525c1 100644
--- a/fun_text_processing/text_normalization/en/taggers/money.py
+++ b/fun_text_processing/text_normalization/en/taggers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/ordinal.py b/fun_text_processing/text_normalization/en/taggers/ordinal.py
index faaac8a..a44e2aa 100644
--- a/fun_text_processing/text_normalization/en/taggers/ordinal.py
+++ b/fun_text_processing/text_normalization/en/taggers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/punctuation.py b/fun_text_processing/text_normalization/en/taggers/punctuation.py
index 62f6823..9b57943 100644
--- a/fun_text_processing/text_normalization/en/taggers/punctuation.py
+++ b/fun_text_processing/text_normalization/en/taggers/punctuation.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import sys
from unicodedata import category
diff --git a/fun_text_processing/text_normalization/en/taggers/range.py b/fun_text_processing/text_normalization/en/taggers/range.py
index 297f542..0c76993 100644
--- a/fun_text_processing/text_normalization/en/taggers/range.py
+++ b/fun_text_processing/text_normalization/en/taggers/range.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, convert_space
diff --git a/fun_text_processing/text_normalization/en/taggers/roman.py b/fun_text_processing/text_normalization/en/taggers/roman.py
index e0590d2..fbd27ca 100644
--- a/fun_text_processing/text_normalization/en/taggers/roman.py
+++ b/fun_text_processing/text_normalization/en/taggers/roman.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/serial.py b/fun_text_processing/text_normalization/en/taggers/serial.py
index 858ff22..f039f6a 100644
--- a/fun_text_processing/text_normalization/en/taggers/serial.py
+++ b/fun_text_processing/text_normalization/en/taggers/serial.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/telephone.py b/fun_text_processing/text_normalization/en/taggers/telephone.py
index 9c7d8e3..8ac473f 100644
--- a/fun_text_processing/text_normalization/en/taggers/telephone.py
+++ b/fun_text_processing/text_normalization/en/taggers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/time.py b/fun_text_processing/text_normalization/en/taggers/time.py
index 9c41588..be2297f 100644
--- a/fun_text_processing/text_normalization/en/taggers/time.py
+++ b/fun_text_processing/text_normalization/en/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify.py b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify.py
index 420e328..3b5590d 100644
--- a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
import time
diff --git a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py
index d4b54a9..cdfa854 100644
--- a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py
+++ b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py
index 269ca63..853704b 100644
--- a/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py
+++ b/fun_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/en/taggers/whitelist.py b/fun_text_processing/text_normalization/en/taggers/whitelist.py
index 5ef6b99..568c12f 100644
--- a/fun_text_processing/text_normalization/en/taggers/whitelist.py
+++ b/fun_text_processing/text_normalization/en/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/taggers/word.py b/fun_text_processing/text_normalization/en/taggers/word.py
index 148a522..9409e15 100644
--- a/fun_text_processing/text_normalization/en/taggers/word.py
+++ b/fun_text_processing/text_normalization/en/taggers/word.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/utils.py b/fun_text_processing/text_normalization/en/utils.py
index 7ce67c1..829d747 100644
--- a/fun_text_processing/text_normalization/en/utils.py
+++ b/fun_text_processing/text_normalization/en/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import csv
import os
diff --git a/fun_text_processing/text_normalization/en/verbalizers/abbreviation.py b/fun_text_processing/text_normalization/en/verbalizers/abbreviation.py
index 50a6190..1c5a0ca 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/abbreviation.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/abbreviation.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/en/verbalizers/cardinal.py b/fun_text_processing/text_normalization/en/verbalizers/cardinal.py
index e0329f4..70bf202 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/cardinal.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/date.py b/fun_text_processing/text_normalization/en/verbalizers/date.py
index cef9487..f3ad374 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/date.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/verbalizers/decimal.py b/fun_text_processing/text_normalization/en/verbalizers/decimal.py
index 0b8924b..27dd772 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/decimal.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, insert_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/electronic.py b/fun_text_processing/text_normalization/en/verbalizers/electronic.py
index f832274..3a30dc2 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/electronic.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/verbalizers/fraction.py b/fun_text_processing/text_normalization/en/verbalizers/fraction.py
index f8df9d6..fc5279d 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/fraction.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/measure.py b/fun_text_processing/text_normalization/en/verbalizers/measure.py
index cc75dd3..29f8939 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/measure.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, insert_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/money.py b/fun_text_processing/text_normalization/en/verbalizers/money.py
index 456bef9..66f7fce 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/money.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/verbalizers/ordinal.py b/fun_text_processing/text_normalization/en/verbalizers/ordinal.py
index 7d95aa2..2a8c3fb 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/ordinal.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
diff --git a/fun_text_processing/text_normalization/en/verbalizers/post_processing.py b/fun_text_processing/text_normalization/en/verbalizers/post_processing.py
index 1f9f136..e0da98f 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/post_processing.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/post_processing.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/en/verbalizers/roman.py b/fun_text_processing/text_normalization/en/verbalizers/roman.py
index c549c67..172c697 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/roman.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/roman.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/en/verbalizers/telephone.py b/fun_text_processing/text_normalization/en/verbalizers/telephone.py
index b105685..cd7cde6 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/telephone.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, insert_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/time.py b/fun_text_processing/text_normalization/en/verbalizers/time.py
index 5d34469..a52b75d 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/time.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/en/verbalizers/verbalize.py b/fun_text_processing/text_normalization/en/verbalizers/verbalize.py
index c993519..0276683 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/verbalize.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from fun_text_processing.text_normalization.en.verbalizers.abbreviation import AbbreviationFst
diff --git a/fun_text_processing/text_normalization/en/verbalizers/verbalize_final.py b/fun_text_processing/text_normalization/en/verbalizers/verbalize_final.py
index a791dde..a63fcdf 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/verbalize_final.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/verbalize_final.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/en/verbalizers/whitelist.py b/fun_text_processing/text_normalization/en/verbalizers/whitelist.py
index 8ab80cb..4f0d5f2 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/whitelist.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/whitelist.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/en/verbalizers/word.py b/fun_text_processing/text_normalization/en/verbalizers/word.py
index fe3b704..e835b2f 100644
--- a/fun_text_processing/text_normalization/en/verbalizers/word.py
+++ b/fun_text_processing/text_normalization/en/verbalizers/word.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, DAMO_SIGMA, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/es/__init__.py b/fun_text_processing/text_normalization/es/__init__.py
index 9fd35d6..0b7d10c 100644
--- a/fun_text_processing/text_normalization/es/__init__.py
+++ b/fun_text_processing/text_normalization/es/__init__.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
LOCALIZATION = "eu" # Set to am for alternate formatting
diff --git a/fun_text_processing/text_normalization/es/graph_utils.py b/fun_text_processing/text_normalization/es/graph_utils.py
index f00996c..733ee0c 100644
--- a/fun_text_processing/text_normalization/es/graph_utils.py
+++ b/fun_text_processing/text_normalization/es/graph_utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, DAMO_SPACE
diff --git a/fun_text_processing/text_normalization/es/taggers/cardinal.py b/fun_text_processing/text_normalization/es/taggers/cardinal.py
index 3d605c4..df18ff4 100644
--- a/fun_text_processing/text_normalization/es/taggers/cardinal.py
+++ b/fun_text_processing/text_normalization/es/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/date.py b/fun_text_processing/text_normalization/es/taggers/date.py
index ef85b37..df09d83 100644
--- a/fun_text_processing/text_normalization/es/taggers/date.py
+++ b/fun_text_processing/text_normalization/es/taggers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, DAMO_SPACE, GraphFst, delete_extra_space
diff --git a/fun_text_processing/text_normalization/es/taggers/decimals.py b/fun_text_processing/text_normalization/es/taggers/decimals.py
index 917e884..0032c27 100644
--- a/fun_text_processing/text_normalization/es/taggers/decimals.py
+++ b/fun_text_processing/text_normalization/es/taggers/decimals.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/electronic.py b/fun_text_processing/text_normalization/es/taggers/electronic.py
index 075ed21..13853cc 100644
--- a/fun_text_processing/text_normalization/es/taggers/electronic.py
+++ b/fun_text_processing/text_normalization/es/taggers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_ALPHA, DAMO_DIGIT, GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/es/taggers/fraction.py b/fun_text_processing/text_normalization/es/taggers/fraction.py
index f34e223..6ec5526 100644
--- a/fun_text_processing/text_normalization/es/taggers/fraction.py
+++ b/fun_text_processing/text_normalization/es/taggers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/measure.py b/fun_text_processing/text_normalization/es/taggers/measure.py
index 0a14d9f..bb69667 100644
--- a/fun_text_processing/text_normalization/es/taggers/measure.py
+++ b/fun_text_processing/text_normalization/es/taggers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/money.py b/fun_text_processing/text_normalization/es/taggers/money.py
index 8830b1a..a7f54dc 100644
--- a/fun_text_processing/text_normalization/es/taggers/money.py
+++ b/fun_text_processing/text_normalization/es/taggers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/ordinal.py b/fun_text_processing/text_normalization/es/taggers/ordinal.py
index 0942b07..eecfe33 100644
--- a/fun_text_processing/text_normalization/es/taggers/ordinal.py
+++ b/fun_text_processing/text_normalization/es/taggers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/telephone.py b/fun_text_processing/text_normalization/es/taggers/telephone.py
index 26b266e..77a46d4 100644
--- a/fun_text_processing/text_normalization/es/taggers/telephone.py
+++ b/fun_text_processing/text_normalization/es/taggers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/es/taggers/time.py b/fun_text_processing/text_normalization/es/taggers/time.py
index 854cd51..d0b6441 100644
--- a/fun_text_processing/text_normalization/es/taggers/time.py
+++ b/fun_text_processing/text_normalization/es/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/taggers/tokenize_and_classify.py b/fun_text_processing/text_normalization/es/taggers/tokenize_and_classify.py
index 5bad3ee..cd4c361 100644
--- a/fun_text_processing/text_normalization/es/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/text_normalization/es/taggers/tokenize_and_classify.py
@@ -1,16 +1,5 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
+
import os
diff --git a/fun_text_processing/text_normalization/es/taggers/whitelist.py b/fun_text_processing/text_normalization/es/taggers/whitelist.py
index 54fb749..11ff0ff 100644
--- a/fun_text_processing/text_normalization/es/taggers/whitelist.py
+++ b/fun_text_processing/text_normalization/es/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
diff --git a/fun_text_processing/text_normalization/es/taggers/word.py b/fun_text_processing/text_normalization/es/taggers/word.py
index 2c2a84b..b2ba79c 100644
--- a/fun_text_processing/text_normalization/es/taggers/word.py
+++ b/fun_text_processing/text_normalization/es/taggers/word.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/text_normalization/es/utils.py b/fun_text_processing/text_normalization/es/utils.py
index 7f9865b..29875f7 100644
--- a/fun_text_processing/text_normalization/es/utils.py
+++ b/fun_text_processing/text_normalization/es/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import csv
diff --git a/fun_text_processing/text_normalization/es/verbalizers/cardinal.py b/fun_text_processing/text_normalization/es/verbalizers/cardinal.py
index cef4d81..cdb2540 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/cardinal.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/es/verbalizers/date.py b/fun_text_processing/text_normalization/es/verbalizers/date.py
index bc99958..0058030 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/date.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/decimals.py b/fun_text_processing/text_normalization/es/verbalizers/decimals.py
index a602f75..43c9d2b 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/decimals.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/decimals.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/electronic.py b/fun_text_processing/text_normalization/es/verbalizers/electronic.py
index dca0c3a..6104ff6 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/electronic.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/fraction.py b/fun_text_processing/text_normalization/es/verbalizers/fraction.py
index 23812c3..3f99d83 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/fraction.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/measure.py b/fun_text_processing/text_normalization/es/verbalizers/measure.py
index bab58cb..0d27ba3 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/measure.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/money.py b/fun_text_processing/text_normalization/es/verbalizers/money.py
index 12cf76e..4bb9a83 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/money.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/ordinal.py b/fun_text_processing/text_normalization/es/verbalizers/ordinal.py
index da7136c..6f17861 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/ordinal.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, DAMO_SPACE, GraphFst
diff --git a/fun_text_processing/text_normalization/es/verbalizers/telephone.py b/fun_text_processing/text_normalization/es/verbalizers/telephone.py
index 82a117d..75acee9 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/telephone.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/es/verbalizers/time.py b/fun_text_processing/text_normalization/es/verbalizers/time.py
index 6664cbf..e209c87 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/time.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/es/verbalizers/verbalize.py b/fun_text_processing/text_normalization/es/verbalizers/verbalize.py
index 5458e8f..512beed 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/verbalize.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
diff --git a/fun_text_processing/text_normalization/es/verbalizers/verbalize_final.py b/fun_text_processing/text_normalization/es/verbalizers/verbalize_final.py
index 088b1bd..c9589e9 100644
--- a/fun_text_processing/text_normalization/es/verbalizers/verbalize_final.py
+++ b/fun_text_processing/text_normalization/es/verbalizers/verbalize_final.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/export_models.py b/fun_text_processing/text_normalization/export_models.py
index 2b9898b..72c3ff8 100644
--- a/fun_text_processing/text_normalization/export_models.py
+++ b/fun_text_processing/text_normalization/export_models.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import os
from time import perf_counter
from argparse import ArgumentParser
diff --git a/fun_text_processing/text_normalization/normalize.py b/fun_text_processing/text_normalization/normalize.py
index 30d4941..d87db4a 100644
--- a/fun_text_processing/text_normalization/normalize.py
+++ b/fun_text_processing/text_normalization/normalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import itertools
import os
diff --git a/fun_text_processing/text_normalization/normalize_with_audio.py b/fun_text_processing/text_normalization/normalize_with_audio.py
index 7769479..61e32f2 100644
--- a/fun_text_processing/text_normalization/normalize_with_audio.py
+++ b/fun_text_processing/text_normalization/normalize_with_audio.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import json
import os
diff --git a/fun_text_processing/text_normalization/ru/alphabet.py b/fun_text_processing/text_normalization/ru/alphabet.py
index 90aa5da..2c0850d 100644
--- a/fun_text_processing/text_normalization/ru/alphabet.py
+++ b/fun_text_processing/text_normalization/ru/alphabet.py
@@ -1,16 +1,7 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
+
+# Copyright 2017 Google Inc.
+
# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
# Russian minimally supervised number grammar.
diff --git a/fun_text_processing/text_normalization/ru/data/utils/g.fst b/fun_text_processing/text_normalization/ru/data/utils/g.fst
new file mode 100644
index 0000000..66665f3
--- /dev/null
+++ b/fun_text_processing/text_normalization/ru/data/utils/g.fst
Binary files differ
diff --git a/fun_text_processing/text_normalization/ru/data/utils/universal_thousands_punct.far b/fun_text_processing/text_normalization/ru/data/utils/universal_thousands_punct.far
new file mode 100644
index 0000000..4c24b65
--- /dev/null
+++ b/fun_text_processing/text_normalization/ru/data/utils/universal_thousands_punct.far
Binary files differ
diff --git a/fun_text_processing/text_normalization/ru/data/utils/util_arithmetic.far b/fun_text_processing/text_normalization/ru/data/utils/util_arithmetic.far
new file mode 100644
index 0000000..584f04d
--- /dev/null
+++ b/fun_text_processing/text_normalization/ru/data/utils/util_arithmetic.far
Binary files differ
diff --git a/fun_text_processing/text_normalization/ru/data/utils/util_byte.far b/fun_text_processing/text_normalization/ru/data/utils/util_byte.far
new file mode 100644
index 0000000..eabf8f1
--- /dev/null
+++ b/fun_text_processing/text_normalization/ru/data/utils/util_byte.far
Binary files differ
diff --git a/fun_text_processing/text_normalization/ru/taggers/date.py b/fun_text_processing/text_normalization/ru/taggers/date.py
index 7f0b931..2cc9c69 100644
--- a/fun_text_processing/text_normalization/ru/taggers/date.py
+++ b/fun_text_processing/text_normalization/ru/taggers/date.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/ru/taggers/decimals.py b/fun_text_processing/text_normalization/ru/taggers/decimals.py
index 9bbe363..95a654c 100644
--- a/fun_text_processing/text_normalization/ru/taggers/decimals.py
+++ b/fun_text_processing/text_normalization/ru/taggers/decimals.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
from collections import defaultdict
diff --git a/fun_text_processing/text_normalization/ru/taggers/electronic.py b/fun_text_processing/text_normalization/ru/taggers/electronic.py
index 792e69e..fff1433 100644
--- a/fun_text_processing/text_normalization/ru/taggers/electronic.py
+++ b/fun_text_processing/text_normalization/ru/taggers/electronic.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
diff --git a/fun_text_processing/text_normalization/ru/taggers/measure.py b/fun_text_processing/text_normalization/ru/taggers/measure.py
index 57270b0..01f3c7b 100644
--- a/fun_text_processing/text_normalization/ru/taggers/measure.py
+++ b/fun_text_processing/text_normalization/ru/taggers/measure.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/ru/taggers/money.py b/fun_text_processing/text_normalization/ru/taggers/money.py
index 91bf398..025a1d8 100644
--- a/fun_text_processing/text_normalization/ru/taggers/money.py
+++ b/fun_text_processing/text_normalization/ru/taggers/money.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, DAMO_SIGMA, DAMO_SPACE, GraphFst
diff --git a/fun_text_processing/text_normalization/ru/taggers/number_names.py b/fun_text_processing/text_normalization/ru/taggers/number_names.py
index 9d23bab..98fddaf 100644
--- a/fun_text_processing/text_normalization/ru/taggers/number_names.py
+++ b/fun_text_processing/text_normalization/ru/taggers/number_names.py
@@ -1,16 +1,8 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
+
+# Copyright 2017 Google Inc.
+
+
# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
# Russian minimally supervised number grammar.
#
diff --git a/fun_text_processing/text_normalization/ru/taggers/telephone.py b/fun_text_processing/text_normalization/ru/taggers/telephone.py
index 8c50ca2..c884b0f 100644
--- a/fun_text_processing/text_normalization/ru/taggers/telephone.py
+++ b/fun_text_processing/text_normalization/ru/taggers/telephone.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, delete_space, insert_space
diff --git a/fun_text_processing/text_normalization/ru/taggers/time.py b/fun_text_processing/text_normalization/ru/taggers/time.py
index 969d9d7..6ab728d 100644
--- a/fun_text_processing/text_normalization/ru/taggers/time.py
+++ b/fun_text_processing/text_normalization/ru/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py b/fun_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py
index 70ac198..2bc5227 100644
--- a/fun_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/ru/taggers/whitelist.py b/fun_text_processing/text_normalization/ru/taggers/whitelist.py
index 532ccd8..60518c6 100644
--- a/fun_text_processing/text_normalization/ru/taggers/whitelist.py
+++ b/fun_text_processing/text_normalization/ru/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, convert_space
diff --git a/fun_text_processing/text_normalization/ru/taggers/word.py b/fun_text_processing/text_normalization/ru/taggers/word.py
index 73a5dbb..362517d 100644
--- a/fun_text_processing/text_normalization/ru/taggers/word.py
+++ b/fun_text_processing/text_normalization/ru/taggers/word.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
diff --git a/fun_text_processing/text_normalization/ru/utils.py b/fun_text_processing/text_normalization/ru/utils.py
index 6f500b2..861f75b 100644
--- a/fun_text_processing/text_normalization/ru/utils.py
+++ b/fun_text_processing/text_normalization/ru/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import csv
import os
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/cardinal.py b/fun_text_processing/text_normalization/ru/verbalizers/cardinal.py
index 79f482b..3728148 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/cardinal.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/cardinal.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/date.py b/fun_text_processing/text_normalization/ru/verbalizers/date.py
index b497787..d959ea3 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/date.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/decimal.py b/fun_text_processing/text_normalization/ru/verbalizers/decimal.py
index 0b3dd8f..4fd5dbd 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/decimal.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/decimal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/electronic.py b/fun_text_processing/text_normalization/ru/verbalizers/electronic.py
index bf83218..1c67b08 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/electronic.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/electronic.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/measure.py b/fun_text_processing/text_normalization/ru/verbalizers/measure.py
index 51cde60..951a400 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/measure.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import (
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/money.py b/fun_text_processing/text_normalization/ru/verbalizers/money.py
index 093861f..7f07de7 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/money.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/ordinal.py b/fun_text_processing/text_normalization/ru/verbalizers/ordinal.py
index ce119aa..ad6c914 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/ordinal.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/ordinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/telephone.py b/fun_text_processing/text_normalization/ru/verbalizers/telephone.py
index 1328923..5513add 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/telephone.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/telephone.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/time.py b/fun_text_processing/text_normalization/ru/verbalizers/time.py
index 0a06dfa..55289cc 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/time.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst, delete_space, insert_space
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/verbalize.py b/fun_text_processing/text_normalization/ru/verbalizers/verbalize.py
index 4795973..ce635aa 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/verbalize.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
diff --git a/fun_text_processing/text_normalization/ru/verbalizers/verbalize_final.py b/fun_text_processing/text_normalization/ru/verbalizers/verbalize_final.py
index 9d4edcf..841a94e 100644
--- a/fun_text_processing/text_normalization/ru/verbalizers/verbalize_final.py
+++ b/fun_text_processing/text_normalization/ru/verbalizers/verbalize_final.py
@@ -1,16 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
import os
diff --git a/fun_text_processing/text_normalization/run_evaluate.py b/fun_text_processing/text_normalization/run_evaluate.py
index d0b636a..b206940 100644
--- a/fun_text_processing/text_normalization/run_evaluate.py
+++ b/fun_text_processing/text_normalization/run_evaluate.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from argparse import ArgumentParser
diff --git a/fun_text_processing/text_normalization/token_parser.py b/fun_text_processing/text_normalization/token_parser.py
index 3e39cc3..10b2ab3 100644
--- a/fun_text_processing/text_normalization/token_parser.py
+++ b/fun_text_processing/text_normalization/token_parser.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import string
from collections import OrderedDict
diff --git a/fun_text_processing/text_normalization/zh/graph_utils.py b/fun_text_processing/text_normalization/zh/graph_utils.py
index 86780c4..83f38ac 100644
--- a/fun_text_processing/text_normalization/zh/graph_utils.py
+++ b/fun_text_processing/text_normalization/zh/graph_utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
import string
diff --git a/fun_text_processing/text_normalization/zh/taggers/cardinal.py b/fun_text_processing/text_normalization/zh/taggers/cardinal.py
index 30775fc..1c14483 100644
--- a/fun_text_processing/text_normalization/zh/taggers/cardinal.py
+++ b/fun_text_processing/text_normalization/zh/taggers/cardinal.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/zh/taggers/char.py b/fun_text_processing/text_normalization/zh/taggers/char.py
index 7effe88..0d702f3 100644
--- a/fun_text_processing/text_normalization/zh/taggers/char.py
+++ b/fun_text_processing/text_normalization/zh/taggers/char.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.zh.graph_utils import FUN_CHAR, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/text_normalization/zh/taggers/date.py b/fun_text_processing/text_normalization/zh/taggers/date.py
index 23f111c..2e7b389 100644
--- a/fun_text_processing/text_normalization/zh/taggers/date.py
+++ b/fun_text_processing/text_normalization/zh/taggers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_CHAR, FUN_DIGIT, GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/zh/taggers/fraction.py b/fun_text_processing/text_normalization/zh/taggers/fraction.py
index ac39a14..059539f 100644
--- a/fun_text_processing/text_normalization/zh/taggers/fraction.py
+++ b/fun_text_processing/text_normalization/zh/taggers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_DIGIT, GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/zh/taggers/math_symbol.py b/fun_text_processing/text_normalization/zh/taggers/math_symbol.py
index 38cac3c..1dc8a7f 100644
--- a/fun_text_processing/text_normalization/zh/taggers/math_symbol.py
+++ b/fun_text_processing/text_normalization/zh/taggers/math_symbol.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/zh/taggers/measure.py b/fun_text_processing/text_normalization/zh/taggers/measure.py
index 0b41667..95b5760 100644
--- a/fun_text_processing/text_normalization/zh/taggers/measure.py
+++ b/fun_text_processing/text_normalization/zh/taggers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/zh/taggers/money.py b/fun_text_processing/text_normalization/zh/taggers/money.py
index 0b7648b..804beaa 100644
--- a/fun_text_processing/text_normalization/zh/taggers/money.py
+++ b/fun_text_processing/text_normalization/zh/taggers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/zh/taggers/preprocessor.py b/fun_text_processing/text_normalization/zh/taggers/preprocessor.py
index c5abeaf..fa296da 100644
--- a/fun_text_processing/text_normalization/zh/taggers/preprocessor.py
+++ b/fun_text_processing/text_normalization/zh/taggers/preprocessor.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_SIGMA, GraphFst
diff --git a/fun_text_processing/text_normalization/zh/taggers/time.py b/fun_text_processing/text_normalization/zh/taggers/time.py
index 214e351..9855707 100644
--- a/fun_text_processing/text_normalization/zh/taggers/time.py
+++ b/fun_text_processing/text_normalization/zh/taggers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst, insert_space
diff --git a/fun_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py b/fun_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py
index b9b925b..439357c 100644
--- a/fun_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py
+++ b/fun_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/zh/taggers/whitelist.py b/fun_text_processing/text_normalization/zh/taggers/whitelist.py
index 04c45c8..6880224 100644
--- a/fun_text_processing/text_normalization/zh/taggers/whitelist.py
+++ b/fun_text_processing/text_normalization/zh/taggers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/zh/utils.py b/fun_text_processing/text_normalization/zh/utils.py
index 63e0712..b9793e4 100644
--- a/fun_text_processing/text_normalization/zh/utils.py
+++ b/fun_text_processing/text_normalization/zh/utils.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import csv
import os
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/cardinal.py b/fun_text_processing/text_normalization/zh/verbalizers/cardinal.py
index e7aff6a..5606b02 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/cardinal.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/cardinal.py
@@ -1,17 +1,3 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/char.py b/fun_text_processing/text_normalization/zh/verbalizers/char.py
index e19e4cc..4d693fc 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/char.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/char.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
from pynini.lib import pynutil
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/date.py b/fun_text_processing/text_normalization/zh/verbalizers/date.py
index 367c8d2..dd0ae12 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/date.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/date.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/fraction.py b/fun_text_processing/text_normalization/zh/verbalizers/fraction.py
index ee72473..111a79d 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/fraction.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/fraction.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
from fun_text_processing.text_normalization.zh.taggers.cardinal import Cardinal
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/math_symbol.py b/fun_text_processing/text_normalization/zh/verbalizers/math_symbol.py
index 93c6b19..939f869 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/math_symbol.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/math_symbol.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/measure.py b/fun_text_processing/text_normalization/zh/verbalizers/measure.py
index 2eea0f8..0f0156b 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/measure.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/measure.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/money.py b/fun_text_processing/text_normalization/zh/verbalizers/money.py
index 4c6bdc5..ad59b46 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/money.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/money.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py b/fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py
index d71f437..77dc9ff 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/postprocessor.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import (
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/time.py b/fun_text_processing/text_normalization/zh/verbalizers/time.py
index d26c2b7..c782375 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/time.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/time.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst, delete_space
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/verbalize.py b/fun_text_processing/text_normalization/zh/verbalizers/verbalize.py
index 4194015..d01c2e7 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/verbalize.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/verbalize.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/verbalize_final.py b/fun_text_processing/text_normalization/zh/verbalizers/verbalize_final.py
index ea318a1..d03e9f3 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/verbalize_final.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/verbalize_final.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import os
diff --git a/fun_text_processing/text_normalization/zh/verbalizers/whitelist.py b/fun_text_processing/text_normalization/zh/verbalizers/whitelist.py
index b36d0de..212eb6b 100644
--- a/fun_text_processing/text_normalization/zh/verbalizers/whitelist.py
+++ b/fun_text_processing/text_normalization/zh/verbalizers/whitelist.py
@@ -1,16 +1,4 @@
-# Copyright NeMo (https://github.com/NVIDIA/NeMo). All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
diff --git a/fun_text_processing/version.txt b/fun_text_processing/version.txt
deleted file mode 100644
index 17e51c3..0000000
--- a/fun_text_processing/version.txt
+++ /dev/null
@@ -1 +0,0 @@
-0.1.1
diff --git a/funasr/bin/asr_inference.py b/funasr/bin/asr_inference.py
old mode 100755
new mode 100644
index b937f88..985ff50
--- a/funasr/bin/asr_inference.py
+++ b/funasr/bin/asr_inference.py
@@ -42,9 +42,6 @@
from funasr.utils import asr_utils, wav_utils, postprocess_utils
from funasr.models.frontend.wav_frontend import WavFrontend
-from modelscope.utils.logger import get_logger
-
-logger = get_logger()
header_colors = '\033[95m'
end_colors = '\033[0m'
@@ -71,6 +68,7 @@
self,
asr_train_config: Union[Path, str] = None,
asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
lm_train_config: Union[Path, str] = None,
lm_file: Union[Path, str] = None,
token_type: str = None,
@@ -95,13 +93,14 @@
# 1. Build ASR model
scorers = {}
asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, device
+ asr_train_config, asr_model_file, cmvn_file, device
)
- if asr_model.frontend is None and frontend_conf is not None:
- frontend = WavFrontend(**frontend_conf)
- asr_model.frontend = frontend
- # logging.info("asr_model: {}".format(asr_model))
- # logging.info("asr_train_args: {}".format(asr_train_args))
+ frontend = None
+ if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
+ frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+
+ logging.info("asr_model: {}".format(asr_model))
+ logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
decoder = asr_model.decoder
@@ -164,7 +163,7 @@
else:
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
- # logging.info(f"Text tokenizer: {tokenizer}")
+ logging.info(f"Text tokenizer: {tokenizer}")
self.asr_model = asr_model
self.asr_train_args = asr_train_args
@@ -177,10 +176,11 @@
self.device = device
self.dtype = dtype
self.nbest = nbest
+ self.frontend = frontend
@torch.no_grad()
def __call__(
- self, speech: Union[torch.Tensor, np.ndarray]
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
) -> List[
Tuple[
Optional[str],
@@ -203,12 +203,16 @@
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
- # data: (Nsamples,) -> (1, Nsamples)
- speech = speech.unsqueeze(0).to(getattr(torch, self.dtype))
- lfr_factor = max(1, (speech.size()[-1] // 80) - 1)
- # lengths: (1,)
- lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
- batch = {"speech": speech, "speech_lengths": lengths}
+ if self.frontend is not None:
+ feats, feats_len = self.frontend.forward(speech, speech_lengths)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ self.asr_model.frontend = None
+ else:
+ feats = speech
+ feats_len = speech_lengths
+ lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
+ batch = {"speech": feats, "speech_lengths": feats_len}
# a. To device
batch = to_device(batch, device=self.device)
@@ -253,6 +257,141 @@
return results
+# def inference(
+# maxlenratio: float,
+# minlenratio: float,
+# batch_size: int,
+# beam_size: int,
+# ngpu: int,
+# ctc_weight: float,
+# lm_weight: float,
+# penalty: float,
+# log_level: Union[int, str],
+# data_path_and_name_and_type,
+# asr_train_config: Optional[str],
+# asr_model_file: Optional[str],
+# cmvn_file: Optional[str] = None,
+# lm_train_config: Optional[str] = None,
+# lm_file: Optional[str] = None,
+# token_type: Optional[str] = None,
+# key_file: Optional[str] = None,
+# word_lm_train_config: Optional[str] = None,
+# bpemodel: Optional[str] = None,
+# allow_variable_data_keys: bool = False,
+# streaming: bool = False,
+# output_dir: Optional[str] = None,
+# dtype: str = "float32",
+# seed: int = 0,
+# ngram_weight: float = 0.9,
+# nbest: int = 1,
+# num_workers: int = 1,
+# **kwargs,
+# ):
+# assert check_argument_types()
+# if batch_size > 1:
+# raise NotImplementedError("batch decoding is not implemented")
+# if word_lm_train_config is not None:
+# raise NotImplementedError("Word LM is not implemented")
+# if ngpu > 1:
+# raise NotImplementedError("only single GPU decoding is supported")
+#
+# logging.basicConfig(
+# level=log_level,
+# format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+# )
+#
+# if ngpu >= 1 and torch.cuda.is_available():
+# device = "cuda"
+# else:
+# device = "cpu"
+#
+# # 1. Set random-seed
+# set_all_random_seed(seed)
+#
+# # 2. Build speech2text
+# speech2text_kwargs = dict(
+# asr_train_config=asr_train_config,
+# asr_model_file=asr_model_file,
+# cmvn_file=cmvn_file,
+# lm_train_config=lm_train_config,
+# lm_file=lm_file,
+# token_type=token_type,
+# bpemodel=bpemodel,
+# device=device,
+# maxlenratio=maxlenratio,
+# minlenratio=minlenratio,
+# dtype=dtype,
+# beam_size=beam_size,
+# ctc_weight=ctc_weight,
+# lm_weight=lm_weight,
+# ngram_weight=ngram_weight,
+# penalty=penalty,
+# nbest=nbest,
+# streaming=streaming,
+# )
+# logging.info("speech2text_kwargs: {}".format(speech2text_kwargs))
+# speech2text = Speech2Text(**speech2text_kwargs)
+#
+# # 3. Build data-iterator
+# loader = ASRTask.build_streaming_iterator(
+# data_path_and_name_and_type,
+# dtype=dtype,
+# batch_size=batch_size,
+# key_file=key_file,
+# num_workers=num_workers,
+# preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
+# collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
+# allow_variable_data_keys=allow_variable_data_keys,
+# inference=True,
+# )
+#
+# finish_count = 0
+# file_count = 1
+# # 7 .Start for-loop
+# # FIXME(kamo): The output format should be discussed about
+# asr_result_list = []
+# if output_dir is not None:
+# writer = DatadirWriter(output_dir)
+# else:
+# writer = None
+#
+# for keys, batch in loader:
+# assert isinstance(batch, dict), type(batch)
+# assert all(isinstance(s, str) for s in keys), keys
+# _bs = len(next(iter(batch.values())))
+# assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+# #batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+#
+# # N-best list of (text, token, token_int, hyp_object)
+# try:
+# results = speech2text(**batch)
+# except TooShortUttError as e:
+# logging.warning(f"Utterance {keys} {e}")
+# hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+# results = [[" ", ["<space>"], [2], hyp]] * nbest
+#
+# # Only supporting batch_size==1
+# key = keys[0]
+# for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
+# # Create a directory: outdir/{n}best_recog
+# if writer is not None:
+# ibest_writer = writer[f"{n}best_recog"]
+#
+# # Write the result to each file
+# ibest_writer["token"][key] = " ".join(token)
+# ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+# ibest_writer["score"][key] = str(hyp.score)
+#
+# if text is not None:
+# text_postprocessed = postprocess_utils.sentence_postprocess(token)
+# item = {'key': key, 'value': text_postprocessed}
+# asr_result_list.append(item)
+# finish_count += 1
+# asr_utils.print_progress(finish_count / file_count)
+# if writer is not None:
+# ibest_writer["text"][key] = text
+# return asr_result_list
+
def inference(
maxlenratio: float,
minlenratio: float,
@@ -266,7 +405,8 @@
data_path_and_name_and_type,
asr_train_config: Optional[str],
asr_model_file: Optional[str],
- audio_lists: Union[List[Any], bytes] = None,
+ cmvn_file: Optional[str] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
lm_train_config: Optional[str] = None,
lm_file: Optional[str] = None,
token_type: Optional[str] = None,
@@ -281,10 +421,69 @@
ngram_weight: float = 0.9,
nbest: int = 1,
num_workers: int = 1,
- frontend_conf: dict = None,
- fs: Union[dict, int] = 16000,
- lang: Optional[str] = None,
**kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ batch_size=batch_size,
+ beam_size=beam_size,
+ ngpu=ngpu,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ penalty=penalty,
+ log_level=log_level,
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ raw_inputs=raw_inputs,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ key_file=key_file,
+ word_lm_train_config=word_lm_train_config,
+ bpemodel=bpemodel,
+ allow_variable_data_keys=allow_variable_data_keys,
+ streaming=streaming,
+ output_dir=output_dir,
+ dtype=dtype,
+ seed=seed,
+ ngram_weight=ngram_weight,
+ nbest=nbest,
+ num_workers=num_workers,
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+def inference_modelscope(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ # data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ cmvn_file: Optional[str] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ streaming: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
+ **kwargs,
):
assert check_argument_types()
if batch_size > 1:
@@ -293,63 +492,25 @@
raise NotImplementedError("Word LM is not implemented")
if ngpu > 1:
raise NotImplementedError("only single GPU decoding is supported")
-
+
logging.basicConfig(
level=log_level,
format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
)
-
- if ngpu >= 1:
+
+ if ngpu >= 1 and torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
- hop_length: int = 160
- sr: int = 16000
- if isinstance(fs, int):
- sr = fs
- else:
- if 'model_fs' in fs and fs['model_fs'] is not None:
- sr = fs['model_fs']
- # data_path_and_name_and_type for modelscope: (data from audio_lists)
- # ['speech', 'sound', 'am.mvn']
- # data_path_and_name_and_type for funasr:
- # [('/mnt/data/jiangyu.xzy/exp/maas/mvn.1.scp', 'speech', 'kaldi_ark')]
- if isinstance(data_path_and_name_and_type[0], Tuple):
- features_type: str = data_path_and_name_and_type[0][1]
- elif isinstance(data_path_and_name_and_type[0], str):
- features_type: str = data_path_and_name_and_type[1]
- else:
- raise NotImplementedError("unknown features type:{0}".format(data_path_and_name_and_type))
- if features_type != 'sound':
- frontend_conf = None
- flag_modelscope = False
- else:
- flag_modelscope = True
- if frontend_conf is not None:
- if 'hop_length' in frontend_conf:
- hop_length = frontend_conf['hop_length']
-
- finish_count = 0
- file_count = 1
- if flag_modelscope and not isinstance(data_path_and_name_and_type[0], Tuple):
- data_path_and_name_and_type_new = [
- audio_lists, data_path_and_name_and_type[0], data_path_and_name_and_type[1]
- ]
- if isinstance(audio_lists, bytes):
- file_count = 1
- else:
- file_count = len(audio_lists)
- if len(data_path_and_name_and_type) >= 3 and frontend_conf is not None:
- mvn_file = data_path_and_name_and_type[2]
- mvn_data = wav_utils.extract_CMVN_featrures(mvn_file)
- frontend_conf['mvn_data'] = mvn_data
+
# 1. Set random-seed
set_all_random_seed(seed)
-
+
# 2. Build speech2text
speech2text_kwargs = dict(
asr_train_config=asr_train_config,
asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
lm_train_config=lm_train_config,
lm_file=lm_file,
token_type=token_type,
@@ -365,26 +526,19 @@
penalty=penalty,
nbest=nbest,
streaming=streaming,
- frontend_conf=frontend_conf,
)
logging.info("speech2text_kwargs: {}".format(speech2text_kwargs))
speech2text = Speech2Text(**speech2text_kwargs)
-
- # 3. Build data-iterator
- if flag_modelscope:
- loader = ASRTask.build_streaming_iterator_modelscope(
- data_path_and_name_and_type_new,
- dtype=dtype,
- batch_size=batch_size,
- key_file=key_file,
- num_workers=num_workers,
- preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
- collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
- allow_variable_data_keys=allow_variable_data_keys,
- inference=True,
- sample_rate=fs
- )
- else:
+
+ def _forward(data_path_and_name_and_type,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ # 3. Build data-iterator
+ if data_path_and_name_and_type is None and raw_inputs is not None:
+ if isinstance(raw_inputs, torch.Tensor):
+ raw_inputs = raw_inputs.numpy()
+ data_path_and_name_and_type = [raw_inputs, "speech", "waveform"]
loader = ASRTask.build_streaming_iterator(
data_path_and_name_and_type,
dtype=dtype,
@@ -396,52 +550,56 @@
allow_variable_data_keys=allow_variable_data_keys,
inference=True,
)
-
- # 7 .Start for-loop
- # FIXME(kamo): The output format should be discussed about
- asr_result_list = []
- if output_dir is not None:
- writer = DatadirWriter(output_dir)
- else:
- writer = None
-
- for keys, batch in loader:
- assert isinstance(batch, dict), type(batch)
- assert all(isinstance(s, str) for s in keys), keys
- _bs = len(next(iter(batch.values())))
- assert len(keys) == _bs, f"{len(keys)} != {_bs}"
- batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
-
- # N-best list of (text, token, token_int, hyp_object)
- try:
- results = speech2text(**batch)
- except TooShortUttError as e:
- logging.warning(f"Utterance {keys} {e}")
- hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
- results = [[" ", ["<space>"], [2], hyp]] * nbest
-
- # Only supporting batch_size==1
- key = keys[0]
- for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
- # Create a directory: outdir/{n}best_recog
- if writer is not None:
- ibest_writer = writer[f"{n}best_recog"]
-
- # Write the result to each file
- ibest_writer["token"][key] = " ".join(token)
- ibest_writer["token_int"][key] = " ".join(map(str, token_int))
- ibest_writer["score"][key] = str(hyp.score)
-
- if text is not None:
- text_postprocessed = postprocess_utils.sentence_postprocess(token)
- item = {'key': key, 'value': text_postprocessed}
- asr_result_list.append(item)
- finish_count += 1
- asr_utils.print_progress(finish_count / file_count)
+
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ # FIXME(kamo): The output format should be discussed about
+ asr_result_list = []
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path is not None:
+ writer = DatadirWriter(output_path)
+ else:
+ writer = None
+
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ # batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+
+ # N-best list of (text, token, token_int, hyp_object)
+ try:
+ results = speech2text(**batch)
+ except TooShortUttError as e:
+ logging.warning(f"Utterance {keys} {e}")
+ hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+ results = [[" ", ["<space>"], [2], hyp]] * nbest
+
+ # Only supporting batch_size==1
+ key = keys[0]
+ for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
+ # Create a directory: outdir/{n}best_recog
if writer is not None:
- ibest_writer["text"][key] = text
- return asr_result_list
-
+ ibest_writer = writer[f"{n}best_recog"]
+
+ # Write the result to each file
+ ibest_writer["token"][key] = " ".join(token)
+ # ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+ ibest_writer["score"][key] = str(hyp.score)
+
+ if text is not None:
+ text_postprocessed = postprocess_utils.sentence_postprocess(token)
+ item = {'key': key, 'value': text_postprocessed}
+ asr_result_list.append(item)
+ finish_count += 1
+ asr_utils.print_progress(finish_count / file_count)
+ if writer is not None:
+ ibest_writer["text"][key] = text
+ return asr_result_list
+
+ return _forward
def set_parameters(language: str = None,
sample_rate: Union[int, Dict[Any, int]] = None):
@@ -500,10 +658,10 @@
group.add_argument(
"--data_path_and_name_and_type",
type=str2triple_str,
- required=True,
+ required=False,
action="append",
)
- group.add_argument("--audio_lists", type=list, default=None)
+ group.add_argument("--raw_inputs", type=list, default=None)
# example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
group.add_argument("--key_file", type=str_or_none)
group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
@@ -520,6 +678,11 @@
help="ASR model parameter file",
)
group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global cmvn file",
+ )
+ group.add_argument(
"--lm_train_config",
type=str,
help="LM training configuration",
diff --git a/funasr/bin/asr_inference_launch.py b/funasr/bin/asr_inference_launch.py
old mode 100755
new mode 100644
index 84e1422..67a85d2
--- a/funasr/bin/asr_inference_launch.py
+++ b/funasr/bin/asr_inference_launch.py
@@ -76,6 +76,21 @@
group = parser.add_argument_group("The model configuration related")
group.add_argument(
+ "--vad_infer_config",
+ type=str,
+ help="VAD infer configuration",
+ )
+ group.add_argument(
+ "--vad_model_file",
+ type=str,
+ help="VAD model parameter file",
+ )
+ group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global CMVN file",
+ )
+ group.add_argument(
"--asr_train_config",
type=str,
help="ASR training configuration",
@@ -147,7 +162,7 @@
group.add_argument(
"--ctc_weight",
type=float,
- default=0.5,
+ default=0.0,
help="CTC weight in joint decoding",
)
group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight")
@@ -182,26 +197,28 @@
return parser
-def set_parameters(language: str = None,
- sample_rate: Union[int, Dict[Any, int]] = None):
- if language is not None:
- global global_asr_language
- global_asr_language = language
- if sample_rate is not None:
- global global_sample_rate
- global_sample_rate = sample_rate
-
-def inference_launch(mode, **kwargs):
+def inference_launch(**kwargs):
+ if 'mode' in kwargs:
+ mode = kwargs['mode']
+ else:
+ logging.info("Unknown decoding mode.")
+ return None
if mode == "asr":
- from funasr.bin.asr_inference import inference
- return inference(**kwargs)
+ from funasr.bin.asr_inference import inference_modelscope
+ return inference_modelscope(**kwargs)
elif mode == "uniasr":
- from funasr.bin.asr_inference_uniasr import inference
- return inference(**kwargs)
+ from funasr.bin.asr_inference_uniasr import inference_modelscope
+ return inference_modelscope(**kwargs)
elif mode == "paraformer":
- from funasr.bin.asr_inference_paraformer import inference
- return inference(**kwargs)
+ from funasr.bin.asr_inference_paraformer import inference_modelscope
+ return inference_modelscope(**kwargs)
+ elif mode == "paraformer_vad_punc":
+ from funasr.bin.asr_inference_paraformer_vad_punc import inference_modelscope
+ return inference_modelscope(**kwargs)
+ elif mode == "vad":
+ from funasr.bin.vad_inference import inference_modelscope
+ return inference_modelscope(**kwargs)
else:
logging.info("Unknown decoding mode: {}".format(mode))
return None
diff --git a/funasr/bin/asr_inference_paraformer.py b/funasr/bin/asr_inference_paraformer.py
old mode 100755
new mode 100644
index 09c61bc..1a73457
--- a/funasr/bin/asr_inference_paraformer.py
+++ b/funasr/bin/asr_inference_paraformer.py
@@ -36,10 +36,6 @@
from funasr.utils import asr_utils, wav_utils, postprocess_utils
from funasr.models.frontend.wav_frontend import WavFrontend
-from modelscope.utils.logger import get_logger
-
-logger = get_logger()
-
header_colors = '\033[95m'
end_colors = '\033[0m'
@@ -48,6 +44,7 @@
'audio_fs': 16000,
'model_fs': 16000
}
+
class Speech2Text:
"""Speech2Text class
@@ -65,6 +62,7 @@
self,
asr_train_config: Union[Path, str] = None,
asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
lm_train_config: Union[Path, str] = None,
lm_file: Union[Path, str] = None,
token_type: str = None,
@@ -87,13 +85,14 @@
# 1. Build ASR model
scorers = {}
asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, device
+ asr_train_config, asr_model_file, cmvn_file, device
)
- if asr_model.frontend is None and frontend_conf is not None:
- frontend = WavFrontend(**frontend_conf)
- asr_model.frontend = frontend
- # logging.info("asr_model: {}".format(asr_model))
- # logging.info("asr_train_args: {}".format(asr_train_args))
+ frontend = None
+ if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
+ frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+
+ logging.info("asr_model: {}".format(asr_model))
+ logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
@@ -141,8 +140,8 @@
for scorer in scorers.values():
if isinstance(scorer, torch.nn.Module):
scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
- # logging.info(f"Beam_search: {beam_search}")
- # logging.info(f"Decoding device={device}, dtype={dtype}")
+
+ logging.info(f"Decoding device={device}, dtype={dtype}")
# 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
if token_type is None:
@@ -160,22 +159,27 @@
else:
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
- # logging.info(f"Text tokenizer: {tokenizer}")
+ logging.info(f"Text tokenizer: {tokenizer}")
self.asr_model = asr_model
self.asr_train_args = asr_train_args
self.converter = converter
self.tokenizer = tokenizer
- has_lm = lm_weight == 0.0 or lm_file is None
- if ctc_weight == 0.0 and has_lm:
+ is_use_lm = lm_weight != 0.0 and lm_file is not None
+ if ctc_weight == 0.0 and not is_use_lm:
beam_search = None
self.beam_search = beam_search
+ logging.info(f"Beam_search: {self.beam_search}")
self.beam_search_transducer = beam_search_transducer
self.maxlenratio = maxlenratio
self.minlenratio = minlenratio
self.device = device
self.dtype = dtype
self.nbest = nbest
+ self.frontend = frontend
+ self.encoder_downsampling_factor = 1
+ if asr_train_args.encoder_conf["input_layer"] == "conv2d":
+ self.encoder_downsampling_factor = 4
@torch.no_grad()
def __call__(
@@ -195,14 +199,16 @@
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
- # data: (Nsamples,) -> (1, Nsamples)
- # lengths: (1,)
- # if len(speech.size()) < 3:
- # speech = speech.unsqueeze(0).to(getattr(torch, self.dtype))
- # speech_lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
- lfr_factor = max(1, (speech.size()[-1]//80)-1)
-
- batch = {"speech": speech, "speech_lengths": speech_lengths}
+ if self.frontend is not None:
+ feats, feats_len = self.frontend.forward(speech, speech_lengths)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ self.asr_model.frontend = None
+ else:
+ feats = speech
+ feats_len = speech_lengths
+ lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
+ batch = {"speech": feats, "speech_lengths": feats_len}
# a. To device
batch = to_device(batch, device=self.device)
@@ -212,10 +218,11 @@
if isinstance(enc, tuple):
enc = enc[0]
# assert len(enc) == 1, len(enc)
- enc_len_batch_total = torch.sum(enc_len).item()
+ enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
predictor_outs = self.asr_model.calc_predictor(enc, enc_len)
- pre_acoustic_embeds, pre_token_length = predictor_outs[0], predictor_outs[1]
+ pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = predictor_outs[0], predictor_outs[1], \
+ predictor_outs[2], predictor_outs[3]
pre_token_length = pre_token_length.round().long()
decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
@@ -229,7 +236,7 @@
nbest_hyps = self.beam_search(
x=x, am_scores=am_scores, maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
)
-
+
nbest_hyps = nbest_hyps[: self.nbest]
else:
yseq = am_scores.argmax(dim=-1)
@@ -240,33 +247,189 @@
[self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
)
nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
-
+
for hyp in nbest_hyps:
assert isinstance(hyp, (Hypothesis)), type(hyp)
-
+
# remove sos/eos and get results
last_pos = -1
if isinstance(hyp.yseq, list):
token_int = hyp.yseq[1:last_pos]
else:
token_int = hyp.yseq[1:last_pos].tolist()
-
+
# remove blank symbol id, which is assumed to be 0
token_int = list(filter(lambda x: x != 0, token_int))
-
+
# Change integer-ids to tokens
token = self.converter.ids2tokens(token_int)
-
+
if self.tokenizer is not None:
text = self.tokenizer.tokens2text(token)
else:
text = None
-
+
results.append((text, token, token_int, hyp, enc_len_batch_total, lfr_factor))
# assert check_return_type(results)
return results
+
+# def inference(
+# maxlenratio: float,
+# minlenratio: float,
+# batch_size: int,
+# beam_size: int,
+# ngpu: int,
+# ctc_weight: float,
+# lm_weight: float,
+# penalty: float,
+# log_level: Union[int, str],
+# data_path_and_name_and_type,
+# asr_train_config: Optional[str],
+# asr_model_file: Optional[str],
+# cmvn_file: Optional[str] = None,
+# raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+# lm_train_config: Optional[str] = None,
+# lm_file: Optional[str] = None,
+# token_type: Optional[str] = None,
+# key_file: Optional[str] = None,
+# word_lm_train_config: Optional[str] = None,
+# bpemodel: Optional[str] = None,
+# allow_variable_data_keys: bool = False,
+# streaming: bool = False,
+# output_dir: Optional[str] = None,
+# dtype: str = "float32",
+# seed: int = 0,
+# ngram_weight: float = 0.9,
+# nbest: int = 1,
+# num_workers: int = 1,
+# frontend_conf: dict = None,
+# fs: Union[dict, int] = 16000,
+# lang: Optional[str] = None,
+# **kwargs,
+# ):
+# assert check_argument_types()
+#
+# if word_lm_train_config is not None:
+# raise NotImplementedError("Word LM is not implemented")
+# if ngpu > 1:
+# raise NotImplementedError("only single GPU decoding is supported")
+#
+# logging.basicConfig(
+# level=log_level,
+# format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+# )
+#
+# if ngpu >= 1 and torch.cuda.is_available():
+# device = "cuda"
+# else:
+# device = "cpu"
+#
+# # 1. Set random-seed
+# set_all_random_seed(seed)
+#
+# # 2. Build speech2text
+# speech2text_kwargs = dict(
+# asr_train_config=asr_train_config,
+# asr_model_file=asr_model_file,
+# cmvn_file=cmvn_file,
+# lm_train_config=lm_train_config,
+# lm_file=lm_file,
+# token_type=token_type,
+# bpemodel=bpemodel,
+# device=device,
+# maxlenratio=maxlenratio,
+# minlenratio=minlenratio,
+# dtype=dtype,
+# beam_size=beam_size,
+# ctc_weight=ctc_weight,
+# lm_weight=lm_weight,
+# ngram_weight=ngram_weight,
+# penalty=penalty,
+# nbest=nbest,
+# frontend_conf=frontend_conf,
+# )
+# speech2text = Speech2Text(**speech2text_kwargs)
+#
+# # 3. Build data-iterator
+# loader = ASRTask.build_streaming_iterator(
+# data_path_and_name_and_type,
+# dtype=dtype,
+# batch_size=batch_size,
+# key_file=key_file,
+# num_workers=num_workers,
+# preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
+# collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
+# allow_variable_data_keys=allow_variable_data_keys,
+# inference=True,
+# )
+#
+# forward_time_total = 0.0
+# length_total = 0.0
+# finish_count = 0
+# file_count = 1
+# # 7 .Start for-loop
+# # FIXME(kamo): The output format should be discussed about
+# asr_result_list = []
+# if output_dir is not None:
+# writer = DatadirWriter(output_dir)
+# else:
+# writer = None
+#
+# for keys, batch in loader:
+# assert isinstance(batch, dict), type(batch)
+# assert all(isinstance(s, str) for s in keys), keys
+# _bs = len(next(iter(batch.values())))
+# assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+# # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
+#
+# logging.info("decoding, utt_id: {}".format(keys))
+# # N-best list of (text, token, token_int, hyp_object)
+#
+# time_beg = time.time()
+# results = speech2text(**batch)
+# if len(results) < 1:
+# hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+# results = [[" ", ["<space>"], [2], hyp, 10, 6]] * nbest
+# time_end = time.time()
+# forward_time = time_end - time_beg
+# lfr_factor = results[0][-1]
+# length = results[0][-2]
+# forward_time_total += forward_time
+# length_total += length
+# logging.info(
+# "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".
+# format(length, forward_time, 100 * forward_time / (length*lfr_factor)))
+#
+# for batch_id in range(_bs):
+# result = [results[batch_id][:-2]]
+#
+# key = keys[batch_id]
+# for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), result):
+# # Create a directory: outdir/{n}best_recog
+# if writer is not None:
+# ibest_writer = writer[f"{n}best_recog"]
+#
+# # Write the result to each file
+# ibest_writer["token"][key] = " ".join(token)
+# ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+# ibest_writer["score"][key] = str(hyp.score)
+#
+# if text is not None:
+# text_postprocessed = postprocess_utils.sentence_postprocess(token)
+# item = {'key': key, 'value': text_postprocessed}
+# asr_result_list.append(item)
+# finish_count += 1
+# # asr_utils.print_progress(finish_count / file_count)
+# if writer is not None:
+# ibest_writer["text"][key] = text
+#
+# logging.info("decoding, utt: {}, predictions: {}".format(key, text))
+#
+# logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".
+# format(length_total, forward_time_total, 100 * forward_time_total / (length_total*lfr_factor)))
+# return asr_result_list
def inference(
maxlenratio: float,
@@ -281,7 +444,8 @@
data_path_and_name_and_type,
asr_train_config: Optional[str],
asr_model_file: Optional[str],
- audio_lists: Union[List[Any], bytes] = None,
+ cmvn_file: Optional[str] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
lm_train_config: Optional[str] = None,
lm_file: Optional[str] = None,
token_type: Optional[str] = None,
@@ -296,9 +460,70 @@
ngram_weight: float = 0.9,
nbest: int = 1,
num_workers: int = 1,
- frontend_conf: dict = None,
- fs: Union[dict, int] = 16000,
- lang: Optional[str] = None,
+
+ **kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ batch_size=batch_size,
+ beam_size=beam_size,
+ ngpu=ngpu,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ penalty=penalty,
+ log_level=log_level,
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ raw_inputs=raw_inputs,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ key_file=key_file,
+ word_lm_train_config=word_lm_train_config,
+ bpemodel=bpemodel,
+ allow_variable_data_keys=allow_variable_data_keys,
+ streaming=streaming,
+ output_dir=output_dir,
+ dtype=dtype,
+ seed=seed,
+ ngram_weight=ngram_weight,
+ nbest=nbest,
+ num_workers=num_workers,
+
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+
+def inference_modelscope(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ # data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ cmvn_file: Optional[str] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
+ output_dir: Optional[str] = None,
**kwargs,
):
assert check_argument_types()
@@ -313,50 +538,12 @@
format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
)
- if ngpu >= 1:
+ if ngpu >= 1 and torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
- hop_length: int = 160
- sr: int = 16000
- if isinstance(fs, int):
- sr = fs
- else:
- if 'model_fs' in fs and fs['model_fs'] is not None:
- sr = fs['model_fs']
- # data_path_and_name_and_type for modelscope: (data from audio_lists)
- # ['speech', 'sound', 'am.mvn']
- # data_path_and_name_and_type for funasr:
- # [('/mnt/data/jiangyu.xzy/exp/maas/mvn.1.scp', 'speech', 'kaldi_ark')]
- if isinstance(data_path_and_name_and_type[0], Tuple):
- features_type: str = data_path_and_name_and_type[0][1]
- elif isinstance(data_path_and_name_and_type[0], str):
- features_type: str = data_path_and_name_and_type[1]
- else:
- raise NotImplementedError("unknown features type:{0}".format(data_path_and_name_and_type))
- if features_type != 'sound':
- frontend_conf = None
- flag_modelscope = False
- else:
- flag_modelscope = True
- if frontend_conf is not None:
- if 'hop_length' in frontend_conf:
- hop_length = frontend_conf['hop_length']
+ batch_size = 1
- finish_count = 0
- file_count = 1
- if flag_modelscope and not isinstance(data_path_and_name_and_type[0], Tuple):
- data_path_and_name_and_type_new = [
- audio_lists, data_path_and_name_and_type[0], data_path_and_name_and_type[1]
- ]
- if isinstance(audio_lists, bytes):
- file_count = 1
- else:
- file_count = len(audio_lists)
- if len(data_path_and_name_and_type) >= 3 and frontend_conf is not None:
- mvn_file = data_path_and_name_and_type[2]
- mvn_data = wav_utils.extract_CMVN_featrures(mvn_file)
- frontend_conf['mvn_data'] = mvn_data
# 1. Set random-seed
set_all_random_seed(seed)
@@ -364,6 +551,7 @@
speech2text_kwargs = dict(
asr_train_config=asr_train_config,
asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
lm_train_config=lm_train_config,
lm_file=lm_file,
token_type=token_type,
@@ -378,25 +566,19 @@
ngram_weight=ngram_weight,
penalty=penalty,
nbest=nbest,
- frontend_conf=frontend_conf,
)
speech2text = Speech2Text(**speech2text_kwargs)
- # 3. Build data-iterator
- if flag_modelscope:
- loader = ASRTask.build_streaming_iterator_modelscope(
- data_path_and_name_and_type_new,
- dtype=dtype,
- batch_size=batch_size,
- key_file=key_file,
- num_workers=num_workers,
- preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
- collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
- allow_variable_data_keys=allow_variable_data_keys,
- inference=True,
- sample_rate=fs
- )
- else:
+ def _forward(
+ data_path_and_name_and_type,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ # 3. Build data-iterator
+ if data_path_and_name_and_type is None and raw_inputs is not None:
+ if isinstance(raw_inputs, torch.Tensor):
+ raw_inputs = raw_inputs.numpy()
+ data_path_and_name_and_type = [raw_inputs, "speech", "waveform"]
loader = ASRTask.build_streaming_iterator(
data_path_and_name_and_type,
dtype=dtype,
@@ -409,76 +591,75 @@
inference=True,
)
- forward_time_total = 0.0
- length_total = 0.0
- # 7 .Start for-loop
- # FIXME(kamo): The output format should be discussed about
- asr_result_list = []
- if output_dir is not None:
- writer = DatadirWriter(output_dir)
- else:
- writer = None
+ forward_time_total = 0.0
+ length_total = 0.0
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ # FIXME(kamo): The output format should be discussed about
+ asr_result_list = []
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path is not None:
+ writer = DatadirWriter(output_path)
+ else:
+ writer = None
- for keys, batch in loader:
- assert isinstance(batch, dict), type(batch)
- assert all(isinstance(s, str) for s in keys), keys
- _bs = len(next(iter(batch.values())))
- assert len(keys) == _bs, f"{len(keys)} != {_bs}"
- # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
- # logging.info("decoding, utt_id: {}".format(keys))
- # N-best list of (text, token, token_int, hyp_object)
+ logging.info("decoding, utt_id: {}".format(keys))
+ # N-best list of (text, token, token_int, hyp_object)
- time_beg = time.time()
- results = speech2text(**batch)
- time_end = time.time()
- forward_time = time_end - time_beg
- lfr_factor = results[0][-1]
- length = results[0][-2]
- forward_time_total += forward_time
- length_total += length
- logging.info(
- "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".
- format(length, forward_time, 100 * forward_time / (length*lfr_factor)))
-
- for batch_id in range(_bs):
- result = [results[batch_id][:-2]]
-
- key = keys[batch_id]
- for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), result):
- # Create a directory: outdir/{n}best_recog
- if writer is not None:
- ibest_writer = writer[f"{n}best_recog"]
-
- # Write the result to each file
- ibest_writer["token"][key] = " ".join(token)
- ibest_writer["token_int"][key] = " ".join(map(str, token_int))
- ibest_writer["score"][key] = str(hyp.score)
-
- if text is not None:
- text_postprocessed = postprocess_utils.sentence_postprocess(token)
- item = {'key': key, 'value': text_postprocessed}
- asr_result_list.append(item)
- finish_count += 1
- asr_utils.print_progress(finish_count / file_count)
+ time_beg = time.time()
+ results = speech2text(**batch)
+ if len(results) < 1:
+ hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+ results = [[" ", ["<space>"], [2], hyp, 10, 6]] * nbest
+ time_end = time.time()
+ forward_time = time_end - time_beg
+ lfr_factor = results[0][-1]
+ length = results[0][-2]
+ forward_time_total += forward_time
+ length_total += length
+ rtf_cur = "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".format(length, forward_time, 100 * forward_time / (length * lfr_factor))
+ logging.info(rtf_cur)
+
+ for batch_id in range(_bs):
+ result = [results[batch_id][:-2]]
+
+ key = keys[batch_id]
+ for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), result):
+ # Create a directory: outdir/{n}best_recog
if writer is not None:
- ibest_writer["text"][key] = text
-
- logging.info("decoding, utt: {}, predictions: {}".format(key, text))
+ ibest_writer = writer[f"{n}best_recog"]
- logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".
- format(length_total, forward_time_total, 100 * forward_time_total / (length_total*lfr_factor)))
- return asr_result_list
+ # Write the result to each file
+ ibest_writer["token"][key] = " ".join(token)
+ # ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+ ibest_writer["score"][key] = str(hyp.score)
+ ibest_writer["rtf"][key] = rtf_cur
+ if text is not None:
+ text_postprocessed = postprocess_utils.sentence_postprocess(token)
+ item = {'key': key, 'value': text_postprocessed}
+ asr_result_list.append(item)
+ finish_count += 1
+ # asr_utils.print_progress(finish_count / file_count)
+ if writer is not None:
+ ibest_writer["text"][key] = text
-def set_parameters(language: str = None,
- sample_rate: Union[int, Dict[Any, int]] = None):
- if language is not None:
- global global_asr_language
- global_asr_language = language
- if sample_rate is not None:
- global global_sample_rate
- global_sample_rate = sample_rate
+ logging.info("decoding, utt: {}, predictions: {}".format(key, text))
+ rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor))
+ logging.info(rtf_avg)
+ if writer is not None:
+ ibest_writer["rtf"]["rtf_avf"] = rtf_avg
+ return asr_result_list
+
+ return _forward
def get_parser():
@@ -522,7 +703,7 @@
group.add_argument(
"--data_path_and_name_and_type",
type=str2triple_str,
- required=True,
+ required=False,
action="append",
)
group.add_argument("--key_file", type=str_or_none)
@@ -538,6 +719,11 @@
"--asr_model_file",
type=str,
help="ASR model parameter file",
+ )
+ group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global cmvn file",
)
group.add_argument(
"--lm_train_config",
@@ -613,7 +799,7 @@
default=None,
help="",
)
- group.add_argument("--audio_lists", type=list, default=None)
+ group.add_argument("--raw_inputs", type=list, default=None)
# example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
group = parser.add_argument_group("Text converter related")
@@ -647,3 +833,13 @@
if __name__ == "__main__":
main()
+
+ # from modelscope.pipelines import pipeline
+ # from modelscope.utils.constant import Tasks
+ #
+ # inference_16k_pipline = pipeline(
+ # task=Tasks.auto_speech_recognition,
+ # model='damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch')
+ #
+ # rec_result = inference_16k_pipline(audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav')
+ # print(rec_result)
diff --git a/funasr/bin/asr_inference_paraformer_timestamp.py b/funasr/bin/asr_inference_paraformer_timestamp.py
new file mode 100644
index 0000000..3fb8764
--- /dev/null
+++ b/funasr/bin/asr_inference_paraformer_timestamp.py
@@ -0,0 +1,639 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+from typing import Dict
+from typing import Any
+from typing import List
+from unittest import result
+
+import numpy as np
+import torch
+from typeguard import check_argument_types
+
+from funasr.fileio.datadir_writer import DatadirWriter
+from funasr.modules.beam_search.beam_search import BeamSearchPara as BeamSearch
+from funasr.modules.beam_search.beam_search import Hypothesis
+from funasr.modules.scorers.ctc import CTCPrefixScorer
+from funasr.modules.scorers.length_bonus import LengthBonus
+from funasr.modules.subsampling import TooShortUttError
+from funasr.tasks.asr import ASRTaskParaformer as ASRTask
+from funasr.tasks.lm import LMTask
+from funasr.text.build_tokenizer import build_tokenizer
+from funasr.text.token_id_converter import TokenIDConverter
+from funasr.torch_utils.device_funcs import to_device
+from funasr.torch_utils.set_all_random_seed import set_all_random_seed
+from funasr.utils import config_argparse
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+from funasr.utils import asr_utils, wav_utils, postprocess_utils
+from funasr.models.frontend.wav_frontend import WavFrontend
+from funasr.models.predictor.cif import CifPredictorV3
+from funasr.utils.timestamp_tools import time_stamp_lfr6_advance
+
+header_colors = '\033[95m'
+end_colors = '\033[0m'
+
+global_asr_language: str = 'zh-cn'
+global_sample_rate: Union[int, Dict[Any, int]] = {
+ 'audio_fs': 16000,
+ 'model_fs': 16000
+}
+
+class Speech2Text:
+ """Speech2Text class
+
+ Examples:
+ >>> import soundfile
+ >>> speech2text = Speech2Text("asr_config.yml", "asr.pth")
+ >>> audio, rate = soundfile.read("speech.wav")
+ >>> speech2text(audio)
+ [(text, token, token_int, hypothesis object), ...]
+
+ """
+
+ def __init__(
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ maxlenratio: float = 0.0,
+ minlenratio: float = 0.0,
+ dtype: str = "float32",
+ beam_size: int = 20,
+ ctc_weight: float = 0.5,
+ lm_weight: float = 1.0,
+ ngram_weight: float = 0.9,
+ penalty: float = 0.0,
+ nbest: int = 1,
+ frontend_conf: dict = None,
+ time_stamp_writer: bool = False,
+ **kwargs,
+ ):
+ assert check_argument_types()
+
+ # 1. Build ASR model
+ scorers = {}
+ asr_model, asr_train_args = ASRTask.build_model_from_file(
+ asr_train_config, asr_model_file, cmvn_file, device
+ )
+ frontend = None
+ if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
+ frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+
+ logging.info("asr_model: {}".format(asr_model))
+ logging.info("asr_train_args: {}".format(asr_train_args))
+ asr_model.to(dtype=getattr(torch, dtype)).eval()
+
+ ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
+ token_list = asr_model.token_list
+ scorers.update(
+ ctc=ctc,
+ length_bonus=LengthBonus(len(token_list)),
+ )
+
+ # 2. Build Language model
+ if lm_train_config is not None:
+ lm, lm_train_args = LMTask.build_model_from_file(
+ lm_train_config, lm_file, device
+ )
+ scorers["lm"] = lm.lm
+
+ # 3. Build ngram model
+ # ngram is not supported now
+ ngram = None
+ scorers["ngram"] = ngram
+
+ # 4. Build BeamSearch object
+ # transducer is not supported now
+ beam_search_transducer = None
+
+ weights = dict(
+ decoder=1.0 - ctc_weight,
+ ctc=ctc_weight,
+ lm=lm_weight,
+ ngram=ngram_weight,
+ length_bonus=penalty,
+ )
+ beam_search = BeamSearch(
+ beam_size=beam_size,
+ weights=weights,
+ scorers=scorers,
+ sos=asr_model.sos,
+ eos=asr_model.eos,
+ vocab_size=len(token_list),
+ token_list=token_list,
+ pre_beam_score_key=None if ctc_weight == 1.0 else "full",
+ )
+
+ beam_search.to(device=device, dtype=getattr(torch, dtype)).eval()
+ for scorer in scorers.values():
+ if isinstance(scorer, torch.nn.Module):
+ scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
+
+ logging.info(f"Decoding device={device}, dtype={dtype}")
+
+ # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
+ if token_type is None:
+ token_type = asr_train_args.token_type
+ if bpemodel is None:
+ bpemodel = asr_train_args.bpemodel
+
+ if token_type is None:
+ tokenizer = None
+ elif token_type == "bpe":
+ if bpemodel is not None:
+ tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel)
+ else:
+ tokenizer = None
+ else:
+ tokenizer = build_tokenizer(token_type=token_type)
+ converter = TokenIDConverter(token_list=token_list)
+ logging.info(f"Text tokenizer: {tokenizer}")
+
+ self.asr_model = asr_model
+ self.asr_train_args = asr_train_args
+ self.converter = converter
+ self.tokenizer = tokenizer
+ is_use_lm = lm_weight != 0.0 and lm_file is not None
+ if ctc_weight == 0.0 and not is_use_lm:
+ beam_search = None
+ self.beam_search = beam_search
+ logging.info(f"Beam_search: {self.beam_search}")
+ self.beam_search_transducer = beam_search_transducer
+ self.maxlenratio = maxlenratio
+ self.minlenratio = minlenratio
+ self.device = device
+ self.dtype = dtype
+ self.nbest = nbest
+ self.frontend = frontend
+ self.encoder_downsampling_factor = 1
+ if asr_train_args.encoder_conf["input_layer"] == "conv2d":
+ self.encoder_downsampling_factor = 4
+ self.time_stamp_writer = time_stamp_writer
+
+
+ @torch.no_grad()
+ def __call__(
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
+ ):
+ """Inference
+
+ Args:
+ speech: Input speech data
+ Returns:
+ text, token, token_int, hyp
+
+ """
+ assert check_argument_types()
+
+ # Input as audio signal
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ if self.frontend is not None:
+ feats, feats_len = self.frontend.forward(speech, speech_lengths)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ self.asr_model.frontend = None
+ else:
+ feats = speech
+ feats_len = speech_lengths
+ lfr_factor = max(1, (feats.size()[-1]//80)-1)
+ batch = {"speech": feats, "speech_lengths": feats_len}
+
+ # a. To device
+ batch = to_device(batch, device=self.device)
+
+ # b. Forward Encoder
+ enc, enc_len = self.asr_model.encode(**batch)
+ if isinstance(enc, tuple):
+ enc = enc[0]
+ # assert len(enc) == 1, len(enc)
+ enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
+
+ predictor_outs = self.asr_model.calc_predictor(enc, enc_len)
+ pre_acoustic_embeds, pre_token_length, _, _ = predictor_outs[0], predictor_outs[1], predictor_outs[2], predictor_outs[3]
+ pre_token_length = pre_token_length.round().long()
+
+ if isinstance(self.asr_model.predictor, CifPredictorV3) and self.time_stamp_writer:
+ ds_alphas, ds_cif_peak, us_alphas, us_cif_peak = self.asr_model.calc_predictor_timestamp(enc, enc_len, pre_token_length)
+ timestamp = (ds_alphas, ds_cif_peak, us_alphas, us_cif_peak)
+ else:
+ timestamp = None
+
+ decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
+ decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
+
+ results = []
+ b, n, d = decoder_out.size()
+ for i in range(b):
+ x = enc[i, :enc_len[i], :]
+ am_scores = decoder_out[i, :pre_token_length[i], :]
+ if self.beam_search is not None:
+ nbest_hyps = self.beam_search(
+ x=x, am_scores=am_scores, maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
+ )
+
+ nbest_hyps = nbest_hyps[: self.nbest]
+ else:
+ yseq = am_scores.argmax(dim=-1)
+ score = am_scores.max(dim=-1)[0]
+ score = torch.sum(score, dim=-1)
+ # pad with mask tokens to ensure compatibility with sos/eos tokens
+ yseq = torch.tensor(
+ [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
+ )
+ nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
+
+ for hyp in nbest_hyps:
+ assert isinstance(hyp, (Hypothesis)), type(hyp)
+
+ # remove sos/eos and get results
+ last_pos = -1
+ if isinstance(hyp.yseq, list):
+ token_int = hyp.yseq[1:last_pos]
+ else:
+ token_int = hyp.yseq[1:last_pos].tolist()
+
+ # remove blank symbol id, which is assumed to be 0
+ token_int = list(filter(lambda x: x != 0, token_int))
+
+ # Change integer-ids to tokens
+ token = self.converter.ids2tokens(token_int)
+
+ if self.tokenizer is not None:
+ text = self.tokenizer.tokens2text(token)
+ else:
+ text = None
+
+ if timestamp is not None:
+ tst = [_tst[i] for _tst in timestamp] # timestamp related tensors
+ results.append((text, token, token_int, hyp, tst, enc_len_batch_total, lfr_factor))
+ else:
+ results.append((text, token, token_int, hyp, None, enc_len_batch_total, lfr_factor))
+
+ return results
+
+
+def inference(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ cmvn_file: Optional[str] = None,
+ audio_lists: Union[List[Any], bytes] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ streaming: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
+ frontend_conf: dict = None,
+ fs: Union[dict, int] = 16000,
+ lang: Optional[str] = None,
+ time_stamp_writer: bool = False,
+ **kwargs,
+):
+ assert check_argument_types()
+
+ if word_lm_train_config is not None:
+ raise NotImplementedError("Word LM is not implemented")
+ if ngpu > 1:
+ raise NotImplementedError("only single GPU decoding is supported")
+
+ logging.basicConfig(
+ level=log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if ngpu >= 1 and torch.cuda.is_available():
+ device = "cuda"
+ else:
+ device = "cpu"
+
+ # 1. Set random-seed
+ set_all_random_seed(seed)
+
+ # 2. Build speech2text
+ speech2text_kwargs = dict(
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ bpemodel=bpemodel,
+ device=device,
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ dtype=dtype,
+ beam_size=beam_size,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ ngram_weight=ngram_weight,
+ penalty=penalty,
+ nbest=nbest,
+ frontend_conf=frontend_conf,
+ time_stamp_writer=time_stamp_writer,
+ )
+ speech2text = Speech2Text(**speech2text_kwargs)
+
+ # 3. Build data-iterator
+ loader = ASRTask.build_streaming_iterator(
+ data_path_and_name_and_type,
+ dtype=dtype,
+ batch_size=batch_size,
+ key_file=key_file,
+ num_workers=num_workers,
+ preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
+ collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
+ allow_variable_data_keys=allow_variable_data_keys,
+ inference=True,
+ )
+
+ forward_time_total = 0.0
+ length_total = 0.0
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ # FIXME(kamo): The output format should be discussed about
+ asr_result_list = []
+ if output_dir is not None:
+ writer = DatadirWriter(output_dir)
+ else:
+ writer = None
+
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
+
+ logging.info("decoding, utt_id: {}".format(keys))
+ # N-best list of (text, token, token_int, hyp_object)
+
+ time_beg = time.time()
+ results = speech2text(**batch)
+ if len(results) < 1:
+ hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+ results = [[" ", ["<space>"], [2], hyp, 10, 6]] * nbest
+ time_end = time.time()
+ forward_time = time_end - time_beg
+ lfr_factor = results[0][-1]
+ length = results[0][-2]
+ forward_time_total += forward_time
+ length_total += length
+ logging.info(
+ "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".
+ format(length, forward_time, 100 * forward_time / (length*lfr_factor)))
+
+ for batch_id in range(_bs):
+ result = [results[batch_id][:5]]
+
+ key = keys[batch_id]
+ for n, (text, token, token_int, hyp, tst) in zip(range(1, nbest + 1), result):
+ # Create a directory: outdir/{n}best_recog
+ if writer is not None:
+ ibest_writer = writer[f"{n}best_recog"]
+
+ # Write the result to each file
+ ibest_writer["token"][key] = " ".join(token)
+ ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+ ibest_writer["score"][key] = str(hyp.score)
+
+ if text is not None:
+ text_postprocessed = postprocess_utils.sentence_postprocess(token)
+ item = {'key': key, 'value': text_postprocessed}
+ asr_result_list.append(item)
+ finish_count += 1
+ # asr_utils.print_progress(finish_count / file_count)
+ if writer is not None:
+ ibest_writer["text"][key] = text
+
+ if tst is not None:
+ timestamp_res = time_stamp_lfr6_advance(tst, text)
+
+
+ logging.info("decoding, utt: {}, predictions: {}".format(key, text))
+
+ logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".
+ format(length_total, forward_time_total, 100 * forward_time_total / (length_total*lfr_factor)))
+ return asr_result_list
+
+
+def set_parameters(language: str = None,
+ sample_rate: Union[int, Dict[Any, int]] = None):
+ if language is not None:
+ global global_asr_language
+ global_asr_language = language
+ if sample_rate is not None:
+ global global_sample_rate
+ global_sample_rate = sample_rate
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="ASR Decoding",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ # Note(kamo): Use '_' instead of '-' as separator.
+ # '-' is confusing if written in yaml.
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=True)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=True,
+ action="append",
+ )
+ group.add_argument("--key_file", type=str_or_none)
+ group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument(
+ "--asr_train_config",
+ type=str,
+ help="ASR training configuration",
+ )
+ group.add_argument(
+ "--asr_model_file",
+ type=str,
+ help="ASR model parameter file",
+ )
+ group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global cmvn file",
+ )
+ group.add_argument(
+ "--lm_train_config",
+ type=str,
+ help="LM training configuration",
+ )
+ group.add_argument(
+ "--lm_file",
+ type=str,
+ help="LM parameter file",
+ )
+ group.add_argument(
+ "--word_lm_train_config",
+ type=str,
+ help="Word LM training configuration",
+ )
+ group.add_argument(
+ "--word_lm_file",
+ type=str,
+ help="Word LM parameter file",
+ )
+ group.add_argument(
+ "--ngram_file",
+ type=str,
+ help="N-gram parameter file",
+ )
+ group.add_argument(
+ "--model_tag",
+ type=str,
+ help="Pretrained model tag. If specify this option, *_train_config and "
+ "*_file will be overwritten",
+ )
+
+ group = parser.add_argument_group("Beam-search related")
+ group.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+ group.add_argument("--nbest", type=int, default=1, help="Output N-best hypotheses")
+ group.add_argument("--beam_size", type=int, default=20, help="Beam size")
+ group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty")
+ group.add_argument(
+ "--maxlenratio",
+ type=float,
+ default=0.0,
+ help="Input length ratio to obtain max output length. "
+ "If maxlenratio=0.0 (default), it uses a end-detect "
+ "function "
+ "to automatically find maximum hypothesis lengths."
+ "If maxlenratio<0.0, its absolute value is interpreted"
+ "as a constant max output length",
+ )
+ group.add_argument(
+ "--minlenratio",
+ type=float,
+ default=0.0,
+ help="Input length ratio to obtain min output length",
+ )
+ group.add_argument(
+ "--ctc_weight",
+ type=float,
+ default=0.5,
+ help="CTC weight in joint decoding",
+ )
+ group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight")
+ group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight")
+ group.add_argument("--streaming", type=str2bool, default=False)
+
+ group.add_argument(
+ "--frontend_conf",
+ default=None,
+ help="",
+ )
+ group.add_argument("--audio_lists", type=list, default=None)
+ # example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
+
+ group = parser.add_argument_group("Text converter related")
+ group.add_argument(
+ "--token_type",
+ type=str_or_none,
+ default=None,
+ choices=["char", "bpe", None],
+ help="The token type for ASR model. "
+ "If not given, refers from the training args",
+ )
+ group.add_argument(
+ "--bpemodel",
+ type=str_or_none,
+ default=None,
+ help="The model path of sentencepiece. "
+ "If not given, refers from the training args",
+ )
+ group.add_argument(
+ "--time_stamp_writer",
+ type=str2bool,
+ default=False,
+ )
+ return parser
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+ inference(**kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/bin/asr_inference_paraformer_vad_punc.py b/funasr/bin/asr_inference_paraformer_vad_punc.py
new file mode 100644
index 0000000..265e054
--- /dev/null
+++ b/funasr/bin/asr_inference_paraformer_vad_punc.py
@@ -0,0 +1,1134 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+from typing import Dict
+from typing import Any
+from typing import List
+import math
+import numpy as np
+import torch
+from typeguard import check_argument_types
+
+from funasr.fileio.datadir_writer import DatadirWriter
+from funasr.modules.beam_search.beam_search import BeamSearchPara as BeamSearch
+from funasr.modules.beam_search.beam_search import Hypothesis
+from funasr.modules.scorers.ctc import CTCPrefixScorer
+from funasr.modules.scorers.length_bonus import LengthBonus
+from funasr.modules.subsampling import TooShortUttError
+from funasr.tasks.asr import ASRTaskParaformer as ASRTask
+from funasr.tasks.lm import LMTask
+from funasr.text.build_tokenizer import build_tokenizer
+from funasr.text.token_id_converter import TokenIDConverter
+from funasr.torch_utils.device_funcs import to_device
+from funasr.torch_utils.set_all_random_seed import set_all_random_seed
+from funasr.utils import config_argparse
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+from funasr.utils import asr_utils, wav_utils, postprocess_utils
+from funasr.models.frontend.wav_frontend import WavFrontend
+from funasr.tasks.vad import VADTask
+from funasr.utils.timestamp_tools import time_stamp_lfr6
+from funasr.tasks.punctuation import PunctuationTask
+from funasr.torch_utils.forward_adaptor import ForwardAdaptor
+from funasr.datasets.preprocessor import CommonPreprocessor
+from funasr.punctuation.text_preprocessor import split_words, split_to_mini_sentence
+
+header_colors = '\033[95m'
+end_colors = '\033[0m'
+
+global_asr_language: str = 'zh-cn'
+global_sample_rate: Union[int, Dict[Any, int]] = {
+ 'audio_fs': 16000,
+ 'model_fs': 16000
+}
+
+class Speech2Text:
+ """Speech2Text class
+
+ Examples:
+ >>> import soundfile
+ >>> speech2text = Speech2Text("asr_config.yml", "asr.pth")
+ >>> audio, rate = soundfile.read("speech.wav")
+ >>> speech2text(audio)
+ [(text, token, token_int, hypothesis object), ...]
+
+ """
+
+ def __init__(
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ maxlenratio: float = 0.0,
+ minlenratio: float = 0.0,
+ dtype: str = "float32",
+ beam_size: int = 20,
+ ctc_weight: float = 0.5,
+ lm_weight: float = 1.0,
+ ngram_weight: float = 0.9,
+ penalty: float = 0.0,
+ nbest: int = 1,
+ frontend_conf: dict = None,
+ **kwargs,
+ ):
+ assert check_argument_types()
+
+ # 1. Build ASR model
+ scorers = {}
+ asr_model, asr_train_args = ASRTask.build_model_from_file(
+ asr_train_config, asr_model_file, cmvn_file=cmvn_file, device=device
+ )
+ frontend = None
+ if asr_model.frontend is not None and asr_train_args.frontend_conf is not None:
+ frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+
+ # logging.info("asr_model: {}".format(asr_model))
+ # logging.info("asr_train_args: {}".format(asr_train_args))
+ asr_model.to(dtype=getattr(torch, dtype)).eval()
+
+ ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
+ token_list = asr_model.token_list
+ scorers.update(
+ ctc=ctc,
+ length_bonus=LengthBonus(len(token_list)),
+ )
+
+ # 2. Build Language model
+ if lm_train_config is not None:
+ lm, lm_train_args = LMTask.build_model_from_file(
+ lm_train_config, lm_file, device
+ )
+ scorers["lm"] = lm.lm
+
+ # 3. Build ngram model
+ # ngram is not supported now
+ ngram = None
+ scorers["ngram"] = ngram
+
+ # 4. Build BeamSearch object
+ # transducer is not supported now
+ beam_search_transducer = None
+
+ weights = dict(
+ decoder=1.0 - ctc_weight,
+ ctc=ctc_weight,
+ lm=lm_weight,
+ ngram=ngram_weight,
+ length_bonus=penalty,
+ )
+ beam_search = BeamSearch(
+ beam_size=beam_size,
+ weights=weights,
+ scorers=scorers,
+ sos=asr_model.sos,
+ eos=asr_model.eos,
+ vocab_size=len(token_list),
+ token_list=token_list,
+ pre_beam_score_key=None if ctc_weight == 1.0 else "full",
+ )
+
+ beam_search.to(device=device, dtype=getattr(torch, dtype)).eval()
+ for scorer in scorers.values():
+ if isinstance(scorer, torch.nn.Module):
+ scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
+
+ logging.info(f"Decoding device={device}, dtype={dtype}")
+
+ # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
+ if token_type is None:
+ token_type = asr_train_args.token_type
+ if bpemodel is None:
+ bpemodel = asr_train_args.bpemodel
+
+ if token_type is None:
+ tokenizer = None
+ elif token_type == "bpe":
+ if bpemodel is not None:
+ tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel)
+ else:
+ tokenizer = None
+ else:
+ tokenizer = build_tokenizer(token_type=token_type)
+ converter = TokenIDConverter(token_list=token_list)
+ logging.info(f"Text tokenizer: {tokenizer}")
+
+ self.asr_model = asr_model
+ self.asr_train_args = asr_train_args
+ self.converter = converter
+ self.tokenizer = tokenizer
+ is_use_lm = lm_weight != 0.0 and lm_file is not None
+ if ctc_weight == 0.0 and not is_use_lm:
+ beam_search = None
+ self.beam_search = beam_search
+ logging.info(f"Beam_search: {self.beam_search}")
+ self.beam_search_transducer = beam_search_transducer
+ self.maxlenratio = maxlenratio
+ self.minlenratio = minlenratio
+ self.device = device
+ self.dtype = dtype
+ self.nbest = nbest
+ self.frontend = frontend
+ self.encoder_downsampling_factor = 1
+ if asr_train_args.encoder_conf["input_layer"] == "conv2d":
+ self.encoder_downsampling_factor = 4
+
+
+
+ @torch.no_grad()
+ def __call__(
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None, begin_time: int = 0, end_time: int = None,
+ ):
+ """Inference
+
+ Args:
+ speech: Input speech data
+ Returns:
+ text, token, token_int, hyp
+
+ """
+ assert check_argument_types()
+
+ # Input as audio signal
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ if self.frontend is not None:
+ # feats, feats_len = self.frontend.forward(speech, speech_lengths)
+ # fbanks, fbanks_len = self.frontend.forward_fbank(speech, speech_lengths)
+ feats, feats_len = self.frontend.forward_lfr_cmvn(speech, speech_lengths)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ self.asr_model.frontend = None
+ else:
+ feats = speech
+ feats_len = speech_lengths
+ lfr_factor = max(1, (feats.size()[-1]//80)-1)
+ batch = {"speech": feats, "speech_lengths": feats_len}
+
+ # a. To device
+ batch = to_device(batch, device=self.device)
+
+ # b. Forward Encoder
+ enc, enc_len = self.asr_model.encode(**batch)
+ if isinstance(enc, tuple):
+ enc = enc[0]
+ # assert len(enc) == 1, len(enc)
+ enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
+
+ predictor_outs = self.asr_model.calc_predictor(enc, enc_len)
+ pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = predictor_outs[0], predictor_outs[1], predictor_outs[2], predictor_outs[3]
+ pre_token_length = pre_token_length.round().long()
+ decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
+ decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
+
+ results = []
+ b, n, d = decoder_out.size()
+ for i in range(b):
+ x = enc[i, :enc_len[i], :]
+ am_scores = decoder_out[i, :pre_token_length[i], :]
+ if self.beam_search is not None:
+ nbest_hyps = self.beam_search(
+ x=x, am_scores=am_scores, maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
+ )
+
+ nbest_hyps = nbest_hyps[: self.nbest]
+ else:
+ yseq = am_scores.argmax(dim=-1)
+ score = am_scores.max(dim=-1)[0]
+ score = torch.sum(score, dim=-1)
+ # pad with mask tokens to ensure compatibility with sos/eos tokens
+ yseq = torch.tensor(
+ [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
+ )
+ nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
+
+ for hyp in nbest_hyps:
+ assert isinstance(hyp, (Hypothesis)), type(hyp)
+
+ # remove sos/eos and get results
+ last_pos = -1
+ if isinstance(hyp.yseq, list):
+ token_int = hyp.yseq[1:last_pos]
+ else:
+ token_int = hyp.yseq[1:last_pos].tolist()
+
+ # remove blank symbol id, which is assumed to be 0
+ token_int = list(filter(lambda x: x != 0 and x != 2, token_int))
+
+ # Change integer-ids to tokens
+ token = self.converter.ids2tokens(token_int)
+
+ if self.tokenizer is not None:
+ text = self.tokenizer.tokens2text(token)
+ else:
+ text = None
+
+ time_stamp = time_stamp_lfr6(alphas[i:i+1,], enc_len[i:i+1,], token, begin_time, end_time)
+
+ results.append((text, token, token_int, time_stamp, enc_len_batch_total, lfr_factor))
+
+ # assert check_return_type(results)
+ return results
+
+class Speech2VadSegment:
+ """Speech2VadSegment class
+
+ Examples:
+ >>> import soundfile
+ >>> speech2segment = Speech2VadSegment("vad_config.yml", "vad.pt")
+ >>> audio, rate = soundfile.read("speech.wav")
+ >>> speech2segment(audio)
+ [[10, 230], [245, 450], ...]
+
+ """
+
+ def __init__(
+ self,
+ vad_infer_config: Union[Path, str] = None,
+ vad_model_file: Union[Path, str] = None,
+ vad_cmvn_file: Union[Path, str] = None,
+ device: str = "cpu",
+ batch_size: int = 1,
+ dtype: str = "float32",
+ **kwargs,
+ ):
+ assert check_argument_types()
+
+ # 1. Build vad model
+ vad_model, vad_infer_args = VADTask.build_model_from_file(
+ vad_infer_config, vad_model_file, device
+ )
+ frontend = None
+ if vad_infer_args.frontend is not None:
+ frontend = WavFrontend(cmvn_file=vad_cmvn_file, **vad_infer_args.frontend_conf)
+
+ # logging.info("vad_model: {}".format(vad_model))
+ # logging.info("vad_infer_args: {}".format(vad_infer_args))
+ vad_model.to(dtype=getattr(torch, dtype)).eval()
+
+ self.vad_model = vad_model
+ self.vad_infer_args = vad_infer_args
+ self.device = device
+ self.dtype = dtype
+ self.frontend = frontend
+
+ @torch.no_grad()
+ def __call__(
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
+ ) -> List[List[int]]:
+ """Inference
+
+ Args:
+ speech: Input speech data
+ Returns:
+ text, token, token_int, hyp
+
+ """
+ assert check_argument_types()
+
+ # Input as audio signal
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ if self.frontend is not None:
+ self.frontend.filter_length_max = math.inf
+ fbanks, fbanks_len = self.frontend.forward_fbank(speech, speech_lengths)
+ feats, feats_len = self.frontend.forward_lfr_cmvn(fbanks, fbanks_len)
+ fbanks = to_device(fbanks, device=self.device)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ else:
+ raise Exception("Need to extract feats first, please configure frontend configuration")
+ batch = {"feats": feats, "feats_lengths": feats_len, "waveform": speech}
+
+ # a. To device
+ batch = to_device(batch, device=self.device)
+
+ # b. Forward Encoder
+ segments = self.vad_model(**batch)
+
+ return fbanks, segments
+
+
+# def inference(
+# maxlenratio: float,
+# minlenratio: float,
+# batch_size: int,
+# beam_size: int,
+# ngpu: int,
+# ctc_weight: float,
+# lm_weight: float,
+# penalty: float,
+# log_level: Union[int, str],
+# data_path_and_name_and_type,
+# asr_train_config: Optional[str],
+# asr_model_file: Optional[str],
+# cmvn_file: Optional[str] = None,
+# raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+# lm_train_config: Optional[str] = None,
+# lm_file: Optional[str] = None,
+# token_type: Optional[str] = None,
+# key_file: Optional[str] = None,
+# word_lm_train_config: Optional[str] = None,
+# bpemodel: Optional[str] = None,
+# allow_variable_data_keys: bool = False,
+# streaming: bool = False,
+# output_dir: Optional[str] = None,
+# dtype: str = "float32",
+# seed: int = 0,
+# ngram_weight: float = 0.9,
+# nbest: int = 1,
+# num_workers: int = 1,
+# vad_infer_config: Optional[str] = None,
+# vad_model_file: Optional[str] = None,
+# vad_cmvn_file: Optional[str] = None,
+# time_stamp_writer: bool = False,
+# punc_infer_config: Optional[str] = None,
+# punc_model_file: Optional[str] = None,
+# **kwargs,
+# ):
+# assert check_argument_types()
+#
+# if word_lm_train_config is not None:
+# raise NotImplementedError("Word LM is not implemented")
+# if ngpu > 1:
+# raise NotImplementedError("only single GPU decoding is supported")
+#
+# logging.basicConfig(
+# level=log_level,
+# format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+# )
+#
+# if ngpu >= 1 and torch.cuda.is_available():
+# device = "cuda"
+# else:
+# device = "cpu"
+#
+# # 1. Set random-seed
+# set_all_random_seed(seed)
+#
+# # 2. Build speech2vadsegment
+# speech2vadsegment_kwargs = dict(
+# vad_infer_config=vad_infer_config,
+# vad_model_file=vad_model_file,
+# vad_cmvn_file=vad_cmvn_file,
+# device=device,
+# dtype=dtype,
+# )
+# # logging.info("speech2vadsegment_kwargs: {}".format(speech2vadsegment_kwargs))
+# speech2vadsegment = Speech2VadSegment(**speech2vadsegment_kwargs)
+#
+# # 3. Build speech2text
+# speech2text_kwargs = dict(
+# asr_train_config=asr_train_config,
+# asr_model_file=asr_model_file,
+# cmvn_file=cmvn_file,
+# lm_train_config=lm_train_config,
+# lm_file=lm_file,
+# token_type=token_type,
+# bpemodel=bpemodel,
+# device=device,
+# maxlenratio=maxlenratio,
+# minlenratio=minlenratio,
+# dtype=dtype,
+# beam_size=beam_size,
+# ctc_weight=ctc_weight,
+# lm_weight=lm_weight,
+# ngram_weight=ngram_weight,
+# penalty=penalty,
+# nbest=nbest,
+# frontend_conf=frontend_conf,
+# )
+# speech2text = Speech2Text(**speech2text_kwargs)
+#
+# text2punc = Text2Punc(punc_infer_config, punc_model_file, device=device, dtype=dtype)
+#
+# # 3. Build data-iterator
+# loader = ASRTask.build_streaming_iterator(
+# data_path_and_name_and_type,
+# dtype=dtype,
+# batch_size=1,
+# key_file=key_file,
+# num_workers=num_workers,
+# preprocess_fn=VADTask.build_preprocess_fn(speech2vadsegment.vad_infer_args, False),
+# collate_fn=VADTask.build_collate_fn(speech2vadsegment.vad_infer_args, False),
+# allow_variable_data_keys=allow_variable_data_keys,
+# inference=True,
+# )
+#
+# forward_time_total = 0.0
+# length_total = 0.0
+# finish_count = 0
+# file_count = 1
+# # 7 .Start for-loop
+# asr_result_list = []
+# if output_dir is not None:
+# writer = DatadirWriter(output_dir)
+# else:
+# writer = None
+#
+# for keys, batch in loader:
+# assert isinstance(batch, dict), type(batch)
+# assert all(isinstance(s, str) for s in keys), keys
+# _bs = len(next(iter(batch.values())))
+# assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+# # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
+#
+# logging.info("decoding, utt_id: {}".format(keys))
+# # N-best list of (text, token, token_int, hyp_object)
+# time_beg = time.time()
+# vad_results = speech2vadsegment(**batch)
+# time_end = time.time()
+# fbanks, vadsegments = vad_results[0], vad_results[1]
+# for i, segments in enumerate(vadsegments):
+# result_segments = [["", [], [], ]]
+# for j, segment_idx in enumerate(segments):
+# bed_idx, end_idx = int(segment_idx[0]/10), int(segment_idx[1]/10)
+# segment = fbanks[:, bed_idx:end_idx, :].to(device)
+# speech_lengths = torch.Tensor([end_idx-bed_idx]).int().to(device)
+# batch = {"speech": segment, "speech_lengths": speech_lengths, "begin_time": vadsegments[i][j][0], "end_time": vadsegments[i][j][1]}
+# results = speech2text(**batch)
+# if len(results) < 1:
+# hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+# results = [[" ", ["<space>"], [2], 10, 6]] * nbest
+# time_end = time.time()
+# forward_time = time_end - time_beg
+# lfr_factor = results[0][-1]
+# length = results[0][-2]
+# forward_time_total += forward_time
+# length_total += length
+# logging.info(
+# "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".
+# format(length, forward_time, 100 * forward_time / (length*lfr_factor)))
+# result_cur = [results[0][:-2]]
+# if j == 0:
+# result_segments = result_cur
+# else:
+# result_segments = [[result_segments[0][i] + result_cur[0][i] for i in range(len(result_cur[0]))]]
+#
+# key = keys[0]
+# result = result_segments[0]
+# text, token, token_int, time_stamp = result
+#
+# # Create a directory: outdir/{n}best_recog
+# if writer is not None:
+# ibest_writer = writer[f"1best_recog"]
+#
+# # Write the result to each file
+# ibest_writer["token"][key] = " ".join(token)
+# ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+#
+# if text is not None:
+# postprocessed_result = postprocess_utils.sentence_postprocess(token, time_stamp)
+# if len(postprocessed_result) == 3:
+# text_postprocessed, time_stamp_postprocessed, word_lists = postprocessed_result[0], postprocessed_result[1], postprocessed_result[2]
+# text_postprocessed_punc, punc_id_list = text2punc(word_lists, 20)
+# text_postprocessed_punc_time_stamp = "predictions: {} time_stamp: {}".format(text_postprocessed_punc, time_stamp_postprocessed)
+# else:
+# text_postprocessed = postprocessed_result
+# time_stamp_postprocessed = None
+# word_lists = None
+# text_postprocessed_punc_time_stamp = None
+# punc_id_list = None
+#
+# item = {'key': key, 'value': text_postprocessed_punc_time_stamp, 'text': text_postprocessed, 'time_stamp': time_stamp_postprocessed, 'punc': punc_id_list}
+# asr_result_list.append(item)
+# finish_count += 1
+# # asr_utils.print_progress(finish_count / file_count)
+# if writer is not None:
+# ibest_writer["text"][key] = text_postprocessed
+# if time_stamp_writer and time_stamp_postprocessed is not None:
+# ibest_writer["time_stamp"][key] = " ".join(["-".join(map(str, ts)) for ts in time_stamp_postprocessed])
+#
+# logging.info("decoding, utt: {}, predictions: {}, time_stamp: {}".format(key, text_postprocessed_punc, time_stamp_postprocessed))
+#
+# logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".
+# format(length_total, forward_time_total, 100 * forward_time_total / (length_total*lfr_factor)))
+# return asr_result_list
+
+def inference(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ cmvn_file: Optional[str] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ streaming: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
+ vad_infer_config: Optional[str] = None,
+ vad_model_file: Optional[str] = None,
+ vad_cmvn_file: Optional[str] = None,
+ time_stamp_writer: bool = False,
+ punc_infer_config: Optional[str] = None,
+ punc_model_file: Optional[str] = None,
+ **kwargs,
+):
+
+ inference_pipeline = inference_modelscope(
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ batch_size=batch_size,
+ beam_size=beam_size,
+ ngpu=ngpu,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ penalty=penalty,
+ log_level=log_level,
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ raw_inputs=raw_inputs,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ key_file=key_file,
+ word_lm_train_config=word_lm_train_config,
+ bpemodel=bpemodel,
+ allow_variable_data_keys=allow_variable_data_keys,
+ streaming=streaming,
+ output_dir=output_dir,
+ dtype=dtype,
+ seed=seed,
+ ngram_weight=ngram_weight,
+ nbest=nbest,
+ num_workers=num_workers,
+ vad_infer_config=vad_infer_config,
+ vad_model_file=vad_model_file,
+ vad_cmvn_file=vad_cmvn_file,
+ time_stamp_writer=time_stamp_writer,
+ punc_infer_config=punc_infer_config,
+ punc_model_file=punc_model_file,
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+def inference_modelscope(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ # data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ cmvn_file: Optional[str] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
+ vad_infer_config: Optional[str] = None,
+ vad_model_file: Optional[str] = None,
+ vad_cmvn_file: Optional[str] = None,
+ time_stamp_writer: bool = False,
+ punc_infer_config: Optional[str] = None,
+ punc_model_file: Optional[str] = None,
+ **kwargs,
+):
+ assert check_argument_types()
+
+ if word_lm_train_config is not None:
+ raise NotImplementedError("Word LM is not implemented")
+ if ngpu > 1:
+ raise NotImplementedError("only single GPU decoding is supported")
+
+ logging.basicConfig(
+ level=log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if ngpu >= 1 and torch.cuda.is_available():
+ device = "cuda"
+ else:
+ device = "cpu"
+
+ # 1. Set random-seed
+ set_all_random_seed(seed)
+
+ # 2. Build speech2vadsegment
+ speech2vadsegment_kwargs = dict(
+ vad_infer_config=vad_infer_config,
+ vad_model_file=vad_model_file,
+ vad_cmvn_file=vad_cmvn_file,
+ device=device,
+ dtype=dtype,
+ )
+ # logging.info("speech2vadsegment_kwargs: {}".format(speech2vadsegment_kwargs))
+ speech2vadsegment = Speech2VadSegment(**speech2vadsegment_kwargs)
+
+ # 3. Build speech2text
+ speech2text_kwargs = dict(
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ bpemodel=bpemodel,
+ device=device,
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ dtype=dtype,
+ beam_size=beam_size,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ ngram_weight=ngram_weight,
+ penalty=penalty,
+ nbest=nbest,
+ )
+ speech2text = Speech2Text(**speech2text_kwargs)
+
+ text2punc = Text2Punc(punc_infer_config, punc_model_file, device=device, dtype=dtype)
+
+ def _forward(data_path_and_name_and_type,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ # 3. Build data-iterator
+ if data_path_and_name_and_type is None and raw_inputs is not None:
+ if isinstance(raw_inputs, torch.Tensor):
+ raw_inputs = raw_inputs.numpy()
+ data_path_and_name_and_type = [raw_inputs, "speech", "waveform"]
+ loader = ASRTask.build_streaming_iterator(
+ data_path_and_name_and_type,
+ dtype=dtype,
+ batch_size=1,
+ key_file=key_file,
+ num_workers=num_workers,
+ preprocess_fn=VADTask.build_preprocess_fn(speech2vadsegment.vad_infer_args, False),
+ collate_fn=VADTask.build_collate_fn(speech2vadsegment.vad_infer_args, False),
+ allow_variable_data_keys=allow_variable_data_keys,
+ inference=True,
+ )
+
+ forward_time_total = 0.0
+ length_total = 0.0
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ asr_result_list = []
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path is not None:
+ writer = DatadirWriter(output_path)
+ else:
+ writer = None
+
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ # batch = {k: v for k, v in batch.items() if not k.endswith("_lengths")}
+
+ logging.info("decoding, utt_id: {}".format(keys))
+ # N-best list of (text, token, token_int, hyp_object)
+ time_beg = time.time()
+ vad_results = speech2vadsegment(**batch)
+ time_end = time.time()
+ fbanks, vadsegments = vad_results[0], vad_results[1]
+ for i, segments in enumerate(vadsegments):
+ result_segments = [["", [], [], ]]
+ for j, segment_idx in enumerate(segments):
+ bed_idx, end_idx = int(segment_idx[0] / 10), int(segment_idx[1] / 10)
+ segment = fbanks[:, bed_idx:end_idx, :].to(device)
+ speech_lengths = torch.Tensor([end_idx - bed_idx]).int().to(device)
+ batch = {"speech": segment, "speech_lengths": speech_lengths, "begin_time": vadsegments[i][j][0],
+ "end_time": vadsegments[i][j][1]}
+ results = speech2text(**batch)
+ if len(results) < 1:
+ hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+ results = [[" ", ["<space>"], [2], 10, 6]] * nbest
+ time_end = time.time()
+ forward_time = time_end - time_beg
+ lfr_factor = results[0][-1]
+ length = results[0][-2]
+ forward_time_total += forward_time
+ length_total += length
+ logging.info(
+ "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}".
+ format(length, forward_time, 100 * forward_time / (length * lfr_factor)))
+ result_cur = [results[0][:-2]]
+ if j == 0:
+ result_segments = result_cur
+ else:
+ result_segments = [[result_segments[0][i] + result_cur[0][i] for i in range(len(result_cur[0]))]]
+
+ key = keys[0]
+ result = result_segments[0]
+ text, token, token_int, time_stamp = result
+
+ # Create a directory: outdir/{n}best_recog
+ if writer is not None:
+ ibest_writer = writer[f"1best_recog"]
+
+ # Write the result to each file
+ ibest_writer["token"][key] = " ".join(token)
+ # ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+
+ if text is not None:
+ postprocessed_result = postprocess_utils.sentence_postprocess(token, time_stamp)
+ if len(postprocessed_result) == 3:
+ text_postprocessed, time_stamp_postprocessed, word_lists = postprocessed_result[0], \
+ postprocessed_result[1], \
+ postprocessed_result[2]
+ text_postprocessed_punc, punc_id_list = text2punc(word_lists, 20)
+ text_postprocessed_punc_time_stamp = "predictions: {} time_stamp: {}".format(
+ text_postprocessed_punc, time_stamp_postprocessed)
+ else:
+ text_postprocessed = postprocessed_result
+ time_stamp_postprocessed = None
+ word_lists = None
+ text_postprocessed_punc_time_stamp = None
+ punc_id_list = None
+
+ item = {'key': key, 'value': text_postprocessed_punc_time_stamp, 'text': text_postprocessed,
+ 'time_stamp': time_stamp_postprocessed, 'punc': punc_id_list}
+ asr_result_list.append(item)
+ finish_count += 1
+ # asr_utils.print_progress(finish_count / file_count)
+ if writer is not None:
+ ibest_writer["text"][key] = text_postprocessed
+ if time_stamp_writer and time_stamp_postprocessed is not None:
+ ibest_writer["time_stamp"][key] = " ".join(
+ ["-".join(map(str, ts)) for ts in time_stamp_postprocessed])
+
+ logging.info("decoding, utt: {}, predictions: {}, time_stamp: {}".format(key, text_postprocessed_punc,
+ time_stamp_postprocessed))
+
+ logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".
+ format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor)))
+ return asr_result_list
+ return _forward
+
+def Text2Punc(
+ train_config: Optional[str],
+ model_file: Optional[str],
+ device: str = "cpu",
+ dtype: str = "float32",
+):
+
+ # 2. Build Model
+ model, train_args = PunctuationTask.build_model_from_file(
+ train_config, model_file, device)
+ # Wrape model to make model.nll() data-parallel
+ wrapped_model = ForwardAdaptor(model, "inference")
+ wrapped_model.to(dtype=getattr(torch, dtype)).to(device=device).eval()
+ # logging.info(f"Model:\n{model}")
+ punc_list = train_args.punc_list
+ period = 0
+ for i in range(len(punc_list)):
+ if punc_list[i] == ",":
+ punc_list[i] = "锛�"
+ elif punc_list[i] == "?":
+ punc_list[i] = "锛�"
+ elif punc_list[i] == "銆�":
+ period = i
+
+ preprocessor = CommonPreprocessor(
+ train=False,
+ token_type="word",
+ token_list=train_args.token_list,
+ bpemodel=train_args.bpemodel,
+ text_cleaner=train_args.cleaner,
+ g2p_type=train_args.g2p,
+ text_name="text",
+ non_linguistic_symbols=train_args.non_linguistic_symbols,
+ )
+
+ print("start decoding!!!")
+
+ def _forward(words, split_size = 20):
+ cache_sent = []
+ mini_sentences = split_to_mini_sentence(words, split_size)
+ new_mini_sentence = ""
+ new_mini_sentence_punc = ""
+ for mini_sentence_i in range(len(mini_sentences)):
+ mini_sentence = mini_sentences[mini_sentence_i]
+ mini_sentence = cache_sent + mini_sentence
+ data = {"text": " ".join(mini_sentence)}
+ batch = preprocessor(data=data, uid="12938712838719")
+ batch["text_lengths"] = torch.from_numpy(np.array([len(batch["text"])], dtype='int32'))
+ batch["text"] = torch.from_numpy(batch["text"])
+ # Extend one dimension to fake a batch dim.
+ batch["text"] = torch.unsqueeze(batch["text"], 0)
+ batch = to_device(batch, device)
+ y, _ = wrapped_model(**batch)
+ _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1)
+ punctuations = indices
+ if indices.size()[0] != 1:
+ punctuations = torch.squeeze(indices)
+ assert punctuations.size()[0] == len(mini_sentence)
+
+ # Search for the last Period/QuestionMark as cache
+ if mini_sentence_i < len(mini_sentences) - 1:
+ sentenceEnd = -1
+ for i in range(len(punctuations) - 2, 1, -1):
+ if punc_list[punctuations[i]] == "銆�" or punc_list[punctuations[i]] == "锛�":
+ sentenceEnd = i
+ break
+
+ cache_sent = mini_sentence[sentenceEnd + 1:]
+ mini_sentence = mini_sentence[0:sentenceEnd + 1]
+ punctuations = punctuations[0:sentenceEnd + 1]
+
+ # if len(punctuations) == 0:
+ # continue
+
+ punctuations_np = punctuations.cpu().numpy()
+ new_mini_sentence_punc += "".join([str(x) for x in punctuations_np])
+ words_with_punc = []
+ for i in range(len(mini_sentence)):
+ if i > 0:
+ if len(mini_sentence[i][0].encode()) == 1 and len(mini_sentence[i - 1][0].encode()) == 1:
+ mini_sentence[i] = " " + mini_sentence[i]
+ words_with_punc.append(mini_sentence[i])
+ if punc_list[punctuations[i]] != "_":
+ words_with_punc.append(punc_list[punctuations[i]])
+ new_mini_sentence += "".join(words_with_punc)
+
+ return new_mini_sentence, new_mini_sentence_punc
+
+ return _forward
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="ASR Decoding",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ # Note(kamo): Use '_' instead of '-' as separator.
+ # '-' is confusing if written in yaml.
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=True)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=False,
+ action="append",
+ )
+ group.add_argument("--key_file", type=str_or_none)
+ group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument(
+ "--asr_train_config",
+ type=str,
+ help="ASR training configuration",
+ )
+ group.add_argument(
+ "--asr_model_file",
+ type=str,
+ help="ASR model parameter file",
+ )
+ group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global cmvn file",
+ )
+ group.add_argument(
+ "--lm_train_config",
+ type=str,
+ help="LM training configuration",
+ )
+ group.add_argument(
+ "--lm_file",
+ type=str,
+ help="LM parameter file",
+ )
+ group.add_argument(
+ "--word_lm_train_config",
+ type=str,
+ help="Word LM training configuration",
+ )
+ group.add_argument(
+ "--word_lm_file",
+ type=str,
+ help="Word LM parameter file",
+ )
+ group.add_argument(
+ "--ngram_file",
+ type=str,
+ help="N-gram parameter file",
+ )
+ group.add_argument(
+ "--model_tag",
+ type=str,
+ help="Pretrained model tag. If specify this option, *_train_config and "
+ "*_file will be overwritten",
+ )
+
+ group = parser.add_argument_group("Beam-search related")
+ group.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+ group.add_argument("--nbest", type=int, default=1, help="Output N-best hypotheses")
+ group.add_argument("--beam_size", type=int, default=20, help="Beam size")
+ group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty")
+ group.add_argument(
+ "--maxlenratio",
+ type=float,
+ default=0.0,
+ help="Input length ratio to obtain max output length. "
+ "If maxlenratio=0.0 (default), it uses a end-detect "
+ "function "
+ "to automatically find maximum hypothesis lengths."
+ "If maxlenratio<0.0, its absolute value is interpreted"
+ "as a constant max output length",
+ )
+ group.add_argument(
+ "--minlenratio",
+ type=float,
+ default=0.0,
+ help="Input length ratio to obtain min output length",
+ )
+ group.add_argument(
+ "--ctc_weight",
+ type=float,
+ default=0.5,
+ help="CTC weight in joint decoding",
+ )
+ group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight")
+ group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight")
+ group.add_argument("--streaming", type=str2bool, default=False)
+ group.add_argument("--time_stamp_writer", type=str2bool, default=False)
+
+ group.add_argument(
+ "--frontend_conf",
+ default=None,
+ help="",
+ )
+ group.add_argument("--raw_inputs", type=list, default=None)
+ # example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
+
+ group = parser.add_argument_group("Text converter related")
+ group.add_argument(
+ "--token_type",
+ type=str_or_none,
+ default=None,
+ choices=["char", "bpe", None],
+ help="The token type for ASR model. "
+ "If not given, refers from the training args",
+ )
+ group.add_argument(
+ "--bpemodel",
+ type=str_or_none,
+ default=None,
+ help="The model path of sentencepiece. "
+ "If not given, refers from the training args",
+ )
+ group.add_argument(
+ "--vad_infer_config",
+ type=str,
+ help="VAD infer configuration",
+ )
+ group.add_argument(
+ "--vad_model_file",
+ type=str,
+ help="VAD model parameter file",
+ )
+ group.add_argument(
+ "--vad_cmvn_file",
+ type=str,
+ help="vad, Global cmvn file",
+ )
+ group.add_argument(
+ "--punc_infer_config",
+ type=str,
+ help="VAD infer configuration",
+ )
+ group.add_argument(
+ "--punc_model_file",
+ type=str,
+ help="VAD model parameter file",
+ )
+ return parser
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+ inference(**kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/bin/asr_inference_uniasr.py b/funasr/bin/asr_inference_uniasr.py
old mode 100755
new mode 100644
index 2e87675..515c0d4
--- a/funasr/bin/asr_inference_uniasr.py
+++ b/funasr/bin/asr_inference_uniasr.py
@@ -36,18 +36,10 @@
from funasr.utils import asr_utils, wav_utils, postprocess_utils
from funasr.models.frontend.wav_frontend import WavFrontend
-from modelscope.utils.logger import get_logger
-
-logger = get_logger()
header_colors = '\033[95m'
end_colors = '\033[0m'
-global_asr_language: str = 'zh-cn'
-global_sample_rate: Union[int, Dict[Any, int]] = {
- 'audio_fs': 16000,
- 'model_fs': 16000
-}
class Speech2Text:
"""Speech2Text class
@@ -65,6 +57,7 @@
self,
asr_train_config: Union[Path, str] = None,
asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
lm_train_config: Union[Path, str] = None,
lm_file: Union[Path, str] = None,
token_type: str = None,
@@ -90,12 +83,13 @@
# 1. Build ASR model
scorers = {}
asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, device
+ asr_train_config, asr_model_file, cmvn_file, device
)
frontend = None
- if asr_model.frontend is None and frontend_conf is not None:
- frontend = WavFrontend(**frontend_conf)
- # asr_model.frontend = frontend
+ if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
+ frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+
+ logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
if decoding_mode == "model1":
decoder = asr_model.decoder
@@ -149,7 +143,7 @@
if isinstance(scorer, torch.nn.Module):
scorer.to(device=device, dtype=getattr(torch, dtype)).eval()
# logging.info(f"Beam_search: {beam_search}")
- # logging.info(f"Decoding device={device}, dtype={dtype}")
+ logging.info(f"Decoding device={device}, dtype={dtype}")
# 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
if token_type is None:
@@ -167,7 +161,7 @@
else:
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
- # logging.info(f"Text tokenizer: {tokenizer}")
+ logging.info(f"Text tokenizer: {tokenizer}")
self.asr_model = asr_model
self.asr_train_args = asr_train_args
@@ -187,7 +181,7 @@
@torch.no_grad()
def __call__(
- self, speech: Union[torch.Tensor, np.ndarray]
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
) -> List[
Tuple[
Optional[str],
@@ -210,25 +204,22 @@
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
- # data: (Nsamples,) -> (1, Nsamples)
- speech = speech.unsqueeze(0).to(getattr(torch, self.dtype))
- lfr_factor = max(1, (speech.size()[-1] // 80) - 1)
- # lengths: (1,)
- lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
if self.frontend is not None:
- feats, feats_len = self.frontend.forward(speech, lengths)
+ feats, feats_len = self.frontend.forward(speech, speech_lengths)
feats = to_device(feats, device=self.device)
feats_len = feats_len.int()
+ self.asr_model.frontend = None
else:
feats = speech
- feats_len = lengths
+ feats_len = speech_lengths
+ lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
feats_raw = feats.clone().to(self.device)
batch = {"speech": feats, "speech_lengths": feats_len}
# a. To device
batch = to_device(batch, device=self.device)
# b. Forward Encoder
- enc, enc_len = self.asr_model.encode(**batch, ind=self.decoding_ind)
+ _, enc, enc_len = self.asr_model.encode(**batch, ind=self.decoding_ind)
if isinstance(enc, tuple):
enc = enc[0]
assert len(enc) == 1, len(enc)
@@ -278,6 +269,150 @@
return results
+# def inference(
+# maxlenratio: float,
+# minlenratio: float,
+# batch_size: int,
+# beam_size: int,
+# ngpu: int,
+# ctc_weight: float,
+# lm_weight: float,
+# penalty: float,
+# log_level: Union[int, str],
+# data_path_and_name_and_type,
+# asr_train_config: Optional[str],
+# asr_model_file: Optional[str],
+# ngram_file: Optional[str] = None,
+# cmvn_file: Optional[str] = None,
+# raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+# lm_train_config: Optional[str] = None,
+# lm_file: Optional[str] = None,
+# token_type: Optional[str] = None,
+# key_file: Optional[str] = None,
+# word_lm_train_config: Optional[str] = None,
+# bpemodel: Optional[str] = None,
+# allow_variable_data_keys: bool = False,
+# streaming: bool = False,
+# output_dir: Optional[str] = None,
+# dtype: str = "float32",
+# seed: int = 0,
+# ngram_weight: float = 0.9,
+# nbest: int = 1,
+# num_workers: int = 1,
+# token_num_relax: int = 1,
+# decoding_ind: int = 0,
+# decoding_mode: str = "model1",
+# **kwargs,
+# ):
+# assert check_argument_types()
+# if batch_size > 1:
+# raise NotImplementedError("batch decoding is not implemented")
+# if word_lm_train_config is not None:
+# raise NotImplementedError("Word LM is not implemented")
+# if ngpu > 1:
+# raise NotImplementedError("only single GPU decoding is supported")
+#
+# logging.basicConfig(
+# level=log_level,
+# format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+# )
+#
+# if ngpu >= 1 and torch.cuda.is_available():
+# device = "cuda"
+# else:
+# device = "cpu"
+#
+# # 1. Set random-seed
+# set_all_random_seed(seed)
+#
+# # 2. Build speech2text
+# speech2text_kwargs = dict(
+# asr_train_config=asr_train_config,
+# asr_model_file=asr_model_file,
+# cmvn_file=cmvn_file,
+# lm_train_config=lm_train_config,
+# lm_file=lm_file,
+# ngram_file=ngram_file,
+# token_type=token_type,
+# bpemodel=bpemodel,
+# device=device,
+# maxlenratio=maxlenratio,
+# minlenratio=minlenratio,
+# dtype=dtype,
+# beam_size=beam_size,
+# ctc_weight=ctc_weight,
+# lm_weight=lm_weight,
+# ngram_weight=ngram_weight,
+# penalty=penalty,
+# nbest=nbest,
+# streaming=streaming,
+# token_num_relax=token_num_relax,
+# decoding_ind=decoding_ind,
+# decoding_mode=decoding_mode,
+# )
+# speech2text = Speech2Text(**speech2text_kwargs)
+#
+# # 3. Build data-iterator
+# loader = ASRTask.build_streaming_iterator(
+# data_path_and_name_and_type,
+# dtype=dtype,
+# batch_size=batch_size,
+# key_file=key_file,
+# num_workers=num_workers,
+# preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
+# collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
+# allow_variable_data_keys=allow_variable_data_keys,
+# inference=True,
+# )
+#
+# finish_count = 0
+# file_count = 1
+# # 7 .Start for-loop
+# # FIXME(kamo): The output format should be discussed about
+# asr_result_list = []
+# if output_dir is not None:
+# writer = DatadirWriter(output_dir)
+# else:
+# writer = None
+#
+# for keys, batch in loader:
+# assert isinstance(batch, dict), type(batch)
+# assert all(isinstance(s, str) for s in keys), keys
+# _bs = len(next(iter(batch.values())))
+# assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+# #batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+#
+# # N-best list of (text, token, token_int, hyp_object)
+# try:
+# results = speech2text(**batch)
+# except TooShortUttError as e:
+# logging.warning(f"Utterance {keys} {e}")
+# hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+# results = [[" ", ["<space>"], [2], hyp]] * nbest
+#
+# # Only supporting batch_size==1
+# key = keys[0]
+# logging.info(f"Utterance: {key}")
+# for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
+# # Create a directory: outdir/{n}best_recog
+# if writer is not None:
+# ibest_writer = writer[f"{n}best_recog"]
+#
+# # Write the result to each file
+# ibest_writer["token"][key] = " ".join(token)
+# ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+# ibest_writer["score"][key] = str(hyp.score)
+#
+# if text is not None:
+# text_postprocessed = postprocess_utils.sentence_postprocess(token)
+# item = {'key': key, 'value': text_postprocessed}
+# asr_result_list.append(item)
+# finish_count += 1
+# asr_utils.print_progress(finish_count / file_count)
+# if writer is not None:
+# ibest_writer["text"][key] = text
+# return asr_result_list
+
def inference(
maxlenratio: float,
minlenratio: float,
@@ -292,7 +427,8 @@
asr_train_config: Optional[str],
asr_model_file: Optional[str],
ngram_file: Optional[str] = None,
- audio_lists: Union[List[Any], bytes] = None,
+ cmvn_file: Optional[str] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
lm_train_config: Optional[str] = None,
lm_file: Optional[str] = None,
token_type: Optional[str] = None,
@@ -307,9 +443,78 @@
ngram_weight: float = 0.9,
nbest: int = 1,
num_workers: int = 1,
- frontend_conf: dict = None,
- fs: Union[dict, int] = 16000,
- lang: Optional[str] = None,
+ token_num_relax: int = 1,
+ decoding_ind: int = 0,
+ decoding_mode: str = "model1",
+ **kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ maxlenratio=maxlenratio,
+ minlenratio=minlenratio,
+ batch_size=batch_size,
+ beam_size=beam_size,
+ ngpu=ngpu,
+ ctc_weight=ctc_weight,
+ lm_weight=lm_weight,
+ penalty=penalty,
+ log_level=log_level,
+ asr_train_config=asr_train_config,
+ asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
+ raw_inputs=raw_inputs,
+ lm_train_config=lm_train_config,
+ lm_file=lm_file,
+ token_type=token_type,
+ key_file=key_file,
+ word_lm_train_config=word_lm_train_config,
+ bpemodel=bpemodel,
+ allow_variable_data_keys=allow_variable_data_keys,
+ streaming=streaming,
+ output_dir=output_dir,
+ dtype=dtype,
+ seed=seed,
+ ngram_weight=ngram_weight,
+ ngram_file=ngram_file,
+ nbest=nbest,
+ num_workers=num_workers,
+ token_num_relax=token_num_relax,
+ decoding_ind=decoding_ind,
+ decoding_mode=decoding_mode,
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+
+def inference_modelscope(
+ maxlenratio: float,
+ minlenratio: float,
+ batch_size: int,
+ beam_size: int,
+ ngpu: int,
+ ctc_weight: float,
+ lm_weight: float,
+ penalty: float,
+ log_level: Union[int, str],
+ # data_path_and_name_and_type,
+ asr_train_config: Optional[str],
+ asr_model_file: Optional[str],
+ ngram_file: Optional[str] = None,
+ cmvn_file: Optional[str] = None,
+ # raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ lm_train_config: Optional[str] = None,
+ lm_file: Optional[str] = None,
+ token_type: Optional[str] = None,
+ key_file: Optional[str] = None,
+ word_lm_train_config: Optional[str] = None,
+ bpemodel: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ streaming: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ ngram_weight: float = 0.9,
+ nbest: int = 1,
+ num_workers: int = 1,
token_num_relax: int = 1,
decoding_ind: int = 0,
decoding_mode: str = "model1",
@@ -328,50 +533,11 @@
format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
)
- if ngpu >= 1:
+ if ngpu >= 1 and torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
- hop_length: int = 160
- sr: int = 16000
- if isinstance(fs, int):
- sr = fs
- else:
- if 'model_fs' in fs and fs['model_fs'] is not None:
- sr = fs['model_fs']
- # data_path_and_name_and_type for modelscope: (data from audio_lists)
- # ['speech', 'sound', 'am.mvn']
- # data_path_and_name_and_type for funasr:
- # [('/mnt/data/jiangyu.xzy/exp/maas/mvn.1.scp', 'speech', 'kaldi_ark')]
- if isinstance(data_path_and_name_and_type[0], Tuple):
- features_type: str = data_path_and_name_and_type[0][1]
- elif isinstance(data_path_and_name_and_type[0], str):
- features_type: str = data_path_and_name_and_type[1]
- else:
- raise NotImplementedError("unknown features type:{0}".format(data_path_and_name_and_type))
- if features_type != 'sound':
- frontend_conf = None
- flag_modelscope = False
- else:
- flag_modelscope = True
- if frontend_conf is not None:
- if 'hop_length' in frontend_conf:
- hop_length = frontend_conf['hop_length']
- finish_count = 0
- file_count = 1
- if flag_modelscope and not isinstance(data_path_and_name_and_type[0], Tuple):
- data_path_and_name_and_type_new = [
- audio_lists, data_path_and_name_and_type[0], data_path_and_name_and_type[1]
- ]
- if isinstance(audio_lists, bytes):
- file_count = 1
- else:
- file_count = len(audio_lists)
- if len(data_path_and_name_and_type) >= 3 and frontend_conf is not None:
- mvn_file = data_path_and_name_and_type[2]
- mvn_data = wav_utils.extract_CMVN_featrures(mvn_file)
- frontend_conf['mvn_data'] = mvn_data
# 1. Set random-seed
set_all_random_seed(seed)
@@ -379,6 +545,7 @@
speech2text_kwargs = dict(
asr_train_config=asr_train_config,
asr_model_file=asr_model_file,
+ cmvn_file=cmvn_file,
lm_train_config=lm_train_config,
lm_file=lm_file,
ngram_file=ngram_file,
@@ -398,25 +565,18 @@
token_num_relax=token_num_relax,
decoding_ind=decoding_ind,
decoding_mode=decoding_mode,
- frontend_conf=frontend_conf,
)
speech2text = Speech2Text(**speech2text_kwargs)
-
- # 3. Build data-iterator
- if flag_modelscope:
- loader = ASRTask.build_streaming_iterator_modelscope(
- data_path_and_name_and_type_new,
- dtype=dtype,
- batch_size=batch_size,
- key_file=key_file,
- num_workers=num_workers,
- preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False),
- collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False),
- allow_variable_data_keys=allow_variable_data_keys,
- inference=True,
- sample_rate=fs
- )
- else:
+
+ def _forward(data_path_and_name_and_type,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ # 3. Build data-iterator
+ if data_path_and_name_and_type is None and raw_inputs is not None:
+ if isinstance(raw_inputs, torch.Tensor):
+ raw_inputs = raw_inputs.numpy()
+ data_path_and_name_and_type = [raw_inputs, "speech", "waveform"]
loader = ASRTask.build_streaming_iterator(
data_path_and_name_and_type,
dtype=dtype,
@@ -428,62 +588,58 @@
allow_variable_data_keys=allow_variable_data_keys,
inference=True,
)
-
- # 7 .Start for-loop
- # FIXME(kamo): The output format should be discussed about
- asr_result_list = []
- if output_dir is not None:
- writer = DatadirWriter(output_dir)
- else:
- writer = None
-
- for keys, batch in loader:
- assert isinstance(batch, dict), type(batch)
- assert all(isinstance(s, str) for s in keys), keys
- _bs = len(next(iter(batch.values())))
- assert len(keys) == _bs, f"{len(keys)} != {_bs}"
- batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
-
- # N-best list of (text, token, token_int, hyp_object)
- try:
- results = speech2text(**batch)
- except TooShortUttError as e:
- logging.warning(f"Utterance {keys} {e}")
- hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
- results = [[" ", ["<space>"], [2], hyp]] * nbest
-
- # Only supporting batch_size==1
- key = keys[0]
- logging.info(f"Utterance: {key}")
- for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
- # Create a directory: outdir/{n}best_recog
- if writer is not None:
- ibest_writer = writer[f"{n}best_recog"]
-
- # Write the result to each file
- ibest_writer["token"][key] = " ".join(token)
- ibest_writer["token_int"][key] = " ".join(map(str, token_int))
- ibest_writer["score"][key] = str(hyp.score)
-
- if text is not None:
- text_postprocessed = postprocess_utils.sentence_postprocess(token)
- item = {'key': key, 'value': text_postprocessed}
- asr_result_list.append(item)
- finish_count += 1
- asr_utils.print_progress(finish_count / file_count)
+
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ # FIXME(kamo): The output format should be discussed about
+ asr_result_list = []
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path is not None:
+ writer = DatadirWriter(output_path)
+ else:
+ writer = None
+
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ #batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+
+ # N-best list of (text, token, token_int, hyp_object)
+ try:
+ results = speech2text(**batch)
+ except TooShortUttError as e:
+ logging.warning(f"Utterance {keys} {e}")
+ hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[])
+ results = [[" ", ["<space>"], [2], hyp]] * nbest
+
+ # Only supporting batch_size==1
+ key = keys[0]
+ logging.info(f"Utterance: {key}")
+ for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results):
+ # Create a directory: outdir/{n}best_recog
if writer is not None:
- ibest_writer["text"][key] = text
- return asr_result_list
+ ibest_writer = writer[f"{n}best_recog"]
+
+ # Write the result to each file
+ ibest_writer["token"][key] = " ".join(token)
+ # ibest_writer["token_int"][key] = " ".join(map(str, token_int))
+ ibest_writer["score"][key] = str(hyp.score)
+
+ if text is not None:
+ text_postprocessed = postprocess_utils.sentence_postprocess(token)
+ item = {'key': key, 'value': text_postprocessed}
+ asr_result_list.append(item)
+ finish_count += 1
+ asr_utils.print_progress(finish_count / file_count)
+ if writer is not None:
+ ibest_writer["text"][key] = text
+ return asr_result_list
+
+ return _forward
-
-def set_parameters(language: str = None,
- sample_rate: Union[int, Dict[Any, int]] = None):
- if language is not None:
- global global_asr_language
- global_asr_language = language
- if sample_rate is not None:
- global global_sample_rate
- global_sample_rate = sample_rate
def get_parser():
@@ -527,10 +683,10 @@
group.add_argument(
"--data_path_and_name_and_type",
type=str2triple_str,
- required=True,
+ required=False,
action="append",
)
- group.add_argument("--audio_lists", type=list, default=None)
+ group.add_argument("--raw_inputs", type=list, default=None)
# example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
group.add_argument("--key_file", type=str_or_none)
group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
@@ -547,6 +703,11 @@
help="ASR model parameter file",
)
group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global cmvn file",
+ )
+ group.add_argument(
"--lm_train_config",
type=str,
help="LM training configuration",
diff --git a/funasr/bin/build_trainer.py b/funasr/bin/build_trainer.py
new file mode 100644
index 0000000..c13f91b
--- /dev/null
+++ b/funasr/bin/build_trainer.py
@@ -0,0 +1,116 @@
+import os
+
+import yaml
+
+
+def update_dct(fin_configs, root):
+ if root == {}:
+ return {}
+ for root_key, root_value in root.items():
+ if not isinstance(root[root_key], dict):
+ fin_configs[root_key] = root[root_key]
+ else:
+ if root_key in fin_configs.keys():
+ result = update_dct(fin_configs[root_key], root[root_key])
+ fin_configs[root_key] = result
+ else:
+ fin_configs[root_key] = root[root_key]
+ return fin_configs
+
+
+def parse_args(mode):
+ if mode == "asr":
+ from funasr.tasks.asr import ASRTask as ASRTask
+ elif mode == "paraformer":
+ from funasr.tasks.asr import ASRTaskParaformer as ASRTask
+ elif mode == "paraformer_vad_punc":
+ from funasr.tasks.asr import ASRTaskParaformer as ASRTask
+ elif mode == "uniasr":
+ from funasr.tasks.asr import ASRTaskUniASR as ASRTask
+ else:
+ raise ValueError("Unknown mode: {}".format(mode))
+ parser = ASRTask.get_parser()
+ args = parser.parse_args()
+ return args, ASRTask
+
+
+def build_trainer(modelscope_dict, data_dir, output_dir, train_set="train", dev_set="validation", distributed=False,
+ dataset_type="small", lr=None, batch_bins=None, max_epoch=None, mate_params=None):
+ mode = modelscope_dict['mode']
+ args, ASRTask = parse_args(mode=mode)
+ # ddp related
+ if args.local_rank is not None:
+ distributed = True
+ else:
+ distributed = False
+ args.local_rank = args.local_rank if args.local_rank is not None else 0
+ local_rank = args.local_rank
+ if "CUDA_VISIBLE_DEVICES" in os.environ.keys():
+ gpu_list = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_list[args.local_rank])
+ else:
+ os.environ['CUDA_VISIBLE_DEVICES'] = str(args.local_rank)
+
+ config = modelscope_dict['am_model_config']
+ finetune_config = modelscope_dict['finetune_config']
+ init_param = modelscope_dict['init_model']
+ cmvn_file = modelscope_dict['cmvn_file']
+ seg_dict_file = modelscope_dict['seg_dict']
+
+ # overwrite parameters
+ with open(config) as f:
+ configs = yaml.safe_load(f)
+ with open(finetune_config) as f:
+ finetune_configs = yaml.safe_load(f)
+ # set data_types
+ if dataset_type == "large":
+ finetune_configs["dataset_conf"]["data_types"] = "sound,text"
+ finetune_configs = update_dct(configs, finetune_configs)
+ for key, value in finetune_configs.items():
+ if hasattr(args, key):
+ setattr(args, key, value)
+
+ # prepare data
+ args.dataset_type = dataset_type
+ if args.dataset_type == "small":
+ args.train_data_path_and_name_and_type = [["{}/{}/wav.scp".format(data_dir, train_set), "speech", "sound"],
+ ["{}/{}/text".format(data_dir, train_set), "text", "text"]]
+ args.valid_data_path_and_name_and_type = [["{}/{}/wav.scp".format(data_dir, dev_set), "speech", "sound"],
+ ["{}/{}/text".format(data_dir, dev_set), "text", "text"]]
+ elif args.dataset_type == "large":
+ args.train_data_file = None
+ args.valid_data_file = None
+ else:
+ raise ValueError(f"Not supported dataset_type={args.dataset_type}")
+ args.init_param = [init_param]
+ args.cmvn_file = cmvn_file
+ if os.path.exists(seg_dict_file):
+ args.seg_dict_file = seg_dict_file
+ else:
+ args.seg_dict_file = None
+ args.data_dir = data_dir
+ args.train_set = train_set
+ args.dev_set = dev_set
+ args.output_dir = output_dir
+ args.gpu_id = args.local_rank
+ args.config = finetune_config
+ if lr is not None:
+ args.optim_conf["lr"] = lr
+ if max_epoch is not None:
+ args.max_epoch = max_epoch
+ if batch_bins is not None:
+ if args.dataset_type == "small":
+ args.batch_bins = batch_bins
+ elif args.dataset_type == "large":
+ args.dataset_conf["batch_conf"]["batch_size"] = batch_bins
+ else:
+ raise ValueError(f"Not supported dataset_type={args.dataset_type}")
+ if args.normalize in ["null", "none", "None"]:
+ args.normalize = None
+ if args.patience in ["null", "none", "None"]:
+ args.patience = None
+ args.local_rank = local_rank
+ args.distributed = distributed
+ ASRTask.finetune_args = args
+
+ return ASRTask
diff --git a/funasr/bin/punc_inference_launch.py b/funasr/bin/punc_inference_launch.py
new file mode 100755
index 0000000..61d4bf4
--- /dev/null
+++ b/funasr/bin/punc_inference_launch.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved.
+# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+
+import argparse
+import logging
+import os
+import sys
+from typing import Union, Dict, Any
+
+from funasr.utils import config_argparse
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+from funasr.utils.types import float_or_none
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="Punctuation inference",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+ parser.add_argument("--output_dir", type=str, required=True)
+ parser.add_argument("--gpuid_list", type=str, required=True)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument("--njob", type=int, default=1, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+ parser.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ action="append",
+ required=False
+ )
+ group.add_argument(
+ "--raw_inputs",
+ type=str,
+ required=False
+ )
+ group.add_argument("--key_file", type=str_or_none)
+
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument("--train_config", type=str)
+ group.add_argument("--model_file", type=str)
+ group.add_argument("--mode", type=str, default="punc")
+ return parser
+
+def inference_launch(mode, **kwargs):
+ if mode == "punc":
+ from funasr.bin.punctuation_infer import inference_modelscope
+ return inference_modelscope(**kwargs)
+ else:
+ logging.info("Unknown decoding mode: {}".format(mode))
+ return None
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+
+ # set logging messages
+ logging.basicConfig(
+ level=args.log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+ logging.info("Decoding args: {}".format(kwargs))
+
+ # gpu setting
+ if args.ngpu > 0:
+ jobid = int(args.output_dir.split(".")[-1])
+ gpuid = args.gpuid_list.split(",")[(jobid - 1) // args.njob]
+ os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+ os.environ["CUDA_VISIBLE_DEVICES"] = gpuid
+
+ kwargs.pop("gpuid_list", None)
+ kwargs.pop("njob", None)
+ results = inference_launch(**kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/bin/punctuation_infer.py b/funasr/bin/punctuation_infer.py
new file mode 100644
index 0000000..b38ff94
--- /dev/null
+++ b/funasr/bin/punctuation_infer.py
@@ -0,0 +1,374 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+from pathlib import Path
+import sys
+import os
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+from typing import Dict
+from typing import Any
+from typing import List
+
+import numpy as np
+import torch
+from torch.nn.parallel import data_parallel
+from typeguard import check_argument_types
+
+from funasr.datasets.preprocessor import CommonPreprocessor
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.fileio.datadir_writer import DatadirWriter
+from funasr.tasks.punctuation import PunctuationTask
+from funasr.torch_utils.device_funcs import to_device
+from funasr.torch_utils.forward_adaptor import ForwardAdaptor
+from funasr.torch_utils.set_all_random_seed import set_all_random_seed
+from funasr.utils import config_argparse
+from funasr.utils.types import float_or_none
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+from funasr.punctuation.text_preprocessor import split_words, split_to_mini_sentence
+
+
+def inference(
+ batch_size: int,
+ dtype: str,
+ ngpu: int,
+ seed: int,
+ num_workers: int,
+ output_dir: str,
+ log_level: Union[int, str],
+ train_config: Optional[str],
+ model_file: Optional[str],
+ key_file: Optional[str] = None,
+ data_path_and_name_and_type: Sequence[Tuple[str, str, str]] = None,
+ raw_inputs: Union[List[Any], bytes, str] = None,
+
+ **kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ output_dir=output_dir,
+ raw_inputs=raw_inputs,
+ batch_size=batch_size,
+ dtype=dtype,
+ ngpu=ngpu,
+ seed=seed,
+ num_workers=num_workers,
+ log_level=log_level,
+ key_file=key_file,
+ train_config=train_config,
+ model_file=model_file,
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+
+def inference_modelscope(
+ batch_size: int,
+ dtype: str,
+ ngpu: int,
+ seed: int,
+ num_workers: int,
+ log_level: Union[int, str],
+ key_file: Optional[str],
+ train_config: Optional[str],
+ model_file: Optional[str],
+ output_dir: Optional[str] = None,
+ **kwargs,
+):
+ assert check_argument_types()
+ logging.basicConfig(
+ level=log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if ngpu >= 1 and torch.cuda.is_available():
+ device = "cuda"
+ else:
+ device = "cpu"
+
+ # 1. Set random-seed
+ set_all_random_seed(seed)
+
+ # 2. Build Model
+ model, train_args = PunctuationTask.build_model_from_file(
+ train_config, model_file, device)
+ # Wrape model to make model.nll() data-parallel
+ wrapped_model = ForwardAdaptor(model, "inference")
+ wrapped_model.to(dtype=getattr(torch, dtype)).to(device=device).eval()
+ logging.info(f"Model:\n{model}")
+ punc_list = train_args.punc_list
+ period = 0
+ for i in range(len(punc_list)):
+ if punc_list[i] == ",":
+ punc_list[i] = "锛�"
+ elif punc_list[i] == "?":
+ punc_list[i] = "锛�"
+ elif punc_list[i] == "銆�":
+ period = i
+
+ preprocessor = CommonPreprocessor(
+ train=False,
+ token_type="word",
+ token_list=train_args.token_list,
+ bpemodel=train_args.bpemodel,
+ text_cleaner=train_args.cleaner,
+ g2p_type=train_args.g2p,
+ text_name="text",
+ non_linguistic_symbols=train_args.non_linguistic_symbols,
+ )
+
+ print("start decoding!!!")
+
+ def _forward(
+ data_path_and_name_and_type,
+ raw_inputs: Union[List[Any], bytes, str] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ results = []
+ split_size = 20
+
+ if raw_inputs != None:
+ line = raw_inputs.strip()
+ key = "demo"
+ if line=="":
+ item = {'key': key, 'value': ""}
+ results.append(item)
+ return results
+ cache_sent = []
+ words = split_words(line)
+ new_mini_sentence = ""
+ new_mini_sentence_punc = ""
+ cache_pop_trigger_limit = 200
+ mini_sentences = split_to_mini_sentence(words, split_size)
+ for mini_sentence_i in range(len(mini_sentences)):
+ mini_sentence = mini_sentences[mini_sentence_i]
+ mini_sentence = cache_sent + mini_sentence
+ data = {"text": " ".join(mini_sentence)}
+ batch = preprocessor(data=data, uid="12938712838719")
+ batch["text_lengths"] = torch.from_numpy(
+ np.array([len(batch["text"])], dtype='int32'))
+ batch["text"] = torch.from_numpy(batch["text"])
+ # Extend one dimension to fake a batch dim.
+ batch["text"] = torch.unsqueeze(batch["text"], 0)
+ batch = to_device(batch, device)
+ y, _ = wrapped_model(**batch)
+ _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1)
+ punctuations = indices
+ if indices.size()[0] != 1:
+ punctuations = torch.squeeze(indices)
+ assert punctuations.size()[0] == len(mini_sentence)
+
+ # Search for the last Period/QuestionMark as cache
+ if mini_sentence_i < len(mini_sentences)-1:
+ sentenceEnd = -1
+ last_comma_index = -1
+ for i in range(len(punctuations)-2,1,-1):
+ if punc_list[punctuations[i]] == "銆�" or punc_list[punctuations[i]] == "锛�":
+ sentenceEnd = i
+ break
+ if last_comma_index < 0 and punc_list[punctuations[i]] == "锛�":
+ last_comma_index = i
+ if sentenceEnd < 0 and len(mini_sentence) > cache_pop_trigger_limit and last_comma_index >= 0:
+ # The sentence it too long, cut off at a comma.
+ sentenceEnd = last_comma_index
+ punctuations[sentenceEnd] = period
+ cache_sent = mini_sentence[sentenceEnd+1:]
+ mini_sentence = mini_sentence[0:sentenceEnd+1]
+ punctuations = punctuations[0:sentenceEnd+1]
+
+ punctuations_np = punctuations.cpu().numpy()
+ new_mini_sentence_punc += "".join([str(x) for x in punctuations_np])
+ words_with_punc = []
+ for i in range(len(mini_sentence)):
+ if i>0:
+ if len(mini_sentence[i][0].encode()) == 1 and len(mini_sentence[i-1][0].encode()) == 1:
+ mini_sentence[i] = " "+ mini_sentence[i]
+ words_with_punc.append(mini_sentence[i])
+ if punc_list[punctuations[i]] != "_":
+ words_with_punc.append(punc_list[punctuations[i]])
+ new_mini_sentence += "".join(words_with_punc)
+
+ # Add Period for the end of the sentence
+ new_mini_sentence_out = new_mini_sentence
+ new_mini_sentence_punc_out = new_mini_sentence_punc
+ if mini_sentence_i == len(mini_sentences)-1:
+ if new_mini_sentence[-1]=="锛�" or new_mini_sentence[-1]=="銆�":
+ new_mini_sentence_out = new_mini_sentence[:-1] + "銆�"
+ new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + str(period)
+ elif new_mini_sentence[-1]!="銆�" and new_mini_sentence[-1]!="锛�":
+ new_mini_sentence_out=new_mini_sentence+"銆�"
+ new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + str(period)
+ item = {'key': key, 'value': new_mini_sentence_out}
+ results.append(item)
+
+ return results
+
+ for inference_text, _, _ in data_path_and_name_and_type:
+ with open(inference_text, "r", encoding="utf-8") as fin:
+ for line in fin:
+ line = line.strip()
+ segs = line.split("\t")
+ if len(segs) != 2:
+ continue
+ key = segs[0]
+ if len(segs[1]) == 0:
+ continue
+ cache_sent = []
+ words = split_words(segs[1])
+ new_mini_sentence = ""
+ new_mini_sentence_punc = ""
+ cache_pop_trigger_limit = 200
+ mini_sentences = split_to_mini_sentence(words, split_size)
+ for mini_sentence_i in range(len(mini_sentences)):
+ mini_sentence = mini_sentences[mini_sentence_i]
+ mini_sentence = cache_sent + mini_sentence
+ data = {"text": " ".join(mini_sentence)}
+ batch = preprocessor(data=data, uid="12938712838719")
+ batch["text_lengths"] = torch.from_numpy(
+ np.array([len(batch["text"])], dtype='int32'))
+ batch["text"] = torch.from_numpy(batch["text"])
+ # Extend one dimension to fake a batch dim.
+ batch["text"] = torch.unsqueeze(batch["text"], 0)
+ batch = to_device(batch, device)
+ y, _ = wrapped_model(**batch)
+ _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1)
+ punctuations = indices
+ if indices.size()[0] != 1:
+ punctuations = torch.squeeze(indices)
+ assert punctuations.size()[0] == len(mini_sentence)
+
+ # Search for the last Period/QuestionMark as cache
+ if mini_sentence_i < len(mini_sentences)-1:
+ sentenceEnd = -1
+ last_comma_index = -1
+ for i in range(len(punctuations)-2,1,-1):
+ if punc_list[punctuations[i]] == "銆�" or punc_list[punctuations[i]] == "锛�":
+ sentenceEnd = i
+ break
+ if last_comma_index < 0 and punc_list[punctuations[i]] == "锛�":
+ last_comma_index = i
+ if sentenceEnd < 0 and len(mini_sentence) > cache_pop_trigger_limit and last_comma_index >= 0:
+ # The sentence it too long, cut off at a comma.
+ sentenceEnd = last_comma_index
+ punctuations[sentenceEnd] = period
+ cache_sent = mini_sentence[sentenceEnd+1:]
+ mini_sentence = mini_sentence[0:sentenceEnd+1]
+ punctuations = punctuations[0:sentenceEnd+1]
+
+ punctuations_np = punctuations.cpu().numpy()
+ new_mini_sentence_punc += "".join([str(x) for x in punctuations_np])
+ words_with_punc = []
+ for i in range(len(mini_sentence)):
+ if i>0:
+ if len(mini_sentence[i][0].encode()) == 1 and len(mini_sentence[i-1][0].encode()) == 1:
+ mini_sentence[i] = " "+ mini_sentence[i]
+ words_with_punc.append(mini_sentence[i])
+ if punc_list[punctuations[i]] != "_":
+ words_with_punc.append(punc_list[punctuations[i]])
+ new_mini_sentence += "".join(words_with_punc)
+
+ # Add Period for the end of the sentence
+ new_mini_sentence_out = new_mini_sentence
+ new_mini_sentence_punc_out = new_mini_sentence_punc
+ if mini_sentence_i == len(mini_sentences)-1:
+ if new_mini_sentence[-1]=="锛�" or new_mini_sentence[-1]=="銆�":
+ new_mini_sentence_out = new_mini_sentence[:-1] + "銆�"
+ new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + str(period)
+ elif new_mini_sentence[-1]!="銆�" and new_mini_sentence[-1]!="锛�":
+ new_mini_sentence_out=new_mini_sentence+"銆�"
+ new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + str(period)
+ item = {'key': key, 'value': new_mini_sentence_out}
+ results.append(item)
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path != None:
+ output_file_name = "infer.out"
+ Path(output_path).mkdir(parents=True, exist_ok=True)
+ output_file_path = (Path(output_path) / output_file_name).absolute()
+ with open(output_file_path, "w", encoding="utf-8") as fout:
+ for item_i in results:
+ key_out = item_i["key"]
+ value_out = item_i["value"]
+ fout.write(f"{key_out}\t{value_out}\n")
+ return results
+ return _forward
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="Punctuation inference",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=False)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+ parser.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ action="append",
+ required=False
+ )
+ group.add_argument(
+ "--raw_inputs",
+ type=str,
+ required=False
+ )
+ group.add_argument("--key_file", type=str_or_none)
+
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument("--train_config", type=str)
+ group.add_argument("--model_file", type=str)
+
+ return parser
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ # kwargs.pop("config", None)
+ inference(**kwargs)
+
+if __name__ == "__main__":
+ main()
+
+
+
diff --git a/funasr/bin/sv_inference.py b/funasr/bin/sv_inference.py
new file mode 100755
index 0000000..b0fae38
--- /dev/null
+++ b/funasr/bin/sv_inference.py
@@ -0,0 +1,434 @@
+#!/usr/bin/env python3
+import argparse
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Any
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+
+import numpy as np
+import torch
+from kaldiio import WriteHelper
+from typeguard import check_argument_types
+from typeguard import check_return_type
+
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.tasks.sv import SVTask
+from funasr.tasks.asr import ASRTask
+from funasr.torch_utils.device_funcs import to_device
+from funasr.torch_utils.set_all_random_seed import set_all_random_seed
+from funasr.utils import config_argparse
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+
+
+class Speech2Xvector:
+ """Speech2Xvector class
+
+ Examples:
+ >>> import soundfile
+ >>> speech2xvector = Speech2Xvector("sv_config.yml", "sv.pth")
+ >>> audio, rate = soundfile.read("speech.wav")
+ >>> speech2xvector(audio)
+ [(text, token, token_int, hypothesis object), ...]
+
+ """
+
+ def __init__(
+ self,
+ sv_train_config: Union[Path, str] = None,
+ sv_model_file: Union[Path, str] = None,
+ device: str = "cpu",
+ batch_size: int = 1,
+ dtype: str = "float32",
+ streaming: bool = False,
+ embedding_node: str = "resnet1_dense",
+ ):
+ assert check_argument_types()
+
+ # TODO: 1. Build SV model
+ sv_model, sv_train_args = SVTask.build_model_from_file(
+ config_file=sv_train_config,
+ model_file=sv_model_file,
+ device=device
+ )
+ logging.info("sv_model: {}".format(sv_model))
+ logging.info("sv_train_args: {}".format(sv_train_args))
+ sv_model.to(dtype=getattr(torch, dtype)).eval()
+
+ self.sv_model = sv_model
+ self.sv_train_args = sv_train_args
+ self.device = device
+ self.dtype = dtype
+ self.embedding_node = embedding_node
+
+ @torch.no_grad()
+ def calculate_embedding(self, speech: Union[torch.Tensor, np.ndarray]) -> torch.Tensor:
+ # Input as audio signal
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ # data: (Nsamples,) -> (1, Nsamples)
+ speech = speech.unsqueeze(0).to(getattr(torch, self.dtype))
+ # lengths: (1,)
+ lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
+ batch = {"speech": speech, "speech_lengths": lengths}
+
+ # a. To device
+ batch = to_device(batch, device=self.device)
+
+ # b. Forward Encoder
+ enc, ilens = self.sv_model.encode(**batch)
+
+ # c. Forward Pooling
+ pooling = self.sv_model.pooling_layer(enc)
+
+ # d. Forward Decoder
+ outputs, embeddings = self.sv_model.decoder(pooling)
+
+ if self.embedding_node not in embeddings:
+ raise ValueError("Required embedding node {} not in {}".format(
+ self.embedding_node, embeddings.keys()))
+
+ return embeddings[self.embedding_node]
+
+ @torch.no_grad()
+ def __call__(
+ self, speech: Union[torch.Tensor, np.ndarray],
+ ref_speech: Optional[Union[torch.Tensor, np.ndarray]] = None,
+ ) -> Tuple[torch.Tensor, Union[torch.Tensor, None], Union[torch.Tensor, None]]:
+ """Inference
+
+ Args:
+ speech: Input speech data
+ ref_speech: Reference speech to compare
+ Returns:
+ embedding, ref_embedding, similarity_score
+
+ """
+ assert check_argument_types()
+ self.sv_model.eval()
+ embedding = self.calculate_embedding(speech)
+ ref_emb, score = None, None
+ if ref_speech is not None:
+ ref_emb = self.calculate_embedding(ref_speech)
+ score = torch.cosine_similarity(embedding, ref_emb)
+
+ results = (embedding, ref_emb, score)
+ assert check_return_type(results)
+ return results
+
+ @staticmethod
+ def from_pretrained(
+ model_tag: Optional[str] = None,
+ **kwargs: Optional[Any],
+ ):
+ """Build Speech2Xvector instance from the pretrained model.
+
+ Args:
+ model_tag (Optional[str]): Model tag of the pretrained models.
+ Currently, the tags of espnet_model_zoo are supported.
+
+ Returns:
+ Speech2Xvector: Speech2Xvector instance.
+
+ """
+ if model_tag is not None:
+ try:
+ from espnet_model_zoo.downloader import ModelDownloader
+
+ except ImportError:
+ logging.error(
+ "`espnet_model_zoo` is not installed. "
+ "Please install via `pip install -U espnet_model_zoo`."
+ )
+ raise
+ d = ModelDownloader()
+ kwargs.update(**d.download_and_unpack(model_tag))
+
+ return Speech2Xvector(**kwargs)
+
+
+def inference_modelscope(
+ output_dir: Optional[str],
+ batch_size: int,
+ dtype: str,
+ ngpu: int,
+ seed: int,
+ num_workers: int,
+ log_level: Union[int, str],
+ key_file: Optional[str],
+ sv_train_config: Optional[str],
+ sv_model_file: Optional[str],
+ model_tag: Optional[str],
+ allow_variable_data_keys: bool = True,
+ streaming: bool = False,
+ embedding_node: str = "resnet1_dense",
+ sv_threshold: float = 0.9465,
+ **kwargs,
+):
+ assert check_argument_types()
+ if batch_size > 1:
+ raise NotImplementedError("batch decoding is not implemented")
+ if ngpu > 1:
+ raise NotImplementedError("only single GPU decoding is supported")
+
+ logging.basicConfig(
+ level=log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if ngpu >= 1 and torch.cuda.is_available():
+ device = "cuda"
+ else:
+ device = "cpu"
+
+ # 1. Set random-seed
+ set_all_random_seed(seed)
+
+ # 2. Build speech2xvector
+ speech2xvector_kwargs = dict(
+ sv_train_config=sv_train_config,
+ sv_model_file=sv_model_file,
+ device=device,
+ dtype=dtype,
+ streaming=streaming,
+ embedding_node=embedding_node
+ )
+ logging.info("speech2xvector_kwargs: {}".format(speech2xvector_kwargs))
+ speech2xvector = Speech2Xvector.from_pretrained(
+ model_tag=model_tag,
+ **speech2xvector_kwargs,
+ )
+ speech2xvector.sv_model.eval()
+
+ def _forward(
+ data_path_and_name_and_type: Sequence[Tuple[str, str, str]] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ if data_path_and_name_and_type is None and raw_inputs is not None:
+ if isinstance(raw_inputs, torch.Tensor):
+ raw_inputs = raw_inputs.numpy()
+ data_path_and_name_and_type = [raw_inputs, "speech", "waveform"]
+
+ # 3. Build data-iterator
+ loader = ASRTask.build_streaming_iterator(
+ data_path_and_name_and_type,
+ dtype=dtype,
+ batch_size=batch_size,
+ key_file=key_file,
+ num_workers=num_workers,
+ preprocess_fn=None,
+ collate_fn=None,
+ allow_variable_data_keys=allow_variable_data_keys,
+ inference=True,
+ )
+
+ # 7 .Start for-loop
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ embd_fd, ref_emb_fd, score_fd = None, None, None
+ if output_path is not None:
+ os.makedirs(output_path, exist_ok=True)
+ embd_writer = WriteHelper("ark:{}/xvector.ark".format(output_path))
+ # embd_fd = open(os.path.join(output_path, "xvector.ark"), "wb")
+ sv_result_list = []
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+
+ embedding, ref_embedding, score = speech2xvector(**batch)
+ # Only supporting batch_size==1
+ key = keys[0]
+ if score is not None:
+ score = score.item()
+ normalized_score = max(score - sv_threshold, 0.0) / (1.0 - sv_threshold) * 100.0
+ item = {"key": key, "value": normalized_score}
+ else:
+ item = {"key": key, "value": embedding.squeeze(0).cpu().numpy()}
+ sv_result_list.append(item)
+ if output_path is not None:
+ # kaldiio.save_mat(embd_fd, embedding[0].cpu().numpy(), key)
+ embd_writer(key, embedding[0].cpu().numpy())
+ if ref_embedding is not None:
+ if ref_emb_fd is None:
+ # ref_emb_fd = open(os.path.join(output_path, "ref_xvector.ark"), "wb")
+ ref_embd_writer = WriteHelper("ark:{}/ref_xvector.ark".format(output_path))
+ score_fd = open(os.path.join(output_path, "score.txt"), "w")
+ # kaldiio.save_mat(ref_emb_fd, ref_embedding[0].cpu().numpy(), key)
+ ref_embd_writer(key, ref_embedding[0].cpu().numpy())
+ score_fd.write("{:.6f}\n".format(score.item()))
+ if output_path is not None:
+ # embd_fd.close()
+ embd_writer.close()
+ if ref_emb_fd is not None:
+ # ref_emb_fd.close()
+ ref_emb_fd.close()
+ score_fd.close()
+
+ return sv_result_list
+
+ return _forward
+
+
+def inference(
+ output_dir: Optional[str],
+ batch_size: int,
+ dtype: str,
+ ngpu: int,
+ seed: int,
+ num_workers: int,
+ log_level: Union[int, str],
+ data_path_and_name_and_type: Sequence[Tuple[str, str, str]],
+ key_file: Optional[str],
+ sv_train_config: Optional[str],
+ sv_model_file: Optional[str],
+ model_tag: Optional[str],
+ allow_variable_data_keys: bool = True,
+ streaming: bool = False,
+ embedding_node: str = "resnet1_dense",
+ sv_threshold: float = 0.9465,
+ **kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ output_dir=output_dir,
+ batch_size=batch_size,
+ dtype=dtype,
+ ngpu=ngpu,
+ seed=seed,
+ num_workers=num_workers,
+ log_level=log_level,
+ key_file=key_file,
+ sv_train_config=sv_train_config,
+ sv_model_file=sv_model_file,
+ model_tag=model_tag,
+ allow_variable_data_keys=allow_variable_data_keys,
+ streaming=streaming,
+ embedding_node=embedding_node,
+ sv_threshold=sv_threshold,
+ **kwargs,
+ )
+
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs=None)
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="Speaker verification/x-vector extraction",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ # Note(kamo): Use '_' instead of '-' as separator.
+ # '-' is confusing if written in yaml.
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=False)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument(
+ "--gpuid_list",
+ type=str,
+ default="",
+ help="The visible gpus",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=False,
+ action="append",
+ )
+ group.add_argument("--key_file", type=str_or_none)
+ group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument(
+ "--sv_train_config",
+ type=str,
+ help="SV training configuration",
+ )
+ group.add_argument(
+ "--sv_model_file",
+ type=str,
+ help="SV model parameter file",
+ )
+ group.add_argument(
+ "--sv_threshold",
+ type=float,
+ default=0.9465,
+ help="The threshold for verification"
+ )
+ group.add_argument(
+ "--model_tag",
+ type=str,
+ help="Pretrained model tag. If specify this option, *_train_config and "
+ "*_file will be overwritten",
+ )
+ parser.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+ parser.add_argument("--streaming", type=str2bool, default=False)
+ parser.add_argument("--embedding_node", type=str, default="resnet1_dense")
+
+ return parser
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+ logging.info("args: {}".format(kwargs))
+ if args.output_dir is None:
+ jobid, n_gpu = 1, 1
+ gpuid = args.gpuid_list.split(",")[jobid-1]
+ else:
+ jobid = int(args.output_dir.split(".")[-1])
+ n_gpu = len(args.gpuid_list.split(","))
+ gpuid = args.gpuid_list.split(",")[(jobid - 1) % n_gpu]
+ os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+ os.environ["CUDA_VISIBLE_DEVICES"] = gpuid
+ results_list = inference(**kwargs)
+ for results in results_list:
+ print("{} {}".format(results["key"], results["value"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/bin/sv_inference_launch.py b/funasr/bin/sv_inference_launch.py
new file mode 100755
index 0000000..c511dc7
--- /dev/null
+++ b/funasr/bin/sv_inference_launch.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved.
+# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+
+import argparse
+import logging
+import os
+import sys
+from typing import Union, Dict, Any
+
+from funasr.utils import config_argparse
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="Speaker Verification",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ # Note(kamo): Use '_' instead of '-' as separator.
+ # '-' is confusing if written in yaml.
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=False)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument(
+ "--njob",
+ type=int,
+ default=1,
+ help="The number of jobs for each gpu",
+ )
+ parser.add_argument(
+ "--gpuid_list",
+ type=str,
+ default="",
+ help="The visible gpus",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=False,
+ action="append",
+ )
+ group.add_argument("--key_file", type=str_or_none)
+ group.add_argument("--allow_variable_data_keys", type=str2bool, default=True)
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument(
+ "--vad_infer_config",
+ type=str,
+ help="VAD infer configuration",
+ )
+ group.add_argument(
+ "--vad_model_file",
+ type=str,
+ help="VAD model parameter file",
+ )
+ group.add_argument(
+ "--sv_train_config",
+ type=str,
+ help="ASR training configuration",
+ )
+ group.add_argument(
+ "--sv_model_file",
+ type=str,
+ help="ASR model parameter file",
+ )
+ group.add_argument(
+ "--cmvn_file",
+ type=str,
+ help="Global CMVN file",
+ )
+ group.add_argument(
+ "--model_tag",
+ type=str,
+ help="Pretrained model tag. If specify this option, *_train_config and "
+ "*_file will be overwritten",
+ )
+
+ group = parser.add_argument_group("The inference configuration related")
+ group.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+ group.add_argument(
+ "--sv_threshold",
+ type=float,
+ default=0.9465,
+ help="The threshold for verification"
+ )
+ parser.add_argument(
+ "--embedding_node",
+ type=str,
+ default="resnet1_dense",
+ help="The network node to extract embedding"
+ )
+
+ return parser
+
+
+def inference_launch(mode, **kwargs):
+ if mode == "sv":
+ from funasr.bin.sv_inference import inference_modelscope
+ return inference_modelscope(**kwargs)
+ else:
+ logging.info("Unknown decoding mode: {}".format(mode))
+ return None
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ parser.add_argument(
+ "--mode",
+ type=str,
+ default="sv",
+ help="The decoding mode",
+ )
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+
+ # set logging messages
+ logging.basicConfig(
+ level=args.log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+ logging.info("Decoding args: {}".format(kwargs))
+
+ # gpu setting
+ if args.ngpu > 0:
+ jobid = int(args.output_dir.split(".")[-1])
+ gpuid = args.gpuid_list.split(",")[(jobid - 1) // args.njob]
+ os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+ os.environ["CUDA_VISIBLE_DEVICES"] = gpuid
+
+ inference_launch(**kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/bin/vad_inference.py b/funasr/bin/vad_inference.py
new file mode 100644
index 0000000..679cc0b
--- /dev/null
+++ b/funasr/bin/vad_inference.py
@@ -0,0 +1,430 @@
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+from typing import Dict
+
+import numpy as np
+import torch
+from typeguard import check_argument_types
+from typeguard import check_return_type
+
+from funasr.fileio.datadir_writer import DatadirWriter
+from funasr.modules.scorers.scorer_interface import BatchScorerInterface
+from funasr.modules.subsampling import TooShortUttError
+from funasr.tasks.vad import VADTask
+from funasr.torch_utils.device_funcs import to_device
+from funasr.torch_utils.set_all_random_seed import set_all_random_seed
+from funasr.utils import config_argparse
+from funasr.utils.cli_utils import get_commandline_args
+from funasr.utils.types import str2bool
+from funasr.utils.types import str2triple_str
+from funasr.utils.types import str_or_none
+from funasr.utils import asr_utils, wav_utils, postprocess_utils
+from funasr.models.frontend.wav_frontend import WavFrontend
+
+header_colors = '\033[95m'
+end_colors = '\033[0m'
+
+global_asr_language: str = 'zh-cn'
+global_sample_rate: Union[int, Dict[Any, int]] = {
+ 'audio_fs': 16000,
+ 'model_fs': 16000
+}
+
+
+class Speech2VadSegment:
+ """Speech2VadSegment class
+
+ Examples:
+ >>> import soundfile
+ >>> speech2segment = Speech2VadSegment("vad_config.yml", "vad.pt")
+ >>> audio, rate = soundfile.read("speech.wav")
+ >>> speech2segment(audio)
+ [[10, 230], [245, 450], ...]
+
+ """
+
+ def __init__(
+ self,
+ vad_infer_config: Union[Path, str] = None,
+ vad_model_file: Union[Path, str] = None,
+ vad_cmvn_file: Union[Path, str] = None,
+ device: str = "cpu",
+ batch_size: int = 1,
+ dtype: str = "float32",
+ **kwargs,
+ ):
+ assert check_argument_types()
+
+ # 1. Build vad model
+ vad_model, vad_infer_args = VADTask.build_model_from_file(
+ vad_infer_config, vad_model_file, device
+ )
+ frontend = None
+ if vad_infer_args.frontend is not None:
+ frontend = WavFrontend(cmvn_file=vad_cmvn_file, **vad_infer_args.frontend_conf)
+
+ logging.info("vad_model: {}".format(vad_model))
+ logging.info("vad_infer_args: {}".format(vad_infer_args))
+ vad_model.to(dtype=getattr(torch, dtype)).eval()
+
+ self.vad_model = vad_model
+ self.vad_infer_args = vad_infer_args
+ self.device = device
+ self.dtype = dtype
+ self.frontend = frontend
+
+ @torch.no_grad()
+ def __call__(
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
+ ) -> List[List[int]]:
+ """Inference
+
+ Args:
+ speech: Input speech data
+ Returns:
+ text, token, token_int, hyp
+
+ """
+ assert check_argument_types()
+
+ # Input as audio signal
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ if self.frontend is not None:
+ feats, feats_len = self.frontend.forward(speech, speech_lengths)
+ feats = to_device(feats, device=self.device)
+ feats_len = feats_len.int()
+ else:
+ raise Exception("Need to extract feats first, please configure frontend configuration")
+ batch = {"feats": feats, "feats_lengths": feats_len, "waveform": speech}
+
+ # a. To device
+ batch = to_device(batch, device=self.device)
+
+ # b. Forward Encoder
+ segments = self.vad_model(**batch)
+
+ return segments
+
+
+#def inference(
+# batch_size: int,
+# ngpu: int,
+# log_level: Union[int, str],
+# data_path_and_name_and_type,
+# vad_infer_config: Optional[str],
+# vad_model_file: Optional[str],
+# vad_cmvn_file: Optional[str] = None,
+# raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+# key_file: Optional[str] = None,
+# allow_variable_data_keys: bool = False,
+# output_dir: Optional[str] = None,
+# dtype: str = "float32",
+# seed: int = 0,
+# num_workers: int = 1,
+# fs: Union[dict, int] = 16000,
+# **kwargs,
+#):
+# assert check_argument_types()
+# if batch_size > 1:
+# raise NotImplementedError("batch decoding is not implemented")
+# if ngpu > 1:
+# raise NotImplementedError("only single GPU decoding is supported")
+#
+# logging.basicConfig(
+# level=log_level,
+# format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+# )
+#
+# if ngpu >= 1 and torch.cuda.is_available():
+# device = "cuda"
+# else:
+# device = "cpu"
+#
+# # 1. Set random-seed
+# set_all_random_seed(seed)
+#
+# # 2. Build speech2vadsegment
+# speech2vadsegment_kwargs = dict(
+# vad_infer_config=vad_infer_config,
+# vad_model_file=vad_model_file,
+# vad_cmvn_file=vad_cmvn_file,
+# device=device,
+# dtype=dtype,
+# )
+# logging.info("speech2vadsegment_kwargs: {}".format(speech2vadsegment_kwargs))
+# speech2vadsegment = Speech2VadSegment(**speech2vadsegment_kwargs)
+# # 3. Build data-iterator
+# loader = VADTask.build_streaming_iterator(
+# data_path_and_name_and_type,
+# dtype=dtype,
+# batch_size=batch_size,
+# key_file=key_file,
+# num_workers=num_workers,
+# preprocess_fn=VADTask.build_preprocess_fn(speech2vadsegment.vad_infer_args, False),
+# collate_fn=VADTask.build_collate_fn(speech2vadsegment.vad_infer_args, False),
+# allow_variable_data_keys=allow_variable_data_keys,
+# inference=True,
+# )
+#
+# finish_count = 0
+# file_count = 1
+# # 7 .Start for-loop
+# # FIXME(kamo): The output format should be discussed about
+# if output_dir is not None:
+# writer = DatadirWriter(output_dir)
+# else:
+# writer = None
+#
+# vad_results = []
+# for keys, batch in loader:
+# assert isinstance(batch, dict), type(batch)
+# assert all(isinstance(s, str) for s in keys), keys
+# _bs = len(next(iter(batch.values())))
+# assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+# # batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+#
+# # do vad segment
+# results = speech2vadsegment(**batch)
+# for i, _ in enumerate(keys):
+# item = {'key': keys[i], 'value': results[i]}
+# vad_results.append(item)
+#
+# return vad_results
+
+
+def inference(
+ batch_size: int,
+ ngpu: int,
+ log_level: Union[int, str],
+ data_path_and_name_and_type,
+ vad_infer_config: Optional[str],
+ vad_model_file: Optional[str],
+ vad_cmvn_file: Optional[str] = None,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ key_file: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ num_workers: int = 1,
+ **kwargs,
+):
+ inference_pipeline = inference_modelscope(
+ batch_size=batch_size,
+ ngpu=ngpu,
+ log_level=log_level,
+ vad_infer_config=vad_infer_config,
+ vad_model_file=vad_model_file,
+ vad_cmvn_file=vad_cmvn_file,
+ key_file=key_file,
+ allow_variable_data_keys=allow_variable_data_keys,
+ output_dir=output_dir,
+ dtype=dtype,
+ seed=seed,
+ num_workers=num_workers,
+ **kwargs,
+ )
+ return inference_pipeline(data_path_and_name_and_type, raw_inputs)
+
+def inference_modelscope(
+ batch_size: int,
+ ngpu: int,
+ log_level: Union[int, str],
+ #data_path_and_name_and_type,
+ vad_infer_config: Optional[str],
+ vad_model_file: Optional[str],
+ vad_cmvn_file: Optional[str] = None,
+ # raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ key_file: Optional[str] = None,
+ allow_variable_data_keys: bool = False,
+ output_dir: Optional[str] = None,
+ dtype: str = "float32",
+ seed: int = 0,
+ num_workers: int = 1,
+ **kwargs,
+):
+ assert check_argument_types()
+ if batch_size > 1:
+ raise NotImplementedError("batch decoding is not implemented")
+ if ngpu > 1:
+ raise NotImplementedError("only single GPU decoding is supported")
+
+ logging.basicConfig(
+ level=log_level,
+ format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+ )
+
+ if ngpu >= 1 and torch.cuda.is_available():
+ device = "cuda"
+ else:
+ device = "cpu"
+
+ # 1. Set random-seed
+ set_all_random_seed(seed)
+
+ # 2. Build speech2vadsegment
+ speech2vadsegment_kwargs = dict(
+ vad_infer_config=vad_infer_config,
+ vad_model_file=vad_model_file,
+ vad_cmvn_file=vad_cmvn_file,
+ device=device,
+ dtype=dtype,
+ )
+ logging.info("speech2vadsegment_kwargs: {}".format(speech2vadsegment_kwargs))
+ speech2vadsegment = Speech2VadSegment(**speech2vadsegment_kwargs)
+
+ def _forward(
+ data_path_and_name_and_type,
+ raw_inputs: Union[np.ndarray, torch.Tensor] = None,
+ output_dir_v2: Optional[str] = None,
+ ):
+ # 3. Build data-iterator
+ loader = VADTask.build_streaming_iterator(
+ data_path_and_name_and_type,
+ dtype=dtype,
+ batch_size=batch_size,
+ key_file=key_file,
+ num_workers=num_workers,
+ preprocess_fn=VADTask.build_preprocess_fn(speech2vadsegment.vad_infer_args, False),
+ collate_fn=VADTask.build_collate_fn(speech2vadsegment.vad_infer_args, False),
+ allow_variable_data_keys=allow_variable_data_keys,
+ inference=True,
+ )
+
+ finish_count = 0
+ file_count = 1
+ # 7 .Start for-loop
+ # FIXME(kamo): The output format should be discussed about
+ output_path = output_dir_v2 if output_dir_v2 is not None else output_dir
+ if output_path is not None:
+ writer = DatadirWriter(output_path)
+ ibest_writer = writer[f"1best_recog"]
+ else:
+ writer = None
+ ibest_writer = None
+
+ vad_results = []
+ for keys, batch in loader:
+ assert isinstance(batch, dict), type(batch)
+ assert all(isinstance(s, str) for s in keys), keys
+ _bs = len(next(iter(batch.values())))
+ assert len(keys) == _bs, f"{len(keys)} != {_bs}"
+ # batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")}
+
+ # do vad segment
+ results = speech2vadsegment(**batch)
+ for i, _ in enumerate(keys):
+ item = {'key': keys[i], 'value': results[i]}
+ vad_results.append(item)
+ if writer is not None:
+ ibest_writer["text"][keys[i]] = "{}".format(results[i])
+
+ return vad_results
+
+ return _forward
+
+
+def get_parser():
+ parser = config_argparse.ArgumentParser(
+ description="VAD Decoding",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ )
+
+ # Note(kamo): Use '_' instead of '-' as separator.
+ # '-' is confusing if written in yaml.
+ parser.add_argument(
+ "--log_level",
+ type=lambda x: x.upper(),
+ default="INFO",
+ choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"),
+ help="The verbose level of logging",
+ )
+
+ parser.add_argument("--output_dir", type=str, required=False)
+ parser.add_argument(
+ "--ngpu",
+ type=int,
+ default=0,
+ help="The number of gpus. 0 indicates CPU mode",
+ )
+ parser.add_argument(
+ "--gpuid_list",
+ type=str,
+ default="",
+ help="The visible gpus",
+ )
+ parser.add_argument("--seed", type=int, default=0, help="Random seed")
+ parser.add_argument(
+ "--dtype",
+ default="float32",
+ choices=["float16", "float32", "float64"],
+ help="Data type",
+ )
+ parser.add_argument(
+ "--num_workers",
+ type=int,
+ default=1,
+ help="The number of workers used for DataLoader",
+ )
+
+ group = parser.add_argument_group("Input data related")
+ group.add_argument(
+ "--data_path_and_name_and_type",
+ type=str2triple_str,
+ required=False,
+ action="append",
+ )
+ group.add_argument("--raw_inputs", type=list, default=None)
+ # example=[{'key':'EdevDEWdIYQ_0021','file':'/mnt/data/jiangyu.xzy/test_data/speech_io/SPEECHIO_ASR_ZH00007_zhibodaihuo/wav/EdevDEWdIYQ_0021.wav'}])
+ group.add_argument("--key_file", type=str_or_none)
+ group.add_argument("--allow_variable_data_keys", type=str2bool, default=False)
+
+ group = parser.add_argument_group("The model configuration related")
+ group.add_argument(
+ "--vad_infer_config",
+ type=str,
+ help="VAD infer configuration",
+ )
+ group.add_argument(
+ "--vad_model_file",
+ type=str,
+ help="VAD model parameter file",
+ )
+ group.add_argument(
+ "--vad_cmvn_file",
+ type=str,
+ help="Global cmvn file",
+ )
+
+ group = parser.add_argument_group("infer related")
+ group.add_argument(
+ "--batch_size",
+ type=int,
+ default=1,
+ help="The batch size for inference",
+ )
+
+ return parser
+
+
+def main(cmd=None):
+ print(get_commandline_args(), file=sys.stderr)
+ parser = get_parser()
+ args = parser.parse_args(cmd)
+ kwargs = vars(args)
+ kwargs.pop("config", None)
+ inference(**kwargs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/datasets/iterable_dataset.py b/funasr/datasets/iterable_dataset.py
index 319dd7f..bed295b 100644
--- a/funasr/datasets/iterable_dataset.py
+++ b/funasr/datasets/iterable_dataset.py
@@ -15,6 +15,7 @@
import torch
from torch.utils.data.dataset import IterableDataset
from typeguard import check_argument_types
+import os.path
from funasr.datasets.dataset import ESPnetDataset
@@ -42,9 +43,27 @@
return array
+def load_bytes(input):
+ middle_data = np.frombuffer(input, dtype=np.int16)
+ middle_data = np.asarray(middle_data)
+ if middle_data.dtype.kind not in 'iu':
+ raise TypeError("'middle_data' must be an array of integers")
+ dtype = np.dtype('float32')
+ if dtype.kind != 'f':
+ raise TypeError("'dtype' must be a floating point type")
+
+ i = np.iinfo(middle_data.dtype)
+ abs_max = 2 ** (i.bits - 1)
+ offset = i.min + abs_max
+ array = np.frombuffer((middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32)
+ return array
+
+
DATA_TYPES = {
"sound": lambda x: soundfile.read(x)[0],
"kaldi_ark": load_kaldi,
+ "bytes": load_bytes,
+ "waveform": lambda x: x,
"npy": np.load,
"text_int": lambda x: np.loadtxt(
StringIO(x), ndmin=1, dtype=np.long, delimiter=" "
@@ -73,14 +92,14 @@
"""
def __init__(
- self,
- path_name_type_list: Collection[Tuple[str, str, str]],
- preprocess: Callable[
- [str, Dict[str, np.ndarray]], Dict[str, np.ndarray]
- ] = None,
- float_dtype: str = "float32",
- int_dtype: str = "long",
- key_file: str = None,
+ self,
+ path_name_type_list: Collection[Tuple[any, str, str]],
+ preprocess: Callable[
+ [str, Dict[str, np.ndarray]], Dict[str, np.ndarray]
+ ] = None,
+ float_dtype: str = "float32",
+ int_dtype: str = "long",
+ key_file: str = None,
):
assert check_argument_types()
if len(path_name_type_list) == 0:
@@ -99,14 +118,22 @@
non_iterable_list = []
self.path_name_type_list = []
- for path, name, _type in path_name_type_list:
- if name in self.debug_info:
- raise RuntimeError(f'"{name}" is duplicated for data-key')
+ if not isinstance(path_name_type_list[0], Tuple):
+ path = path_name_type_list[0]
+ name = path_name_type_list[1]
+ _type = path_name_type_list[2]
self.debug_info[name] = path, _type
if _type not in DATA_TYPES:
non_iterable_list.append((path, name, _type))
else:
self.path_name_type_list.append((path, name, _type))
+ else:
+ for path, name, _type in path_name_type_list:
+ self.debug_info[name] = path, _type
+ if _type not in DATA_TYPES:
+ non_iterable_list.append((path, name, _type))
+ else:
+ self.path_name_type_list.append((path, name, _type))
if len(non_iterable_list) != 0:
# Some types doesn't support iterable mode
@@ -119,10 +146,7 @@
else:
self.non_iterable_dataset = None
- if Path(Path(path_name_type_list[0][0]).parent, "utt2category").exists():
- self.apply_utt2category = True
- else:
- self.apply_utt2category = False
+ self.apply_utt2category = False
def has_name(self, name) -> bool:
return name in self.debug_info
@@ -139,99 +163,162 @@
return _mes
def __iter__(self) -> Iterator[Tuple[Union[str, int], Dict[str, np.ndarray]]]:
- if self.key_file is not None:
- uid_iter = (
- line.rstrip().split(maxsplit=1)[0]
- for line in open(self.key_file, encoding="utf-8")
- )
- elif len(self.path_name_type_list) != 0:
- uid_iter = (
- line.rstrip().split(maxsplit=1)[0]
- for line in open(self.path_name_type_list[0][0], encoding="utf-8")
- )
- else:
- uid_iter = iter(self.non_iterable_dataset)
-
- files = [open(lis[0], encoding="utf-8") for lis in self.path_name_type_list]
-
- worker_info = torch.utils.data.get_worker_info()
-
- linenum = 0
count = 0
- for count, uid in enumerate(uid_iter, 1):
- # If num_workers>=1, split keys
- if worker_info is not None:
- if (count - 1) % worker_info.num_workers != worker_info.id:
- continue
-
- # 1. Read a line from each file
- while True:
- keys = []
- values = []
- for f in files:
- linenum += 1
- try:
- line = next(f)
- except StopIteration:
- raise RuntimeError(f"{uid} is not found in the files")
- sps = line.rstrip().split(maxsplit=1)
- if len(sps) != 2:
- raise RuntimeError(
- f"This line doesn't include a space:"
- f" {f}:L{linenum}: {line})"
- )
- key, value = sps
- keys.append(key)
- values.append(value)
-
- for k_idx, k in enumerate(keys):
- if k != keys[0]:
- raise RuntimeError(
- f"Keys are mismatched. Text files (idx={k_idx}) is "
- f"not sorted or not having same keys at L{linenum}"
- )
-
- # If the key is matched, break the loop
- if len(keys) == 0 or keys[0] == uid:
- break
-
- # 2. Load the entry from each line and create a dict
+ if len(self.path_name_type_list) != 0 and (self.path_name_type_list[0][2] == "bytes" or self.path_name_type_list[0][2] == "waveform"):
data = {}
- # 2.a. Load data streamingly
- for value, (path, name, _type) in zip(values, self.path_name_type_list):
- func = DATA_TYPES[_type]
- # Load entry
- array = func(value)
- data[name] = array
- if self.non_iterable_dataset is not None:
- # 2.b. Load data from non-iterable dataset
- _, from_non_iterable = self.non_iterable_dataset[uid]
- data.update(from_non_iterable)
+ value = self.path_name_type_list[0][0]
+ uid = 'utt_id'
+ name = self.path_name_type_list[0][1]
+ _type = self.path_name_type_list[0][2]
+ func = DATA_TYPES[_type]
+ array = func(value)
+ data[name] = array
- # 3. [Option] Apply preprocessing
- # e.g. funasr.train.preprocessor:CommonPreprocessor
if self.preprocess is not None:
data = self.preprocess(uid, data)
-
- # 4. Force data-precision
for name in data:
+ count += 1
value = data[name]
if not isinstance(value, np.ndarray):
raise RuntimeError(
- f"All values must be converted to np.ndarray object "
- f'by preprocessing, but "{name}" is still {type(value)}.'
- )
-
+ f'All values must be converted to np.ndarray object '
+ f'by preprocessing, but "{name}" is still {type(value)}.')
# Cast to desired type
- if value.dtype.kind == "f":
+ if value.dtype.kind == 'f':
value = value.astype(self.float_dtype)
- elif value.dtype.kind == "i":
+ elif value.dtype.kind == 'i':
value = value.astype(self.int_dtype)
else:
- raise NotImplementedError(f"Not supported dtype: {value.dtype}")
+ raise NotImplementedError(
+ f'Not supported dtype: {value.dtype}')
data[name] = value
yield uid, data
+ elif len(self.path_name_type_list) != 0 and self.path_name_type_list[0][2] == "sound" and not self.path_name_type_list[0][0].lower().endswith(".scp"):
+ data = {}
+ value = self.path_name_type_list[0][0]
+ uid = os.path.basename(self.path_name_type_list[0][0]).split(".")[0]
+ name = self.path_name_type_list[0][1]
+ _type = self.path_name_type_list[0][2]
+ func = DATA_TYPES[_type]
+ array = func(value)
+ data[name] = array
+
+ if self.preprocess is not None:
+ data = self.preprocess(uid, data)
+ for name in data:
+ count += 1
+ value = data[name]
+ if not isinstance(value, np.ndarray):
+ raise RuntimeError(
+ f'All values must be converted to np.ndarray object '
+ f'by preprocessing, but "{name}" is still {type(value)}.')
+ # Cast to desired type
+ if value.dtype.kind == 'f':
+ value = value.astype(self.float_dtype)
+ elif value.dtype.kind == 'i':
+ value = value.astype(self.int_dtype)
+ else:
+ raise NotImplementedError(
+ f'Not supported dtype: {value.dtype}')
+ data[name] = value
+
+ yield uid, data
+
+ else:
+ if self.key_file is not None:
+ uid_iter = (
+ line.rstrip().split(maxsplit=1)[0]
+ for line in open(self.key_file, encoding="utf-8")
+ )
+ elif len(self.path_name_type_list) != 0:
+ uid_iter = (
+ line.rstrip().split(maxsplit=1)[0]
+ for line in open(self.path_name_type_list[0][0], encoding="utf-8")
+ )
+ else:
+ uid_iter = iter(self.non_iterable_dataset)
+
+ files = [open(lis[0], encoding="utf-8") for lis in self.path_name_type_list]
+
+ worker_info = torch.utils.data.get_worker_info()
+
+ linenum = 0
+ for count, uid in enumerate(uid_iter, 1):
+ # If num_workers>=1, split keys
+ if worker_info is not None:
+ if (count - 1) % worker_info.num_workers != worker_info.id:
+ continue
+
+ # 1. Read a line from each file
+ while True:
+ keys = []
+ values = []
+ for f in files:
+ linenum += 1
+ try:
+ line = next(f)
+ except StopIteration:
+ raise RuntimeError(f"{uid} is not found in the files")
+ sps = line.rstrip().split(maxsplit=1)
+ if len(sps) != 2:
+ raise RuntimeError(
+ f"This line doesn't include a space:"
+ f" {f}:L{linenum}: {line})"
+ )
+ key, value = sps
+ keys.append(key)
+ values.append(value)
+
+ for k_idx, k in enumerate(keys):
+ if k != keys[0]:
+ raise RuntimeError(
+ f"Keys are mismatched. Text files (idx={k_idx}) is "
+ f"not sorted or not having same keys at L{linenum}"
+ )
+
+ # If the key is matched, break the loop
+ if len(keys) == 0 or keys[0] == uid:
+ break
+
+ # 2. Load the entry from each line and create a dict
+ data = {}
+ # 2.a. Load data streamingly
+ for value, (path, name, _type) in zip(values, self.path_name_type_list):
+ func = DATA_TYPES[_type]
+ # Load entry
+ array = func(value)
+ data[name] = array
+ if self.non_iterable_dataset is not None:
+ # 2.b. Load data from non-iterable dataset
+ _, from_non_iterable = self.non_iterable_dataset[uid]
+ data.update(from_non_iterable)
+
+ # 3. [Option] Apply preprocessing
+ # e.g. funasr.train.preprocessor:CommonPreprocessor
+ if self.preprocess is not None:
+ data = self.preprocess(uid, data)
+
+ # 4. Force data-precision
+ for name in data:
+ value = data[name]
+ if not isinstance(value, np.ndarray):
+ raise RuntimeError(
+ f"All values must be converted to np.ndarray object "
+ f'by preprocessing, but "{name}" is still {type(value)}.'
+ )
+
+ # Cast to desired type
+ if value.dtype.kind == "f":
+ value = value.astype(self.float_dtype)
+ elif value.dtype.kind == "i":
+ value = value.astype(self.int_dtype)
+ else:
+ raise NotImplementedError(f"Not supported dtype: {value.dtype}")
+ data[name] = value
+
+ yield uid, data
+
if count == 0:
raise RuntimeError("No iteration")
diff --git a/funasr/datasets/large_datasets/build_dataloader.py b/funasr/datasets/large_datasets/build_dataloader.py
index 37fbb7c..146723d 100644
--- a/funasr/datasets/large_datasets/build_dataloader.py
+++ b/funasr/datasets/large_datasets/build_dataloader.py
@@ -21,15 +21,28 @@
symbol_table[char] = i
return symbol_table
+def load_seg_dict(seg_dict_file):
+ seg_dict = {}
+ assert isinstance(seg_dict_file, str)
+ with open(seg_dict_file, "r", encoding="utf8") as f:
+ lines = f.readlines()
+ for line in lines:
+ s = line.strip().split()
+ key = s[0]
+ value = s[1:]
+ seg_dict[key] = " ".join(value)
+ return seg_dict
class ArkDataLoader(AbsIterFactory):
- def __init__(self, data_list, dict_file, config_file, mode="train"):
+ def __init__(self, data_list, dict_file, dataset_conf, seg_dict_file=None, mode="train"):
symbol_table = read_symbol_table(dict_file)
- with open(config_file, "r") as fin:
- configs = yaml.load(fin, Loader=yaml.FullLoader)
- self.dataset_conf = configs["dataset_conf"]
+ if seg_dict_file is not None:
+ seg_dict = load_seg_dict(seg_dict_file)
+ else:
+ seg_dict = None
+ self.dataset_conf = dataset_conf
logging.info("dataloader config: {}".format(self.dataset_conf))
- self.dataset = Dataset(data_list, symbol_table,
+ self.dataset = Dataset(data_list, symbol_table, seg_dict,
self.dataset_conf, mode=mode)
def build_iter(self, epoch, shuffle=True):
diff --git a/funasr/datasets/large_datasets/dataset.py b/funasr/datasets/large_datasets/dataset.py
index 60c5abd..41d34ab 100644
--- a/funasr/datasets/large_datasets/dataset.py
+++ b/funasr/datasets/large_datasets/dataset.py
@@ -1,5 +1,6 @@
import os
import random
+import soundfile
from functools import partial
import torch
@@ -97,7 +98,7 @@
if data_type == "kaldi_ark":
ark_reader = ReadHelper('ark:{}'.format(data_file))
reader_list.append(ark_reader)
- elif data_type == "text":
+ elif data_type == "text" or data_type == "sound":
text_reader = open(data_file, "r")
reader_list.append(text_reader)
else:
@@ -109,6 +110,13 @@
if data_type == "kaldi_ark":
key, mat = item
sample_dict[data_name] = mat
+ if data_name == "speech":
+ sample_dict["key"] = key
+ elif data_type == "sound":
+ key, path = item.strip().split()
+ mat, sampling_rate = soundfile.read(path)
+ sample_dict[data_name] = mat
+ sample_dict["sampling_rate"] = sampling_rate
if data_name == "speech":
sample_dict["key"] = key
else:
@@ -125,11 +133,15 @@
def len_fn_token(data):
assert "speech" in data
- return data["speech"].shape[0]
+ if "sampling_rate" in data:
+ return (data["speech"].shape[0] / data["sampling_rate"]) * 1000.
+ else:
+ return data["speech"].shape[0]
def Dataset(data_list_file,
dict,
+ seg_dict,
conf,
mode="train"):
scp_lists = read_lists(data_list_file)
@@ -142,7 +154,7 @@
filter_fn = partial(filter, **filter_conf)
dataset = FilterIterDataPipe(dataset, fn=filter_fn)
- vocab = {'vocab': dict}
+ vocab = {'vocab': dict, 'seg_dict': seg_dict}
tokenize_fn = partial(tokenize, **vocab)
dataset = MapperIterDataPipe(dataset, fn=tokenize_fn)
diff --git a/funasr/datasets/large_datasets/utils/filter.py b/funasr/datasets/large_datasets/utils/filter.py
index 5dc911f..91ba4be 100644
--- a/funasr/datasets/large_datasets/utils/filter.py
+++ b/funasr/datasets/large_datasets/utils/filter.py
@@ -2,14 +2,17 @@
def filter(data,
- min_length=10,
- max_length=10000,
- min_token_length=0,
- max_token_length=200):
+ speech_length_min=100,
+ speech_length_max=15000,
+ token_length_min=0,
+ token_length_max=200):
assert "speech" in data
assert "text" in data
- num_frames = data["speech"].shape[0]
+ if "sampling_rate" in data:
+ speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.
+ else:
+ speech_length = data["speech"].shape[0]
num_tokens = len(data['text'])
- return min_length < num_frames < max_length and min_token_length < num_tokens < max_token_length
\ No newline at end of file
+ return speech_length_min < speech_length < speech_length_max and token_length_min < num_tokens < token_length_max
diff --git a/funasr/datasets/large_datasets/utils/padding.py b/funasr/datasets/large_datasets/utils/padding.py
index 2e91e78..e814b1c 100644
--- a/funasr/datasets/large_datasets/utils/padding.py
+++ b/funasr/datasets/large_datasets/utils/padding.py
@@ -14,7 +14,7 @@
batch = {}
data_names = data[0].keys()
for data_name in data_names:
- if data_name == "key":
+ if data_name == "key" or data_name =="sampling_rate":
continue
else:
if data[0][data_name].dtype.kind == "i":
diff --git a/funasr/datasets/large_datasets/utils/tokenize.py b/funasr/datasets/large_datasets/utils/tokenize.py
index 937e144..0c01885 100644
--- a/funasr/datasets/large_datasets/utils/tokenize.py
+++ b/funasr/datasets/large_datasets/utils/tokenize.py
@@ -1,12 +1,47 @@
#!/usr/bin/env python
+import re
import numpy as np
+def forward_segment(text, seg_dict):
+ word_list = []
+ i = 0
+ while i < len(text):
+ longest_word = text[i]
+ for j in range(i + 1, len(text) + 1):
+ word = text[i:j]
+ if word in seg_dict:
+ if len(word) > len(longest_word):
+ longest_word = word
+ word_list.append(longest_word)
+ i += len(longest_word)
+ return word_list
+
+def seg_tokenize(txt, seg_dict):
+ out_txt = ""
+ pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
+ for word in txt:
+ if pattern.match(word):
+ if word in seg_dict:
+ out_txt += seg_dict[word] + " "
+ else:
+ out_txt += "<unk>" + " "
+ else:
+ continue
+ return out_txt.strip().split()
+
def tokenize(data,
- vocab=None):
+ vocab=None,
+ seg_dict=None):
assert "text" in data
assert isinstance(vocab, dict)
text = data["text"]
token = []
+
+ if seg_dict is not None:
+ assert isinstance(seg_dict, dict)
+ txt = forward_segment("".join(text).lower(), seg_dict)
+ text = seg_tokenize(txt, seg_dict)
+
for x in text:
if x in vocab:
token.append(vocab[x])
diff --git a/funasr/datasets/ms_dataset.py b/funasr/datasets/ms_dataset.py
new file mode 100644
index 0000000..ae3c1bb
--- /dev/null
+++ b/funasr/datasets/ms_dataset.py
@@ -0,0 +1,33 @@
+import os
+
+
+class MsDataset(object):
+ @classmethod
+ def load_core(cls, data_dir, data_set):
+ wav_file = os.path.join(data_dir, data_set, "wav.scp")
+ text_file = os.path.join(data_dir, data_set, "text")
+ with open(wav_file) as f:
+ wav_lines = f.readlines()
+ with open(text_file) as f:
+ text_lines = f.readlines()
+ data_list = []
+ for wav_line, text_line in zip(wav_lines, text_lines):
+ item = {}
+ item["Audio:FILE"] = wav_line.strip().split()[-1]
+ item["Text:LABEL"] = " ".join(text_line.strip().split()[1:])
+ data_list.append(item)
+ return data_list
+
+ @classmethod
+ def load(cls, dataset_name, namespace="speech_asr", train_set="train", dev_set="validation"):
+ if os.path.exists(dataset_name):
+ data_dir = dataset_name
+ ds_dict = {}
+ ds_dict["train"] = cls.load_core(data_dir, train_set)
+ ds_dict["validation"] = cls.load_core(data_dir, dev_set)
+ ds_dict["raw_data_dir"] = data_dir
+ return ds_dict
+ else:
+ from modelscope.msdatasets import MsDataset
+ ds_dict = MsDataset.load(dataset_name=dataset_name, namespace=namespace)
+ return ds_dict
diff --git a/funasr/datasets/preprocessor.py b/funasr/datasets/preprocessor.py
index 80d1adc..75bee86 100644
--- a/funasr/datasets/preprocessor.py
+++ b/funasr/datasets/preprocessor.py
@@ -1,3 +1,4 @@
+import re
from abc import ABC
from abc import abstractmethod
from pathlib import Path
@@ -24,17 +25,46 @@
@abstractmethod
def __call__(
- self, uid: str, data: Dict[str, Union[str, np.ndarray]]
+ self, uid: str, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
raise NotImplementedError
+def forward_segment(text, dic):
+ word_list = []
+ i = 0
+ while i < len(text):
+ longest_word = text[i]
+ for j in range(i + 1, len(text) + 1):
+ word = text[i:j]
+ if word in dic:
+ if len(word) > len(longest_word):
+ longest_word = word
+ word_list.append(longest_word)
+ i += len(longest_word)
+ return word_list
+
+
+def seg_tokenize(txt, seg_dict):
+ out_txt = ""
+ pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
+ for word in txt:
+ if pattern.match(word):
+ if word in seg_dict:
+ out_txt += seg_dict[word] + " "
+ else:
+ out_txt += "<unk>" + " "
+ else:
+ continue
+ return out_txt.strip().split()
+
+
def framing(
- x,
- frame_length: int = 512,
- frame_shift: int = 256,
- centered: bool = True,
- padded: bool = True,
+ x,
+ frame_length: int = 512,
+ frame_shift: int = 256,
+ centered: bool = True,
+ padded: bool = True,
):
if x.size == 0:
raise ValueError("Input array size is zero")
@@ -73,11 +103,11 @@
def detect_non_silence(
- x: np.ndarray,
- threshold: float = 0.01,
- frame_length: int = 1024,
- frame_shift: int = 512,
- window: str = "boxcar",
+ x: np.ndarray,
+ threshold: float = 0.01,
+ frame_length: int = 1024,
+ frame_shift: int = 512,
+ window: str = "boxcar",
) -> np.ndarray:
"""Power based voice activity detection.
@@ -103,7 +133,7 @@
)
framed_w *= scipy.signal.get_window(window, frame_length).astype(framed_w.dtype)
# power: (C, T)
- power = (framed_w**2).mean(axis=-1)
+ power = (framed_w ** 2).mean(axis=-1)
# mean_power: (C, 1)
mean_power = np.mean(power, axis=-1, keepdims=True)
if np.all(mean_power == 0):
@@ -126,26 +156,27 @@
class CommonPreprocessor(AbsPreprocessor):
def __init__(
- self,
- train: bool,
- token_type: str = None,
- token_list: Union[Path, str, Iterable[str]] = None,
- bpemodel: Union[Path, str, Iterable[str]] = None,
- text_cleaner: Collection[str] = None,
- g2p_type: str = None,
- unk_symbol: str = "<unk>",
- space_symbol: str = "<space>",
- non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
- delimiter: str = None,
- rir_scp: str = None,
- rir_apply_prob: float = 1.0,
- noise_scp: str = None,
- noise_apply_prob: float = 1.0,
- noise_db_range: str = "3_10",
- speech_volume_normalize: float = None,
- speech_name: str = "speech",
- text_name: str = "text",
- split_with_space: bool = False,
+ self,
+ train: bool,
+ token_type: str = None,
+ token_list: Union[Path, str, Iterable[str]] = None,
+ bpemodel: Union[Path, str, Iterable[str]] = None,
+ text_cleaner: Collection[str] = None,
+ g2p_type: str = None,
+ unk_symbol: str = "<unk>",
+ space_symbol: str = "<space>",
+ non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
+ delimiter: str = None,
+ rir_scp: str = None,
+ rir_apply_prob: float = 1.0,
+ noise_scp: str = None,
+ noise_apply_prob: float = 1.0,
+ noise_db_range: str = "3_10",
+ speech_volume_normalize: float = None,
+ speech_name: str = "speech",
+ text_name: str = "text",
+ split_with_space: bool = False,
+ seg_dict_file: str = None,
):
super().__init__(train)
self.train = train
@@ -155,6 +186,16 @@
self.rir_apply_prob = rir_apply_prob
self.noise_apply_prob = noise_apply_prob
self.split_with_space = split_with_space
+ self.seg_dict = None
+ if seg_dict_file is not None:
+ self.seg_dict = {}
+ with open(seg_dict_file) as f:
+ lines = f.readlines()
+ for line in lines:
+ s = line.strip().split()
+ key = s[0]
+ value = s[1:]
+ self.seg_dict[key] = " ".join(value)
if token_type is not None:
if token_list is None:
@@ -212,7 +253,7 @@
self.noises = None
def _speech_process(
- self, data: Dict[str, Union[str, np.ndarray]]
+ self, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, Union[str, np.ndarray]]:
assert check_argument_types()
if self.speech_name in data:
@@ -242,16 +283,16 @@
# speech: (Nmic, Time)
# Note that this operation doesn't change the signal length
speech = scipy.signal.convolve(speech, rir, mode="full")[
- :, : speech.shape[1]
- ]
+ :, : speech.shape[1]
+ ]
# Reverse mean power to the original power
power2 = (speech[detect_non_silence(speech)] ** 2).mean()
speech = np.sqrt(power / max(power2, 1e-10)) * speech
# 2. Add Noise
if (
- self.noises is not None
- and self.noise_apply_prob >= np.random.random()
+ self.noises is not None
+ and self.noise_apply_prob >= np.random.random()
):
noise_path = np.random.choice(self.noises)
if noise_path is not None:
@@ -283,11 +324,11 @@
# noise: (Nmic, Time)
noise = noise.T
- noise_power = (noise**2).mean()
+ noise_power = (noise ** 2).mean()
scale = (
- 10 ** (-noise_db / 20)
- * np.sqrt(power)
- / np.sqrt(max(noise_power, 1e-10))
+ 10 ** (-noise_db / 20)
+ * np.sqrt(power)
+ / np.sqrt(max(noise_power, 1e-10))
)
speech = speech + scale * noise
@@ -305,13 +346,16 @@
return data
def _text_process(
- self, data: Dict[str, Union[str, np.ndarray]]
+ self, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
if self.text_name in data and self.tokenizer is not None:
text = data[self.text_name]
text = self.text_cleaner(text)
if self.split_with_space:
tokens = text.strip().split(" ")
+ if self.seg_dict is not None:
+ tokens = forward_segment("".join(tokens).lower(), self.seg_dict)
+ tokens = seg_tokenize(tokens, self.seg_dict)
else:
tokens = self.tokenizer.text2tokens(text)
text_ints = self.token_id_converter.tokens2ids(tokens)
@@ -320,7 +364,7 @@
return data
def __call__(
- self, uid: str, data: Dict[str, Union[str, np.ndarray]]
+ self, uid: str, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
assert check_argument_types()
@@ -331,19 +375,19 @@
class CommonPreprocessor_multi(AbsPreprocessor):
def __init__(
- self,
- train: bool,
- token_type: str = None,
- token_list: Union[Path, str, Iterable[str]] = None,
- bpemodel: Union[Path, str, Iterable[str]] = None,
- text_cleaner: Collection[str] = None,
- g2p_type: str = None,
- unk_symbol: str = "<unk>",
- space_symbol: str = "<space>",
- non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
- delimiter: str = None,
- speech_name: str = "speech",
- text_name: List[str] = ["text"],
+ self,
+ train: bool,
+ token_type: str = None,
+ token_list: Union[Path, str, Iterable[str]] = None,
+ bpemodel: Union[Path, str, Iterable[str]] = None,
+ text_cleaner: Collection[str] = None,
+ g2p_type: str = None,
+ unk_symbol: str = "<unk>",
+ space_symbol: str = "<space>",
+ non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
+ delimiter: str = None,
+ speech_name: str = "speech",
+ text_name: List[str] = ["text"],
):
super().__init__(train)
self.train = train
@@ -373,7 +417,7 @@
self.token_id_converter = None
def _text_process(
- self, data: Dict[str, Union[str, np.ndarray]]
+ self, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
for text_n in self.text_name:
if text_n in data and self.tokenizer is not None:
@@ -386,7 +430,7 @@
return data
def __call__(
- self, uid: str, data: Dict[str, Union[str, np.ndarray]]
+ self, uid: str, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
assert check_argument_types()
@@ -404,25 +448,25 @@
class MutliTokenizerCommonPreprocessor(CommonPreprocessor):
def __init__(
- self,
- train: bool,
- token_type: List[str] = [None],
- token_list: List[Union[Path, str, Iterable[str]]] = [None],
- bpemodel: List[Union[Path, str, Iterable[str]]] = [None],
- text_cleaner: Collection[str] = None,
- g2p_type: str = None,
- unk_symbol: str = "<unk>",
- space_symbol: str = "<space>",
- non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
- delimiter: str = None,
- rir_scp: str = None,
- rir_apply_prob: float = 1.0,
- noise_scp: str = None,
- noise_apply_prob: float = 1.0,
- noise_db_range: str = "3_10",
- speech_volume_normalize: float = None,
- speech_name: str = "speech",
- text_name: List[str] = ["text"],
+ self,
+ train: bool,
+ token_type: List[str] = [None],
+ token_list: List[Union[Path, str, Iterable[str]]] = [None],
+ bpemodel: List[Union[Path, str, Iterable[str]]] = [None],
+ text_cleaner: Collection[str] = None,
+ g2p_type: str = None,
+ unk_symbol: str = "<unk>",
+ space_symbol: str = "<space>",
+ non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
+ delimiter: str = None,
+ rir_scp: str = None,
+ rir_apply_prob: float = 1.0,
+ noise_scp: str = None,
+ noise_apply_prob: float = 1.0,
+ noise_db_range: str = "3_10",
+ speech_volume_normalize: float = None,
+ speech_name: str = "speech",
+ text_name: List[str] = ["text"],
):
# TODO(jiatong): sync with Kamo and Jing on interface for preprocessor
super().__init__(
@@ -447,7 +491,7 @@
)
assert (
- len(token_type) == len(token_list) == len(bpemodel) == len(text_name)
+ len(token_type) == len(token_list) == len(bpemodel) == len(text_name)
), "token_type, token_list, bpemodel, or processing text_name mismatched"
self.num_tokenizer = len(token_type)
self.tokenizer = []
@@ -482,7 +526,7 @@
self.text_name = text_name # override the text_name from CommonPreprocessor
def _text_process(
- self, data: Dict[str, Union[str, np.ndarray]]
+ self, data: Dict[str, Union[str, np.ndarray]]
) -> Dict[str, np.ndarray]:
for i in range(self.num_tokenizer):
text_name = self.text_name[i]
diff --git a/funasr/layers/stft.py b/funasr/layers/stft.py
index 21beaae..376b5a3 100644
--- a/funasr/layers/stft.py
+++ b/funasr/layers/stft.py
@@ -42,7 +42,8 @@
self.normalized = normalized
self.onesided = onesided
if window is not None and not hasattr(torch, f"{window}_window"):
- raise ValueError(f"{window} window is not implemented")
+ if window.lower() != "povey":
+ raise ValueError(f"{window} window is not implemented")
self.window = window
def extra_repr(self):
@@ -83,10 +84,14 @@
# output: (Batch, Freq, Frames, 2=real_imag)
# or (Batch, Channel, Freq, Frames, 2=real_imag)
if self.window is not None:
- window_func = getattr(torch, f"{self.window}_window")
- window = window_func(
- self.win_length, dtype=input.dtype, device=input.device
- )
+ if self.window.lower() == "povey":
+ window = torch.hann_window(self.win_length, periodic=False,
+ device=input.device, dtype=input.dtype).pow(0.85)
+ else:
+ window_func = getattr(torch, f"{self.window}_window")
+ window = window_func(
+ self.win_length, dtype=input.dtype, device=input.device
+ )
else:
window = None
diff --git a/funasr/models/decoder/sanm_decoder.py b/funasr/models/decoder/sanm_decoder.py
index a5db353..ab03f0b 100644
--- a/funasr/models/decoder/sanm_decoder.py
+++ b/funasr/models/decoder/sanm_decoder.py
@@ -1,8 +1,10 @@
from typing import List
from typing import Tuple
-
+import logging
import torch
import torch.nn as nn
+import numpy as np
+
from funasr.modules.streaming_utils import utils as myutils
from funasr.models.decoder.transformer_decoder import BaseTransformerDecoder
from typeguard import check_argument_types
@@ -136,6 +138,9 @@
sanm_shfit: int = None,
concat_embeds: bool = False,
attention_dim: int = None,
+ tf2torch_tensor_name_prefix_torch: str = "decoder",
+ tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
+ embed_tensor_name_prefix_tf: str = None,
):
assert check_argument_types()
super().__init__(
@@ -241,6 +246,9 @@
else:
self.embed_concat_ffn = None
self.concat_embeds = concat_embeds
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
+ self.embed_tensor_name_prefix_tf = embed_tensor_name_prefix_tf
def forward(
self,
@@ -382,6 +390,387 @@
return y, new_cache
+ def gen_tf2torch_map_dict(self):
+
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ embed_tensor_name_prefix_tf = self.embed_tensor_name_prefix_tf if self.embed_tensor_name_prefix_tf is not None else tensor_name_prefix_tf
+ map_dict_local = {
+
+ ## decoder
+ # ffn
+ "{}.decoders.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # fsmn
+ "{}.decoders.layeridx.norm2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/gamma".format(
+ tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/beta".format(
+ tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.self_attn.fsmn_block.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/depth_conv_w".format(
+ tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 2, 0),
+ }, # (256,1,31),(1,31,256,1)
+ # src att
+ "{}.decoders.layeridx.norm3.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm3.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.src_attn.linear_q.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.decoders.layeridx.src_attn.linear_q.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.src_attn.linear_k_v.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders.layeridx.src_attn.linear_k_v.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.src_attn.linear_out.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.decoders.layeridx.src_attn.linear_out.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # dnn
+ "{}.decoders3.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders3.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders3.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders3.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # embed_concat_ffn
+ "{}.embed_concat_ffn.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.embed_concat_ffn.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # out norm
+ "{}.after_norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.after_norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+
+ # in embed
+ "{}.embed.0.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/w_embs".format(embed_tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (4235,256),(4235,256)
+
+ # out layer
+ "{}.output_layer.weight".format(tensor_name_prefix_torch):
+ {"name": ["{}/dense/kernel".format(tensor_name_prefix_tf),
+ "{}/w_embs".format(embed_tensor_name_prefix_tf)],
+ "squeeze": [None, None],
+ "transpose": [(1, 0), None],
+ }, # (4235,256),(256,4235)
+ "{}.output_layer.bias".format(tensor_name_prefix_torch):
+ {"name": ["{}/dense/bias".format(tensor_name_prefix_tf),
+ "seq2seq/2bias" if tensor_name_prefix_tf == "seq2seq/decoder/inputter_1" else "seq2seq/bias"],
+ "squeeze": [None, None],
+ "transpose": [None, None],
+ }, # (4235,),(4235,)
+
+ }
+ return map_dict_local
+
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+
+ map_dict = self.gen_tf2torch_map_dict()
+ var_dict_torch_update = dict()
+ decoder_layeridx_sets = set()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ if names[1] == "decoders":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "decoders2":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ name_q = name_q.replace("decoders2", "decoders")
+ layeridx_bias = len(decoder_layeridx_sets)
+
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "decoders3":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "embed" or names[1] == "output_layer":
+ name_tf = map_dict[name]["name"]
+ if isinstance(name_tf, list):
+ idx_list = 0
+ if name_tf[idx_list] in var_dict_tf.keys():
+ pass
+ else:
+ idx_list = 1
+ data_tf = var_dict_tf[name_tf[idx_list]]
+ if map_dict[name]["squeeze"][idx_list] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"][idx_list])
+ if map_dict[name]["transpose"][idx_list] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"][idx_list])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info("torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(),
+ name_tf[idx_list],
+ var_dict_tf[name_tf[
+ idx_list]].shape))
+
+ else:
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
+ if map_dict[name]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "after_norm":
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "embed_concat_ffn":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ return var_dict_torch_update
+
+
class ParaformerSANMDecoder(BaseTransformerDecoder):
"""
author: Speech Lab, Alibaba Group, China
@@ -407,6 +796,8 @@
att_layer_num: int = 6,
kernel_size: int = 21,
sanm_shfit: int = 0,
+ tf2torch_tensor_name_prefix_torch: str = "decoder",
+ tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
):
assert check_argument_types()
super().__init__(
@@ -496,6 +887,8 @@
concat_after,
),
)
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def forward(
self,
@@ -613,4 +1006,381 @@
if self.output_layer is not None:
y = torch.log_softmax(self.output_layer(y), dim=-1)
- return y, new_cache
\ No newline at end of file
+ return y, new_cache
+
+ def gen_tf2torch_map_dict(self):
+
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ map_dict_local = {
+
+ ## decoder
+ # ffn
+ "{}.decoders.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_ffn/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # fsmn
+ "{}.decoders.layeridx.norm2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/gamma".format(
+ tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/LayerNorm/beta".format(
+ tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.self_attn.fsmn_block.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/decoder_memory_block/depth_conv_w".format(
+ tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 2, 0),
+ }, # (256,1,31),(1,31,256,1)
+ # src att
+ "{}.decoders.layeridx.norm3.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.norm3.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.src_attn.linear_q.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.decoders.layeridx.src_attn.linear_q.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders.layeridx.src_attn.linear_k_v.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders.layeridx.src_attn.linear_k_v.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders.layeridx.src_attn.linear_out.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.decoders.layeridx.src_attn.linear_out.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_fsmn_layer_layeridx/multi_head/conv1d_2/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # dnn
+ "{}.decoders3.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders3.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.decoders3.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.decoders3.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.decoders3.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/decoder_dnn_layer_layeridx/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # embed_concat_ffn
+ "{}.embed_concat_ffn.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.embed_concat_ffn.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm_1/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/LayerNorm_1/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.embed_concat_ffn.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/cif_concat/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+
+ # out norm
+ "{}.after_norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.after_norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+
+ # in embed
+ "{}.embed.0.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/w_embs".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (4235,256),(4235,256)
+
+ # out layer
+ "{}.output_layer.weight".format(tensor_name_prefix_torch):
+ {"name": ["{}/dense/kernel".format(tensor_name_prefix_tf), "{}/w_embs".format(tensor_name_prefix_tf)],
+ "squeeze": [None, None],
+ "transpose": [(1, 0), None],
+ }, # (4235,256),(256,4235)
+ "{}.output_layer.bias".format(tensor_name_prefix_torch):
+ {"name": ["{}/dense/bias".format(tensor_name_prefix_tf),
+ "seq2seq/2bias" if tensor_name_prefix_tf == "seq2seq/decoder/inputter_1" else "seq2seq/bias"],
+ "squeeze": [None, None],
+ "transpose": [None, None],
+ }, # (4235,),(4235,)
+
+ }
+ return map_dict_local
+
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+ map_dict = self.gen_tf2torch_map_dict()
+ var_dict_torch_update = dict()
+ decoder_layeridx_sets = set()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ if names[1] == "decoders":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "decoders2":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ name_q = name_q.replace("decoders2", "decoders")
+ layeridx_bias = len(decoder_layeridx_sets)
+
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "decoders3":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "embed" or names[1] == "output_layer":
+ name_tf = map_dict[name]["name"]
+ if isinstance(name_tf, list):
+ idx_list = 0
+ if name_tf[idx_list] in var_dict_tf.keys():
+ pass
+ else:
+ idx_list = 1
+ data_tf = var_dict_tf[name_tf[idx_list]]
+ if map_dict[name]["squeeze"][idx_list] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"][idx_list])
+ if map_dict[name]["transpose"][idx_list] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"][idx_list])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info("torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(),
+ name_tf[idx_list],
+ var_dict_tf[name_tf[
+ idx_list]].shape))
+
+ else:
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
+ if map_dict[name]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "after_norm":
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "embed_concat_ffn":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if "decoders." in name:
+ decoder_layeridx_sets.add(layeridx)
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ return var_dict_torch_update
diff --git a/funasr/models/decoder/sv_decoder.py b/funasr/models/decoder/sv_decoder.py
new file mode 100644
index 0000000..ee6b0e5
--- /dev/null
+++ b/funasr/models/decoder/sv_decoder.py
@@ -0,0 +1,37 @@
+import torch
+from torch.nn import functional as F
+from funasr.models.decoder.abs_decoder import AbsDecoder
+
+
+class DenseDecoder(AbsDecoder):
+ def __init__(
+ self,
+ vocab_size,
+ encoder_output_size,
+ num_nodes_resnet1: int = 256,
+ num_nodes_last_layer: int = 256,
+ batchnorm_momentum: float = 0.5,
+ ):
+ super(DenseDecoder, self).__init__()
+ self.resnet1_dense = torch.nn.Linear(encoder_output_size, num_nodes_resnet1)
+ self.resnet1_bn = torch.nn.BatchNorm1d(num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum)
+
+ self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer)
+ self.resnet2_bn = torch.nn.BatchNorm1d(num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum)
+
+ self.output_dense = torch.nn.Linear(num_nodes_last_layer, vocab_size, bias=False)
+
+ def forward(self, features):
+ embeddings = {}
+ features = self.resnet1_dense(features)
+ embeddings["resnet1_dense"] = features
+ features = F.relu(features)
+ features = self.resnet1_bn(features)
+
+ features = self.resnet2_dense(features)
+ embeddings["resnet2_dense"] = features
+ features = F.relu(features)
+ features = self.resnet2_bn(features)
+
+ features = self.output_dense(features)
+ return features, embeddings
diff --git a/funasr/models/e2e_asr_paraformer.py b/funasr/models/e2e_asr_paraformer.py
index 3f8359d..65c70df 100644
--- a/funasr/models/e2e_asr_paraformer.py
+++ b/funasr/models/e2e_asr_paraformer.py
@@ -12,7 +12,7 @@
from funasr.layers.abs_normalize import AbsNormalize
from funasr.losses.label_smoothing_loss import (
- LabelSmoothingLoss, # noqa: H301
+ LabelSmoothingLoss, # noqa: H301
)
from funasr.models.ctc import CTC
from funasr.models.decoder.abs_decoder import AbsDecoder
@@ -20,800 +20,1046 @@
from funasr.models.encoder.abs_encoder import AbsEncoder
from funasr.models.frontend.abs_frontend import AbsFrontend
from funasr.models.postencoder.abs_postencoder import AbsPostEncoder
+from funasr.models.predictor.cif import mae_loss
from funasr.models.preencoder.abs_preencoder import AbsPreEncoder
from funasr.models.specaug.abs_specaug import AbsSpecAug
from funasr.modules.add_sos_eos import add_sos_eos
from funasr.modules.nets_utils import make_pad_mask
from funasr.modules.nets_utils import th_accuracy
-from funasr.models.predictor.cif import mae_loss
from funasr.torch_utils.device_funcs import force_gatherable
from funasr.train.abs_espnet_model import AbsESPnetModel
+from funasr.models.predictor.cif import CifPredictorV3
+
if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):
- from torch.cuda.amp import autocast
+ from torch.cuda.amp import autocast
else:
- # Nothing to do if torch<1.6.0
- @contextmanager
- def autocast(enabled=True):
- yield
+ # Nothing to do if torch<1.6.0
+ @contextmanager
+ def autocast(enabled=True):
+ yield
+
class Paraformer(AbsESPnetModel):
- """
- Author: Speech Lab, Alibaba Group, China
- Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
- https://arxiv.org/abs/2206.08317
- """
+ """
+ Author: Speech Lab, Alibaba Group, China
+ Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
+ https://arxiv.org/abs/2206.08317
+ """
- def __init__(
- self,
- vocab_size: int,
- token_list: Union[Tuple[str, ...], List[str]],
- frontend: Optional[AbsFrontend],
- specaug: Optional[AbsSpecAug],
- normalize: Optional[AbsNormalize],
- preencoder: Optional[AbsPreEncoder],
- encoder: AbsEncoder,
- postencoder: Optional[AbsPostEncoder],
- decoder: AbsDecoder,
- ctc: CTC,
- ctc_weight: float = 0.5,
- interctc_weight: float = 0.0,
- ignore_id: int = -1,
- blank_id: int = 0,
- sos: int = 1,
- eos: int = 2,
- lsm_weight: float = 0.0,
- length_normalized_loss: bool = False,
- report_cer: bool = True,
- report_wer: bool = True,
- sym_space: str = "<space>",
- sym_blank: str = "<blank>",
- extract_feats_in_collect_stats: bool = True,
- predictor = None,
- predictor_weight: float = 0.0,
- predictor_bias: int = 0,
- sampling_ratio: float = 0.2,
+ def __init__(
+ self,
+ vocab_size: int,
+ token_list: Union[Tuple[str, ...], List[str]],
+ frontend: Optional[AbsFrontend],
+ specaug: Optional[AbsSpecAug],
+ normalize: Optional[AbsNormalize],
+ preencoder: Optional[AbsPreEncoder],
+ encoder: AbsEncoder,
+ postencoder: Optional[AbsPostEncoder],
+ decoder: AbsDecoder,
+ ctc: CTC,
+ ctc_weight: float = 0.5,
+ interctc_weight: float = 0.0,
+ ignore_id: int = -1,
+ blank_id: int = 0,
+ sos: int = 1,
+ eos: int = 2,
+ lsm_weight: float = 0.0,
+ length_normalized_loss: bool = False,
+ report_cer: bool = True,
+ report_wer: bool = True,
+ sym_space: str = "<space>",
+ sym_blank: str = "<blank>",
+ extract_feats_in_collect_stats: bool = True,
+ predictor=None,
+ predictor_weight: float = 0.0,
+ predictor_bias: int = 0,
+ sampling_ratio: float = 0.2,
+ share_embedding: bool = False,
+ ):
+ assert check_argument_types()
+ assert 0.0 <= ctc_weight <= 1.0, ctc_weight
+ assert 0.0 <= interctc_weight < 1.0, interctc_weight
- ):
- assert check_argument_types()
- assert 0.0 <= ctc_weight <= 1.0, ctc_weight
- assert 0.0 <= interctc_weight < 1.0, interctc_weight
+ super().__init__()
+ # note that eos is the same as sos (equivalent ID)
+ self.blank_id = blank_id
+ self.sos = vocab_size - 1 if sos is None else sos
+ self.eos = vocab_size - 1 if eos is None else eos
+ self.vocab_size = vocab_size
+ self.ignore_id = ignore_id
+ self.ctc_weight = ctc_weight
+ self.interctc_weight = interctc_weight
+ self.token_list = token_list.copy()
- super().__init__()
- # note that eos is the same as sos (equivalent ID)
- self.blank_id = blank_id
- self.sos = vocab_size - 1 if sos is None else sos
- self.eos = vocab_size - 1 if eos is None else eos
- self.vocab_size = vocab_size
- self.ignore_id = ignore_id
- self.ctc_weight = ctc_weight
- self.interctc_weight = interctc_weight
- self.token_list = token_list.copy()
+ self.frontend = frontend
+ self.specaug = specaug
+ self.normalize = normalize
+ self.preencoder = preencoder
+ self.postencoder = postencoder
+ self.encoder = encoder
- self.frontend = frontend
- self.specaug = specaug
- self.normalize = normalize
- self.preencoder = preencoder
- self.postencoder = postencoder
- self.encoder = encoder
+ if not hasattr(self.encoder, "interctc_use_conditioning"):
+ self.encoder.interctc_use_conditioning = False
+ if self.encoder.interctc_use_conditioning:
+ self.encoder.conditioning_layer = torch.nn.Linear(
+ vocab_size, self.encoder.output_size()
+ )
- if not hasattr(self.encoder, "interctc_use_conditioning"):
- self.encoder.interctc_use_conditioning = False
- if self.encoder.interctc_use_conditioning:
- self.encoder.conditioning_layer = torch.nn.Linear(
- vocab_size, self.encoder.output_size()
- )
+ self.error_calculator = None
- self.error_calculator = None
+ if ctc_weight == 1.0:
+ self.decoder = None
+ else:
+ self.decoder = decoder
+ self.criterion_att = LabelSmoothingLoss(
+ size=vocab_size,
+ padding_idx=ignore_id,
+ smoothing=lsm_weight,
+ normalize_length=length_normalized_loss,
+ )
- if ctc_weight == 1.0:
- self.decoder = None
- else:
- self.decoder = decoder
+ if report_cer or report_wer:
+ self.error_calculator = ErrorCalculator(
+ token_list, sym_space, sym_blank, report_cer, report_wer
+ )
- self.criterion_att = LabelSmoothingLoss(
- size=vocab_size,
- padding_idx=ignore_id,
- smoothing=lsm_weight,
- normalize_length=length_normalized_loss,
- )
+ if ctc_weight == 0.0:
+ self.ctc = None
+ else:
+ self.ctc = ctc
- if report_cer or report_wer:
- self.error_calculator = ErrorCalculator(
- token_list, sym_space, sym_blank, report_cer, report_wer
- )
+ self.extract_feats_in_collect_stats = extract_feats_in_collect_stats
+ self.predictor = predictor
+ self.predictor_weight = predictor_weight
+ self.predictor_bias = predictor_bias
+ self.sampling_ratio = sampling_ratio
+ self.criterion_pre = mae_loss(normalize_length=length_normalized_loss)
+ self.step_cur = 0
- if ctc_weight == 0.0:
- self.ctc = None
- else:
- self.ctc = ctc
+ self.share_embedding = share_embedding
+ if self.share_embedding:
+ self.decoder.embed = None
- self.extract_feats_in_collect_stats = extract_feats_in_collect_stats
- self.predictor = predictor
- self.predictor_weight = predictor_weight
- self.predictor_bias = predictor_bias
- self.sampling_ratio = sampling_ratio
- self.criterion_pre = mae_loss(normalize_length=length_normalized_loss)
- self.step_cur = 0
+ def forward(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
+ """Frontend + Encoder + Decoder + Calc loss
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ text: (Batch, Length)
+ text_lengths: (Batch,)
+ """
+ assert text_lengths.dim() == 1, text_lengths.shape
+ # Check that batch_size is unified
+ assert (
+ speech.shape[0]
+ == speech_lengths.shape[0]
+ == text.shape[0]
+ == text_lengths.shape[0]
+ ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
+ batch_size = speech.shape[0]
+ self.step_cur += 1
+ # for data-parallel
+ text = text[:, : text_lengths.max()]
+ speech = speech[:, :speech_lengths.max()]
- def forward(
- self,
- speech: torch.Tensor,
- speech_lengths: torch.Tensor,
- text: torch.Tensor,
- text_lengths: torch.Tensor,
- ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
- """Frontend + Encoder + Decoder + Calc loss
+ # 1. Encoder
+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+ intermediate_outs = None
+ if isinstance(encoder_out, tuple):
+ intermediate_outs = encoder_out[1]
+ encoder_out = encoder_out[0]
- Args:
- speech: (Batch, Length, ...)
- speech_lengths: (Batch, )
- text: (Batch, Length)
- text_lengths: (Batch,)
- """
- assert text_lengths.dim() == 1, text_lengths.shape
- # Check that batch_size is unified
- assert (
- speech.shape[0]
- == speech_lengths.shape[0]
- == text.shape[0]
- == text_lengths.shape[0]
- ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
- batch_size = speech.shape[0]
- self.step_cur += 1
- # for data-parallel
- text = text[:, : text_lengths.max()]
- speech = speech[:, :speech_lengths.max(), :]
+ loss_att, acc_att, cer_att, wer_att = None, None, None, None
+ loss_ctc, cer_ctc = None, None
+ loss_pre = None
+ stats = dict()
- # 1. Encoder
- encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
- intermediate_outs = None
- if isinstance(encoder_out, tuple):
- intermediate_outs = encoder_out[1]
- encoder_out = encoder_out[0]
+ # 1. CTC branch
+ if self.ctc_weight != 0.0:
+ loss_ctc, cer_ctc = self._calc_ctc_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
+ # Collect CTC branch stats
+ stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
+ stats["cer_ctc"] = cer_ctc
- loss_att, acc_att, cer_att, wer_att = None, None, None, None
- loss_ctc, cer_ctc = None, None
- loss_pre = None
- stats = dict()
+ # Intermediate CTC (optional)
+ loss_interctc = 0.0
+ if self.interctc_weight != 0.0 and intermediate_outs is not None:
+ for layer_idx, intermediate_out in intermediate_outs:
+ # we assume intermediate_out has the same length & padding
+ # as those of encoder_out
+ loss_ic, cer_ic = self._calc_ctc_loss(
+ intermediate_out, encoder_out_lens, text, text_lengths
+ )
+ loss_interctc = loss_interctc + loss_ic
- # 1. CTC branch
- if self.ctc_weight != 0.0:
- loss_ctc, cer_ctc = self._calc_ctc_loss(
- encoder_out, encoder_out_lens, text, text_lengths
- )
+ # Collect Intermedaite CTC stats
+ stats["loss_interctc_layer{}".format(layer_idx)] = (
+ loss_ic.detach() if loss_ic is not None else None
+ )
+ stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
- # Collect CTC branch stats
- stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
- stats["cer_ctc"] = cer_ctc
+ loss_interctc = loss_interctc / len(intermediate_outs)
- # Intermediate CTC (optional)
- loss_interctc = 0.0
- if self.interctc_weight != 0.0 and intermediate_outs is not None:
- for layer_idx, intermediate_out in intermediate_outs:
- # we assume intermediate_out has the same length & padding
- # as those of encoder_out
- loss_ic, cer_ic = self._calc_ctc_loss(
- intermediate_out, encoder_out_lens, text, text_lengths
- )
- loss_interctc = loss_interctc + loss_ic
+ # calculate whole encoder loss
+ loss_ctc = (
+ 1 - self.interctc_weight
+ ) * loss_ctc + self.interctc_weight * loss_interctc
- # Collect Intermedaite CTC stats
- stats["loss_interctc_layer{}".format(layer_idx)] = (
- loss_ic.detach() if loss_ic is not None else None
- )
- stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
+ # 2b. Attention decoder branch
+ if self.ctc_weight != 1.0:
+ loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
- loss_interctc = loss_interctc / len(intermediate_outs)
+ # 3. CTC-Att loss definition
+ if self.ctc_weight == 0.0:
+ loss = loss_att + loss_pre * self.predictor_weight
+ elif self.ctc_weight == 1.0:
+ loss = loss_ctc
+ else:
+ loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight
- # calculate whole encoder loss
- loss_ctc = (
- 1 - self.interctc_weight
- ) * loss_ctc + self.interctc_weight * loss_interctc
+ # Collect Attn branch stats
+ stats["loss_att"] = loss_att.detach() if loss_att is not None else None
+ stats["acc"] = acc_att
+ stats["cer"] = cer_att
+ stats["wer"] = wer_att
+ stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None
+ stats["loss"] = torch.clone(loss.detach())
- # 2b. Attention decoder branch
- if self.ctc_weight != 1.0:
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
+ return loss, stats, weight
- loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_loss(
- encoder_out, encoder_out_lens, text, text_lengths
- )
+ def collect_feats(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ ) -> Dict[str, torch.Tensor]:
+ if self.extract_feats_in_collect_stats:
+ feats, feats_lengths = self._extract_feats(speech, speech_lengths)
+ else:
+ # Generate dummy stats if extract_feats_in_collect_stats is False
+ logging.warning(
+ "Generating dummy stats for feats and feats_lengths, "
+ "because encoder_conf.extract_feats_in_collect_stats is "
+ f"{self.extract_feats_in_collect_stats}"
+ )
+ feats, feats_lengths = speech, speech_lengths
+ return {"feats": feats, "feats_lengths": feats_lengths}
- # 3. CTC-Att loss definition
- if self.ctc_weight == 0.0:
- loss = loss_att + loss_pre * self.predictor_weight
- elif self.ctc_weight == 1.0:
- loss = loss_ctc
- else:
- loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight
+ def encode(
+ self, speech: torch.Tensor, speech_lengths: torch.Tensor
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Frontend + Encoder. Note that this method is used by asr_inference.py
- # Collect Attn branch stats
- stats["loss_att"] = loss_att.detach() if loss_att is not None else None
- stats["acc"] = acc_att
- stats["cer"] = cer_att
- stats["wer"] = wer_att
- stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ """
+ with autocast(False):
+ # 1. Extract feats
+ feats, feats_lengths = self._extract_feats(speech, speech_lengths)
- stats["loss"] =torch.clone(loss.detach())
+ # 2. Data augmentation
+ if self.specaug is not None and self.training:
+ feats, feats_lengths = self.specaug(feats, feats_lengths)
- # force_gatherable: to-device and to-tensor if scalar for DataParallel
- loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
- return loss, stats, weight
+ # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
+ if self.normalize is not None:
+ feats, feats_lengths = self.normalize(feats, feats_lengths)
- def collect_feats(
- self,
- speech: torch.Tensor,
- speech_lengths: torch.Tensor,
- text: torch.Tensor,
- text_lengths: torch.Tensor,
- ) -> Dict[str, torch.Tensor]:
- if self.extract_feats_in_collect_stats:
- feats, feats_lengths = self._extract_feats(speech, speech_lengths)
- else:
- # Generate dummy stats if extract_feats_in_collect_stats is False
- logging.warning(
- "Generating dummy stats for feats and feats_lengths, "
- "because encoder_conf.extract_feats_in_collect_stats is "
- f"{self.extract_feats_in_collect_stats}"
- )
- feats, feats_lengths = speech, speech_lengths
- return {"feats": feats, "feats_lengths": feats_lengths}
+ # Pre-encoder, e.g. used for raw input data
+ if self.preencoder is not None:
+ feats, feats_lengths = self.preencoder(feats, feats_lengths)
- def encode(
- self, speech: torch.Tensor, speech_lengths: torch.Tensor
- ) -> Tuple[torch.Tensor, torch.Tensor]:
- """Frontend + Encoder. Note that this method is used by asr_inference.py
+ # 4. Forward encoder
+ # feats: (Batch, Length, Dim)
+ # -> encoder_out: (Batch, Length2, Dim2)
+ if self.encoder.interctc_use_conditioning:
+ encoder_out, encoder_out_lens, _ = self.encoder(
+ feats, feats_lengths, ctc=self.ctc
+ )
+ else:
+ encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths)
+ intermediate_outs = None
+ if isinstance(encoder_out, tuple):
+ intermediate_outs = encoder_out[1]
+ encoder_out = encoder_out[0]
- Args:
- speech: (Batch, Length, ...)
- speech_lengths: (Batch, )
- """
- with autocast(False):
- # 1. Extract feats
- feats, feats_lengths = self._extract_feats(speech, speech_lengths)
+ # Post-encoder, e.g. NLU
+ if self.postencoder is not None:
+ encoder_out, encoder_out_lens = self.postencoder(
+ encoder_out, encoder_out_lens
+ )
- # 2. Data augmentation
- if self.specaug is not None and self.training:
- feats, feats_lengths = self.specaug(feats, feats_lengths)
-
- # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
- if self.normalize is not None:
- feats, feats_lengths = self.normalize(feats, feats_lengths)
-
- # Pre-encoder, e.g. used for raw input data
- if self.preencoder is not None:
- feats, feats_lengths = self.preencoder(feats, feats_lengths)
-
- # 4. Forward encoder
- # feats: (Batch, Length, Dim)
- # -> encoder_out: (Batch, Length2, Dim2)
- if self.encoder.interctc_use_conditioning:
- encoder_out, encoder_out_lens, _ = self.encoder(
- feats, feats_lengths, ctc=self.ctc
- )
- else:
- encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths)
- intermediate_outs = None
- if isinstance(encoder_out, tuple):
- intermediate_outs = encoder_out[1]
- encoder_out = encoder_out[0]
-
- # Post-encoder, e.g. NLU
- if self.postencoder is not None:
- encoder_out, encoder_out_lens = self.postencoder(
- encoder_out, encoder_out_lens
- )
-
- assert encoder_out.size(0) == speech.size(0), (
- encoder_out.size(),
- speech.size(0),
- )
- assert encoder_out.size(1) <= encoder_out_lens.max(), (
- encoder_out.size(),
- encoder_out_lens.max(),
- )
-
- if intermediate_outs is not None:
- return (encoder_out, intermediate_outs), encoder_out_lens
+ assert encoder_out.size(0) == speech.size(0), (
+ encoder_out.size(),
+ speech.size(0),
+ )
+ assert encoder_out.size(1) <= encoder_out_lens.max(), (
+ encoder_out.size(),
+ encoder_out_lens.max(),
+ )
- return encoder_out, encoder_out_lens
+ if intermediate_outs is not None:
+ return (encoder_out, intermediate_outs), encoder_out_lens
- def calc_predictor(self, encoder_out, encoder_out_lens):
+ return encoder_out, encoder_out_lens
- encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)
- pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id)
- return pre_acoustic_embeds, pre_token_length
+ def calc_predictor(self, encoder_out, encoder_out_lens):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = self.predictor(encoder_out, None, encoder_out_mask,
+ ignore_id=self.ignore_id)
+ return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index
- def cal_decoder_with_predictor(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):
+ def cal_decoder_with_predictor(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):
- decoder_outs = self.decoder(
- encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
- )
- decoder_out = decoder_outs[0]
- decoder_out = torch.log_softmax(decoder_out, dim=-1)
- return decoder_out, ys_pad_lens
+ decoder_outs = self.decoder(
+ encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
+ )
+ decoder_out = decoder_outs[0]
+ decoder_out = torch.log_softmax(decoder_out, dim=-1)
+ return decoder_out, ys_pad_lens
- def _extract_feats(
- self, speech: torch.Tensor, speech_lengths: torch.Tensor
- ) -> Tuple[torch.Tensor, torch.Tensor]:
- assert speech_lengths.dim() == 1, speech_lengths.shape
+ def _extract_feats(
+ self, speech: torch.Tensor, speech_lengths: torch.Tensor
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ assert speech_lengths.dim() == 1, speech_lengths.shape
- # for data-parallel
- speech = speech[:, : speech_lengths.max()]
- if self.frontend is not None:
- # Frontend
- # e.g. STFT and Feature extract
- # data_loader may send time-domain signal in this case
- # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim)
- feats, feats_lengths = self.frontend(speech, speech_lengths)
- else:
- # No frontend and no feature extract
- feats, feats_lengths = speech, speech_lengths
- return feats, feats_lengths
+ # for data-parallel
+ speech = speech[:, : speech_lengths.max()]
+ if self.frontend is not None:
+ # Frontend
+ # e.g. STFT and Feature extract
+ # data_loader may send time-domain signal in this case
+ # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim)
+ feats, feats_lengths = self.frontend(speech, speech_lengths)
+ else:
+ # No frontend and no feature extract
+ feats, feats_lengths = speech, speech_lengths
+ return feats, feats_lengths
- def nll(
- self,
- encoder_out: torch.Tensor,
- encoder_out_lens: torch.Tensor,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- ) -> torch.Tensor:
- """Compute negative log likelihood(nll) from transformer-decoder
+ def nll(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ ) -> torch.Tensor:
+ """Compute negative log likelihood(nll) from transformer-decoder
- Normally, this function is called in batchify_nll.
+ Normally, this function is called in batchify_nll.
- Args:
- encoder_out: (Batch, Length, Dim)
- encoder_out_lens: (Batch,)
- ys_pad: (Batch, Length)
- ys_pad_lens: (Batch,)
- """
- ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
- ys_in_lens = ys_pad_lens + 1
+ Args:
+ encoder_out: (Batch, Length, Dim)
+ encoder_out_lens: (Batch,)
+ ys_pad: (Batch, Length)
+ ys_pad_lens: (Batch,)
+ """
+ ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
+ ys_in_lens = ys_pad_lens + 1
- # 1. Forward decoder
- decoder_out, _ = self.decoder(
- encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens
- ) # [batch, seqlen, dim]
- batch_size = decoder_out.size(0)
- decoder_num_class = decoder_out.size(2)
- # nll: negative log-likelihood
- nll = torch.nn.functional.cross_entropy(
- decoder_out.view(-1, decoder_num_class),
- ys_out_pad.view(-1),
- ignore_index=self.ignore_id,
- reduction="none",
- )
- nll = nll.view(batch_size, -1)
- nll = nll.sum(dim=1)
- assert nll.size(0) == batch_size
- return nll
+ # 1. Forward decoder
+ decoder_out, _ = self.decoder(
+ encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens
+ ) # [batch, seqlen, dim]
+ batch_size = decoder_out.size(0)
+ decoder_num_class = decoder_out.size(2)
+ # nll: negative log-likelihood
+ nll = torch.nn.functional.cross_entropy(
+ decoder_out.view(-1, decoder_num_class),
+ ys_out_pad.view(-1),
+ ignore_index=self.ignore_id,
+ reduction="none",
+ )
+ nll = nll.view(batch_size, -1)
+ nll = nll.sum(dim=1)
+ assert nll.size(0) == batch_size
+ return nll
- def batchify_nll(
- self,
- encoder_out: torch.Tensor,
- encoder_out_lens: torch.Tensor,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- batch_size: int = 100,
- ):
- """Compute negative log likelihood(nll) from transformer-decoder
+ def batchify_nll(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ batch_size: int = 100,
+ ):
+ """Compute negative log likelihood(nll) from transformer-decoder
- To avoid OOM, this fuction seperate the input into batches.
- Then call nll for each batch and combine and return results.
- Args:
- encoder_out: (Batch, Length, Dim)
- encoder_out_lens: (Batch,)
- ys_pad: (Batch, Length)
- ys_pad_lens: (Batch,)
- batch_size: int, samples each batch contain when computing nll,
- you may change this to avoid OOM or increase
- GPU memory usage
- """
- total_num = encoder_out.size(0)
- if total_num <= batch_size:
- nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
- else:
- nll = []
- start_idx = 0
- while True:
- end_idx = min(start_idx + batch_size, total_num)
- batch_encoder_out = encoder_out[start_idx:end_idx, :, :]
- batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx]
- batch_ys_pad = ys_pad[start_idx:end_idx, :]
- batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx]
- batch_nll = self.nll(
- batch_encoder_out,
- batch_encoder_out_lens,
- batch_ys_pad,
- batch_ys_pad_lens,
- )
- nll.append(batch_nll)
- start_idx = end_idx
- if start_idx == total_num:
- break
- nll = torch.cat(nll)
- assert nll.size(0) == total_num
- return nll
+ To avoid OOM, this fuction seperate the input into batches.
+ Then call nll for each batch and combine and return results.
+ Args:
+ encoder_out: (Batch, Length, Dim)
+ encoder_out_lens: (Batch,)
+ ys_pad: (Batch, Length)
+ ys_pad_lens: (Batch,)
+ batch_size: int, samples each batch contain when computing nll,
+ you may change this to avoid OOM or increase
+ GPU memory usage
+ """
+ total_num = encoder_out.size(0)
+ if total_num <= batch_size:
+ nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
+ else:
+ nll = []
+ start_idx = 0
+ while True:
+ end_idx = min(start_idx + batch_size, total_num)
+ batch_encoder_out = encoder_out[start_idx:end_idx, :, :]
+ batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx]
+ batch_ys_pad = ys_pad[start_idx:end_idx, :]
+ batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx]
+ batch_nll = self.nll(
+ batch_encoder_out,
+ batch_encoder_out_lens,
+ batch_ys_pad,
+ batch_ys_pad_lens,
+ )
+ nll.append(batch_nll)
+ start_idx = end_idx
+ if start_idx == total_num:
+ break
+ nll = torch.cat(nll)
+ assert nll.size(0) == total_num
+ return nll
- def _calc_att_loss(
- self,
- encoder_out: torch.Tensor,
- encoder_out_lens: torch.Tensor,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- ):
- encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)
- if self.predictor_bias == 1:
- _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
- ys_pad_lens = ys_pad_lens + self.predictor_bias
- pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id)
+ def _calc_att_loss(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ ):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ if self.predictor_bias == 1:
+ _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
+ ys_pad_lens = ys_pad_lens + self.predictor_bias
+ pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask,
+ ignore_id=self.ignore_id)
- # 0. sampler
- decoder_out_1st = None
- if self.sampling_ratio > 0.0:
- if self.step_cur < 2:
- logging.info("enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
- sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds)
- else:
- if self.step_cur < 2:
- logging.info("disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
- sematic_embeds = pre_acoustic_embeds
+ # 0. sampler
+ decoder_out_1st = None
+ if self.sampling_ratio > 0.0:
+ if self.step_cur < 2:
+ logging.info("enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens,
+ pre_acoustic_embeds)
+ else:
+ if self.step_cur < 2:
+ logging.info("disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds = pre_acoustic_embeds
- # 1. Forward decoder
- decoder_outs = self.decoder(
- encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
- )
- decoder_out, _ = decoder_outs[0], decoder_outs[1]
+ # 1. Forward decoder
+ decoder_outs = self.decoder(
+ encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
+ )
+ decoder_out, _ = decoder_outs[0], decoder_outs[1]
- if decoder_out_1st is None:
- decoder_out_1st = decoder_out
- # 2. Compute attention loss
- loss_att = self.criterion_att(decoder_out, ys_pad)
- acc_att = th_accuracy(
- decoder_out_1st.view(-1, self.vocab_size),
- ys_pad,
- ignore_label=self.ignore_id,
- )
- loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length)
+ if decoder_out_1st is None:
+ decoder_out_1st = decoder_out
+ # 2. Compute attention loss
+ loss_att = self.criterion_att(decoder_out, ys_pad)
+ acc_att = th_accuracy(
+ decoder_out_1st.view(-1, self.vocab_size),
+ ys_pad,
+ ignore_label=self.ignore_id,
+ )
+ loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length)
- # Compute cer/wer using attention-decoder
- if self.training or self.error_calculator is None:
- cer_att, wer_att = None, None
- else:
- ys_hat = decoder_out_1st.argmax(dim=-1)
- cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu())
+ # Compute cer/wer using attention-decoder
+ if self.training or self.error_calculator is None:
+ cer_att, wer_att = None, None
+ else:
+ ys_hat = decoder_out_1st.argmax(dim=-1)
+ cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu())
- return loss_att, acc_att, cer_att, wer_att, loss_pre
+ return loss_att, acc_att, cer_att, wer_att, loss_pre
- def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):
+ def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):
- tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)
- ys_pad = ys_pad * tgt_mask[:, :, 0]
- ys_pad_embed = self.decoder.embed(ys_pad)
- with torch.no_grad():
- decoder_outs = self.decoder(
- encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_pad_lens
- )
- decoder_out, _ = decoder_outs[0], decoder_outs[1]
- pred_tokens = decoder_out.argmax(-1)
- nonpad_positions = ys_pad.ne(self.ignore_id)
- seq_lens = (nonpad_positions).sum(1)
- same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1)
- input_mask = torch.ones_like(nonpad_positions)
- bsz, seq_len = ys_pad.size()
- for li in range(bsz):
- target_num = (((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio).long()
- if target_num > 0:
- input_mask[li].scatter_(dim=0, index=torch.randperm(seq_lens[li])[:target_num].cuda(), value=0)
- input_mask = input_mask.eq(1)
- input_mask = input_mask.masked_fill(~nonpad_positions, False)
- input_mask_expand_dim = input_mask.unsqueeze(2).to(pre_acoustic_embeds.device)
+ tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)
+ ys_pad = ys_pad * tgt_mask[:, :, 0]
+ if self.share_embedding:
+ ys_pad_embed = self.decoder.output_layer.weight[ys_pad]
+ else:
+ ys_pad_embed = self.decoder.embed(ys_pad)
+ with torch.no_grad():
+ decoder_outs = self.decoder(
+ encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_pad_lens
+ )
+ decoder_out, _ = decoder_outs[0], decoder_outs[1]
+ pred_tokens = decoder_out.argmax(-1)
+ nonpad_positions = ys_pad.ne(self.ignore_id)
+ seq_lens = (nonpad_positions).sum(1)
+ same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1)
+ input_mask = torch.ones_like(nonpad_positions)
+ bsz, seq_len = ys_pad.size()
+ for li in range(bsz):
+ target_num = (((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio).long()
+ if target_num > 0:
+ input_mask[li].scatter_(dim=0, index=torch.randperm(seq_lens[li])[:target_num].cuda(), value=0)
+ input_mask = input_mask.eq(1)
+ input_mask = input_mask.masked_fill(~nonpad_positions, False)
+ input_mask_expand_dim = input_mask.unsqueeze(2).to(pre_acoustic_embeds.device)
- sematic_embeds = pre_acoustic_embeds.masked_fill(~input_mask_expand_dim, 0) + ys_pad_embed.masked_fill(
- input_mask_expand_dim, 0)
- return sematic_embeds * tgt_mask, decoder_out * tgt_mask
+ sematic_embeds = pre_acoustic_embeds.masked_fill(~input_mask_expand_dim, 0) + ys_pad_embed.masked_fill(
+ input_mask_expand_dim, 0)
+ return sematic_embeds * tgt_mask, decoder_out * tgt_mask
+ def _calc_ctc_loss(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ ):
+ # Calc CTC loss
+ loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
- def _calc_ctc_loss(
- self,
- encoder_out: torch.Tensor,
- encoder_out_lens: torch.Tensor,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- ):
- # Calc CTC loss
- loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
+ # Calc CER using CTC
+ cer_ctc = None
+ if not self.training and self.error_calculator is not None:
+ ys_hat = self.ctc.argmax(encoder_out).data
+ cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
+ return loss_ctc, cer_ctc
- # Calc CER using CTC
- cer_ctc = None
- if not self.training and self.error_calculator is not None:
- ys_hat = self.ctc.argmax(encoder_out).data
- cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
- return loss_ctc, cer_ctc
class ParaformerBert(Paraformer):
- """
- Author: Speech Lab, Alibaba Group, China
- Paraformer2: advanced paraformer with LFMMI and bert for non-autoregressive end-to-end speech recognition
- """
+ """
+ Author: Speech Lab, Alibaba Group, China
+ Paraformer2: advanced paraformer with LFMMI and bert for non-autoregressive end-to-end speech recognition
+ """
- def __init__(
- self,
- vocab_size: int,
- token_list: Union[Tuple[str, ...], List[str]],
- frontend: Optional[AbsFrontend],
- specaug: Optional[AbsSpecAug],
- normalize: Optional[AbsNormalize],
- preencoder: Optional[AbsPreEncoder],
- encoder: AbsEncoder,
- postencoder: Optional[AbsPostEncoder],
- decoder: AbsDecoder,
- ctc: CTC,
- ctc_weight: float = 0.5,
- interctc_weight: float = 0.0,
- ignore_id: int = -1,
- blank_id: int = 0,
- sos: int = 1,
- eos: int = 2,
- lsm_weight: float = 0.0,
- length_normalized_loss: bool = False,
- report_cer: bool = True,
- report_wer: bool = True,
- sym_space: str = "<space>",
- sym_blank: str = "<blank>",
- extract_feats_in_collect_stats: bool = True,
- predictor = None,
- predictor_weight: float = 0.0,
- predictor_bias: int = 0,
- sampling_ratio: float = 0.2,
- embeds_id: int = 2,
- embeds_loss_weight: float = 0.0,
- embed_dims: int = 768,
- ):
- assert check_argument_types()
- assert 0.0 <= ctc_weight <= 1.0, ctc_weight
- assert 0.0 <= interctc_weight < 1.0, interctc_weight
+ def __init__(
+ self,
+ vocab_size: int,
+ token_list: Union[Tuple[str, ...], List[str]],
+ frontend: Optional[AbsFrontend],
+ specaug: Optional[AbsSpecAug],
+ normalize: Optional[AbsNormalize],
+ preencoder: Optional[AbsPreEncoder],
+ encoder: AbsEncoder,
+ postencoder: Optional[AbsPostEncoder],
+ decoder: AbsDecoder,
+ ctc: CTC,
+ ctc_weight: float = 0.5,
+ interctc_weight: float = 0.0,
+ ignore_id: int = -1,
+ blank_id: int = 0,
+ sos: int = 1,
+ eos: int = 2,
+ lsm_weight: float = 0.0,
+ length_normalized_loss: bool = False,
+ report_cer: bool = True,
+ report_wer: bool = True,
+ sym_space: str = "<space>",
+ sym_blank: str = "<blank>",
+ extract_feats_in_collect_stats: bool = True,
+ predictor=None,
+ predictor_weight: float = 0.0,
+ predictor_bias: int = 0,
+ sampling_ratio: float = 0.2,
+ embeds_id: int = 2,
+ embeds_loss_weight: float = 0.0,
+ embed_dims: int = 768,
+ ):
+ assert check_argument_types()
+ assert 0.0 <= ctc_weight <= 1.0, ctc_weight
+ assert 0.0 <= interctc_weight < 1.0, interctc_weight
- super().__init__(
- vocab_size=vocab_size,
- token_list=token_list,
- frontend=frontend,
- specaug=specaug,
- normalize=normalize,
- preencoder=preencoder,
- encoder=encoder,
- postencoder=postencoder,
- decoder=decoder,
- ctc=ctc,
- ctc_weight=ctc_weight,
- interctc_weight=interctc_weight,
- ignore_id=ignore_id,
- blank_id=blank_id,
- sos=sos,
- eos=eos,
- lsm_weight=lsm_weight,
- length_normalized_loss=length_normalized_loss,
- report_cer=report_cer,
- report_wer=report_wer,
- sym_space=sym_space,
- sym_blank=sym_blank,
- extract_feats_in_collect_stats=extract_feats_in_collect_stats,
- predictor=predictor,
- predictor_weight=predictor_weight,
- predictor_bias=predictor_bias,
- sampling_ratio=sampling_ratio,
- )
- self.decoder.embeds_id = embeds_id
- decoder_attention_dim = self.decoder.attention_dim
- self.pro_nn = torch.nn.Linear(decoder_attention_dim, embed_dims)
- self.cos = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
- self.embeds_loss_weight = embeds_loss_weight
- self.length_normalized_loss = length_normalized_loss
+ super().__init__(
+ vocab_size=vocab_size,
+ token_list=token_list,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ decoder=decoder,
+ ctc=ctc,
+ ctc_weight=ctc_weight,
+ interctc_weight=interctc_weight,
+ ignore_id=ignore_id,
+ blank_id=blank_id,
+ sos=sos,
+ eos=eos,
+ lsm_weight=lsm_weight,
+ length_normalized_loss=length_normalized_loss,
+ report_cer=report_cer,
+ report_wer=report_wer,
+ sym_space=sym_space,
+ sym_blank=sym_blank,
+ extract_feats_in_collect_stats=extract_feats_in_collect_stats,
+ predictor=predictor,
+ predictor_weight=predictor_weight,
+ predictor_bias=predictor_bias,
+ sampling_ratio=sampling_ratio,
+ )
+ self.decoder.embeds_id = embeds_id
+ decoder_attention_dim = self.decoder.attention_dim
+ self.pro_nn = torch.nn.Linear(decoder_attention_dim, embed_dims)
+ self.cos = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+ self.embeds_loss_weight = embeds_loss_weight
+ self.length_normalized_loss = length_normalized_loss
- def _calc_embed_loss(self,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- embed: torch.Tensor = None,
- embed_lengths: torch.Tensor = None,
- embeds_outputs: torch.Tensor = None,
- ):
- embeds_outputs = self.pro_nn(embeds_outputs)
- tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)
- embeds_outputs *= tgt_mask # b x l x d
- embed *= tgt_mask # b x l x d
- cos_loss = 1.0 - self.cos(embeds_outputs, embed)
- cos_loss *= tgt_mask.squeeze(2)
- if self.length_normalized_loss:
- token_num_total = torch.sum(tgt_mask)
- else:
- token_num_total = tgt_mask.size()[0]
- cos_loss_total = torch.sum(cos_loss)
- cos_loss = cos_loss_total / token_num_total
- # print("cos_loss: {}".format(cos_loss))
- return cos_loss
+ def _calc_embed_loss(self,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ embed: torch.Tensor = None,
+ embed_lengths: torch.Tensor = None,
+ embeds_outputs: torch.Tensor = None,
+ ):
+ embeds_outputs = self.pro_nn(embeds_outputs)
+ tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)
+ embeds_outputs *= tgt_mask # b x l x d
+ embed *= tgt_mask # b x l x d
+ cos_loss = 1.0 - self.cos(embeds_outputs, embed)
+ cos_loss *= tgt_mask.squeeze(2)
+ if self.length_normalized_loss:
+ token_num_total = torch.sum(tgt_mask)
+ else:
+ token_num_total = tgt_mask.size()[0]
+ cos_loss_total = torch.sum(cos_loss)
+ cos_loss = cos_loss_total / token_num_total
+ # print("cos_loss: {}".format(cos_loss))
+ return cos_loss
+
+ def _calc_att_loss(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ ):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ if self.predictor_bias == 1:
+ _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
+ ys_pad_lens = ys_pad_lens + self.predictor_bias
+ pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask,
+ ignore_id=self.ignore_id)
+
+ # 0. sampler
+ decoder_out_1st = None
+ if self.sampling_ratio > 0.0:
+ if self.step_cur < 2:
+ logging.info(
+ "enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens,
+ pre_acoustic_embeds)
+ else:
+ if self.step_cur < 2:
+ logging.info(
+ "disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds = pre_acoustic_embeds
+
+ # 1. Forward decoder
+ decoder_outs = self.decoder(
+ encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
+ )
+ decoder_out, _ = decoder_outs[0], decoder_outs[1]
+ embeds_outputs = None
+ if len(decoder_outs) > 2:
+ embeds_outputs = decoder_outs[2]
+
+ if decoder_out_1st is None:
+ decoder_out_1st = decoder_out
+ # 2. Compute attention loss
+ loss_att = self.criterion_att(decoder_out, ys_pad)
+ acc_att = th_accuracy(
+ decoder_out_1st.view(-1, self.vocab_size),
+ ys_pad,
+ ignore_label=self.ignore_id,
+ )
+ loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length)
+
+ # Compute cer/wer using attention-decoder
+ if self.training or self.error_calculator is None:
+ cer_att, wer_att = None, None
+ else:
+ ys_hat = decoder_out_1st.argmax(dim=-1)
+ cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu())
+
+ return loss_att, acc_att, cer_att, wer_att, loss_pre, embeds_outputs
+
+ def forward(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ embed: torch.Tensor = None,
+ embed_lengths: torch.Tensor = None,
+ ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
+ """Frontend + Encoder + Decoder + Calc loss
+
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ text: (Batch, Length)
+ text_lengths: (Batch,)
+ """
+ assert text_lengths.dim() == 1, text_lengths.shape
+ # Check that batch_size is unified
+ assert (
+ speech.shape[0]
+ == speech_lengths.shape[0]
+ == text.shape[0]
+ == text_lengths.shape[0]
+ ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
+ batch_size = speech.shape[0]
+ self.step_cur += 1
+ # for data-parallel
+ text = text[:, : text_lengths.max()]
+ speech = speech[:, :speech_lengths.max(), :]
+ if embed is not None:
+ embed = embed[:, :embed_lengths.max(), :]
+
+ # 1. Encoder
+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+ intermediate_outs = None
+ if isinstance(encoder_out, tuple):
+ intermediate_outs = encoder_out[1]
+ encoder_out = encoder_out[0]
+
+ loss_att, acc_att, cer_att, wer_att = None, None, None, None
+ loss_ctc, cer_ctc = None, None
+ loss_pre = 0.0
+ cos_loss = 0.0
+ stats = dict()
+
+ # 1. CTC branch
+ if self.ctc_weight != 0.0:
+ loss_ctc, cer_ctc = self._calc_ctc_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
+
+ # Collect CTC branch stats
+ stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
+ stats["cer_ctc"] = cer_ctc
+
+ # Intermediate CTC (optional)
+ loss_interctc = 0.0
+ if self.interctc_weight != 0.0 and intermediate_outs is not None:
+ for layer_idx, intermediate_out in intermediate_outs:
+ # we assume intermediate_out has the same length & padding
+ # as those of encoder_out
+ loss_ic, cer_ic = self._calc_ctc_loss(
+ intermediate_out, encoder_out_lens, text, text_lengths
+ )
+ loss_interctc = loss_interctc + loss_ic
+
+ # Collect Intermedaite CTC stats
+ stats["loss_interctc_layer{}".format(layer_idx)] = (
+ loss_ic.detach() if loss_ic is not None else None
+ )
+ stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
+
+ loss_interctc = loss_interctc / len(intermediate_outs)
+
+ # calculate whole encoder loss
+ loss_ctc = (
+ 1 - self.interctc_weight
+ ) * loss_ctc + self.interctc_weight * loss_interctc
+
+ # 2b. Attention decoder branch
+ if self.ctc_weight != 1.0:
+
+ loss_ret = self._calc_att_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
+ loss_att, acc_att, cer_att, wer_att, loss_pre = loss_ret[0], loss_ret[1], loss_ret[2], loss_ret[3], \
+ loss_ret[4]
+ embeds_outputs = None
+ if len(loss_ret) > 5:
+ embeds_outputs = loss_ret[5]
+ if embeds_outputs is not None:
+ cos_loss = self._calc_embed_loss(text, text_lengths, embed, embed_lengths, embeds_outputs)
+
+ # 3. CTC-Att loss definition
+ if self.ctc_weight == 0.0:
+ loss = loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight
+ elif self.ctc_weight == 1.0:
+ loss = loss_ctc
+ else:
+ loss = self.ctc_weight * loss_ctc + (
+ 1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight
+
+ # Collect Attn branch stats
+ stats["loss_att"] = loss_att.detach() if loss_att is not None else None
+ stats["acc"] = acc_att
+ stats["cer"] = cer_att
+ stats["wer"] = wer_att
+ stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre > 0.0 else None
+ stats["cos_loss"] = cos_loss.detach().cpu() if cos_loss > 0.0 else None
+
+ stats["loss"] = torch.clone(loss.detach())
+
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
+ return loss, stats, weight
- def _calc_att_loss(
- self,
- encoder_out: torch.Tensor,
- encoder_out_lens: torch.Tensor,
- ys_pad: torch.Tensor,
- ys_pad_lens: torch.Tensor,
- ):
- encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)
- if self.predictor_bias == 1:
- _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
- ys_pad_lens = ys_pad_lens + self.predictor_bias
- pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id)
+class BiCifParaformer(Paraformer):
- # 0. sampler
- decoder_out_1st = None
- if self.sampling_ratio > 0.0:
- if self.step_cur < 2:
- logging.info(
- "enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
- sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds)
- else:
- if self.step_cur < 2:
- logging.info(
- "disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
- sematic_embeds = pre_acoustic_embeds
+ """CTC-attention hybrid Encoder-Decoder model"""
- # 1. Forward decoder
- decoder_outs = self.decoder(
- encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
- )
- decoder_out, _ = decoder_outs[0], decoder_outs[1]
- embeds_outputs = None
- if len(decoder_outs) > 2:
- embeds_outputs = decoder_outs[2]
+ def __init__(
+ self,
+ vocab_size: int,
+ token_list: Union[Tuple[str, ...], List[str]],
+ frontend: Optional[AbsFrontend],
+ specaug: Optional[AbsSpecAug],
+ normalize: Optional[AbsNormalize],
+ preencoder: Optional[AbsPreEncoder],
+ encoder: AbsEncoder,
+ postencoder: Optional[AbsPostEncoder],
+ decoder: AbsDecoder,
+ ctc: CTC,
+ ctc_weight: float = 0.5,
+ interctc_weight: float = 0.0,
+ ignore_id: int = -1,
+ blank_id: int = 0,
+ sos: int = 1,
+ eos: int = 2,
+ lsm_weight: float = 0.0,
+ length_normalized_loss: bool = False,
+ report_cer: bool = True,
+ report_wer: bool = True,
+ sym_space: str = "<space>",
+ sym_blank: str = "<blank>",
+ extract_feats_in_collect_stats: bool = True,
+ predictor = None,
+ predictor_weight: float = 0.0,
+ predictor_bias: int = 0,
+ sampling_ratio: float = 0.2,
+ ):
+ assert check_argument_types()
+ assert 0.0 <= ctc_weight <= 1.0, ctc_weight
+ assert 0.0 <= interctc_weight < 1.0, interctc_weight
- if decoder_out_1st is None:
- decoder_out_1st = decoder_out
- # 2. Compute attention loss
- loss_att = self.criterion_att(decoder_out, ys_pad)
- acc_att = th_accuracy(
- decoder_out_1st.view(-1, self.vocab_size),
- ys_pad,
- ignore_label=self.ignore_id,
- )
- loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length)
+ super().__init__(
+ vocab_size=vocab_size,
+ token_list=token_list,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ decoder=decoder,
+ ctc=ctc,
+ ctc_weight=ctc_weight,
+ interctc_weight=interctc_weight,
+ ignore_id=ignore_id,
+ blank_id=blank_id,
+ sos=sos,
+ eos=eos,
+ lsm_weight=lsm_weight,
+ length_normalized_loss=length_normalized_loss,
+ report_cer=report_cer,
+ report_wer=report_wer,
+ sym_space=sym_space,
+ sym_blank=sym_blank,
+ extract_feats_in_collect_stats=extract_feats_in_collect_stats,
+ predictor=predictor,
+ predictor_weight=predictor_weight,
+ predictor_bias=predictor_bias,
+ sampling_ratio=sampling_ratio,
+ )
+ assert isinstance(self.predictor, CifPredictorV3), "BiCifParaformer should use CIFPredictorV3"
- # Compute cer/wer using attention-decoder
- if self.training or self.error_calculator is None:
- cer_att, wer_att = None, None
- else:
- ys_hat = decoder_out_1st.argmax(dim=-1)
- cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu())
+ def _calc_att_loss(
+ self,
+ encoder_out: torch.Tensor,
+ encoder_out_lens: torch.Tensor,
+ ys_pad: torch.Tensor,
+ ys_pad_lens: torch.Tensor,
+ ):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ if self.predictor_bias == 1:
+ _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
+ ys_pad_lens = ys_pad_lens + self.predictor_bias
+ pre_acoustic_embeds, pre_token_length, _, pre_peak_index, pre_token_length2 = self.predictor(encoder_out, ys_pad, encoder_out_mask,
+ ignore_id=self.ignore_id)
- return loss_att, acc_att, cer_att, wer_att, loss_pre, embeds_outputs
+ # 0. sampler
+ decoder_out_1st = None
+ if self.sampling_ratio > 0.0:
+ if self.step_cur < 2:
+ logging.info("enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens,
+ pre_acoustic_embeds)
+ else:
+ if self.step_cur < 2:
+ logging.info("disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio))
+ sematic_embeds = pre_acoustic_embeds
+ # 1. Forward decoder
+ decoder_outs = self.decoder(
+ encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
+ )
+ decoder_out, _ = decoder_outs[0], decoder_outs[1]
- def forward(
- self,
- speech: torch.Tensor,
- speech_lengths: torch.Tensor,
- text: torch.Tensor,
- text_lengths: torch.Tensor,
- embed: torch.Tensor = None,
- embed_lengths: torch.Tensor = None,
- ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
- """Frontend + Encoder + Decoder + Calc loss
+ if decoder_out_1st is None:
+ decoder_out_1st = decoder_out
+ # 2. Compute attention loss
+ loss_att = self.criterion_att(decoder_out, ys_pad)
+ acc_att = th_accuracy(
+ decoder_out_1st.view(-1, self.vocab_size),
+ ys_pad,
+ ignore_label=self.ignore_id,
+ )
+ loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length)
+ loss_pre2 = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length2)
- Args:
- speech: (Batch, Length, ...)
- speech_lengths: (Batch, )
- text: (Batch, Length)
- text_lengths: (Batch,)
- """
- assert text_lengths.dim() == 1, text_lengths.shape
- # Check that batch_size is unified
- assert (
- speech.shape[0]
- == speech_lengths.shape[0]
- == text.shape[0]
- == text_lengths.shape[0]
- ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
- batch_size = speech.shape[0]
- self.step_cur += 1
- # for data-parallel
- text = text[:, : text_lengths.max()]
- speech = speech[:, :speech_lengths.max(), :]
- if embed is not None:
- embed = embed[:, :embed_lengths.max(), :]
+ # Compute cer/wer using attention-decoder
+ if self.training or self.error_calculator is None:
+ cer_att, wer_att = None, None
+ else:
+ ys_hat = decoder_out_1st.argmax(dim=-1)
+ cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu())
- # 1. Encoder
- encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
- intermediate_outs = None
- if isinstance(encoder_out, tuple):
- intermediate_outs = encoder_out[1]
- encoder_out = encoder_out[0]
+ return loss_att, acc_att, cer_att, wer_att, loss_pre, loss_pre2
+
+ def calc_predictor(self, encoder_out, encoder_out_lens):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index, pre_token_length2 = self.predictor(encoder_out, None, encoder_out_mask,
+ ignore_id=self.ignore_id)
+ return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index
+
+ def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):
+ encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
+ encoder_out.device)
+ ds_alphas, ds_cif_peak, us_alphas, us_cif_peak = self.predictor.get_upsample_timestamp(encoder_out, None, encoder_out_mask, token_num=token_num,
+ ignore_id=self.ignore_id)
+ import pdb; pdb.set_trace()
+ return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak
- loss_att, acc_att, cer_att, wer_att = None, None, None, None
- loss_ctc, cer_ctc = None, None
- loss_pre = 0.0
- cos_loss = 0.0
- stats = dict()
+ def forward(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
+ """Frontend + Encoder + Decoder + Calc loss
- # 1. CTC branch
- if self.ctc_weight != 0.0:
- loss_ctc, cer_ctc = self._calc_ctc_loss(
- encoder_out, encoder_out_lens, text, text_lengths
- )
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ text: (Batch, Length)
+ text_lengths: (Batch,)
+ """
+ assert text_lengths.dim() == 1, text_lengths.shape
+ # Check that batch_size is unified
+ assert (
+ speech.shape[0]
+ == speech_lengths.shape[0]
+ == text.shape[0]
+ == text_lengths.shape[0]
+ ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
+ batch_size = speech.shape[0]
+ self.step_cur += 1
+ # for data-parallel
+ text = text[:, : text_lengths.max()]
+ speech = speech[:, :speech_lengths.max()]
- # Collect CTC branch stats
- stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
- stats["cer_ctc"] = cer_ctc
+ # 1. Encoder
+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+ intermediate_outs = None
+ if isinstance(encoder_out, tuple):
+ intermediate_outs = encoder_out[1]
+ encoder_out = encoder_out[0]
- # Intermediate CTC (optional)
- loss_interctc = 0.0
- if self.interctc_weight != 0.0 and intermediate_outs is not None:
- for layer_idx, intermediate_out in intermediate_outs:
- # we assume intermediate_out has the same length & padding
- # as those of encoder_out
- loss_ic, cer_ic = self._calc_ctc_loss(
- intermediate_out, encoder_out_lens, text, text_lengths
- )
- loss_interctc = loss_interctc + loss_ic
+ loss_att, acc_att, cer_att, wer_att = None, None, None, None
+ loss_ctc, cer_ctc = None, None
+ loss_pre = None
+ stats = dict()
- # Collect Intermedaite CTC stats
- stats["loss_interctc_layer{}".format(layer_idx)] = (
- loss_ic.detach() if loss_ic is not None else None
- )
- stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
+ # 1. CTC branch
+ if self.ctc_weight != 0.0:
+ loss_ctc, cer_ctc = self._calc_ctc_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
- loss_interctc = loss_interctc / len(intermediate_outs)
+ # Collect CTC branch stats
+ stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
+ stats["cer_ctc"] = cer_ctc
- # calculate whole encoder loss
- loss_ctc = (
- 1 - self.interctc_weight
- ) * loss_ctc + self.interctc_weight * loss_interctc
+ # Intermediate CTC (optional)
+ loss_interctc = 0.0
+ if self.interctc_weight != 0.0 and intermediate_outs is not None:
+ for layer_idx, intermediate_out in intermediate_outs:
+ # we assume intermediate_out has the same length & padding
+ # as those of encoder_out
+ loss_ic, cer_ic = self._calc_ctc_loss(
+ intermediate_out, encoder_out_lens, text, text_lengths
+ )
+ loss_interctc = loss_interctc + loss_ic
+ # Collect Intermedaite CTC stats
+ stats["loss_interctc_layer{}".format(layer_idx)] = (
+ loss_ic.detach() if loss_ic is not None else None
+ )
+ stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
- # 2b. Attention decoder branch
- if self.ctc_weight != 1.0:
+ loss_interctc = loss_interctc / len(intermediate_outs)
- loss_ret = self._calc_att_loss(
- encoder_out, encoder_out_lens, text, text_lengths
- )
- loss_att, acc_att, cer_att, wer_att, loss_pre = loss_ret[0], loss_ret[1], loss_ret[2], loss_ret[3], loss_ret[4]
- embeds_outputs = None
- if len(loss_ret) > 5:
- embeds_outputs = loss_ret[5]
- if embeds_outputs is not None:
- cos_loss = self._calc_embed_loss(text, text_lengths, embed, embed_lengths, embeds_outputs)
+ # calculate whole encoder loss
+ loss_ctc = (
+ 1 - self.interctc_weight
+ ) * loss_ctc + self.interctc_weight * loss_interctc
- # 3. CTC-Att loss definition
- if self.ctc_weight == 0.0:
- loss = loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight
- elif self.ctc_weight == 1.0:
- loss = loss_ctc
- else:
- loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight
+ # 2b. Attention decoder branch
+ if self.ctc_weight != 1.0:
+ loss_att, acc_att, cer_att, wer_att, loss_pre, loss_pre2 = self._calc_att_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
- # Collect Attn branch stats
- stats["loss_att"] = loss_att.detach() if loss_att is not None else None
- stats["acc"] = acc_att
- stats["cer"] = cer_att
- stats["wer"] = wer_att
- stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre > 0.0 else None
- stats["cos_loss"] = cos_loss.detach().cpu() if cos_loss > 0.0 else None
+ # 3. CTC-Att loss definition
+ if self.ctc_weight == 0.0:
+ loss = loss_att + loss_pre * self.predictor_weight + loss_pre2 * self.predictor_weight
+ elif self.ctc_weight == 1.0:
+ loss = loss_ctc
+ else:
+ loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + loss_pre2 * self.predictor_weight
- stats["loss"] =torch.clone(loss.detach())
+ # Collect Attn branch stats
+ stats["loss_att"] = loss_att.detach() if loss_att is not None else None
+ stats["acc"] = acc_att
+ stats["cer"] = cer_att
+ stats["wer"] = wer_att
+ stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None
+ stats["loss_pre2"] = loss_pre2.detach().cpu() if loss_pre is not None else None
- # force_gatherable: to-device and to-tensor if scalar for DataParallel
- loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
- return loss, stats, weight
+ stats["loss"] = torch.clone(loss.detach())
-
-
-
-
-
-
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
+ return loss, stats, weight
\ No newline at end of file
diff --git a/funasr/models/e2e_sv.py b/funasr/models/e2e_sv.py
new file mode 100644
index 0000000..eff63d9
--- /dev/null
+++ b/funasr/models/e2e_sv.py
@@ -0,0 +1,270 @@
+import logging
+from contextlib import contextmanager
+from distutils.version import LooseVersion
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+from typeguard import check_argument_types
+
+from funasr.layers.abs_normalize import AbsNormalize
+from funasr.losses.label_smoothing_loss import (
+ LabelSmoothingLoss, # noqa: H301
+)
+from funasr.models.ctc import CTC
+from funasr.models.decoder.abs_decoder import AbsDecoder
+from funasr.models.encoder.abs_encoder import AbsEncoder
+from funasr.models.frontend.abs_frontend import AbsFrontend
+from funasr.models.postencoder.abs_postencoder import AbsPostEncoder
+from funasr.models.preencoder.abs_preencoder import AbsPreEncoder
+from funasr.models.specaug.abs_specaug import AbsSpecAug
+from funasr.modules.add_sos_eos import add_sos_eos
+from funasr.modules.e2e_asr_common import ErrorCalculator
+from funasr.modules.nets_utils import th_accuracy
+from funasr.torch_utils.device_funcs import force_gatherable
+from funasr.train.abs_espnet_model import AbsESPnetModel
+
+if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):
+ from torch.cuda.amp import autocast
+else:
+ # Nothing to do if torch<1.6.0
+ @contextmanager
+ def autocast(enabled=True):
+ yield
+
+
+class ESPnetSVModel(AbsESPnetModel):
+ """CTC-attention hybrid Encoder-Decoder model"""
+
+ def __init__(
+ self,
+ vocab_size: int,
+ token_list: Union[Tuple[str, ...], List[str]],
+ frontend: Optional[AbsFrontend],
+ specaug: Optional[AbsSpecAug],
+ normalize: Optional[AbsNormalize],
+ preencoder: Optional[AbsPreEncoder],
+ encoder: AbsEncoder,
+ postencoder: Optional[AbsPostEncoder],
+ pooling_layer: torch.nn.Module,
+ decoder: AbsDecoder,
+ ):
+ assert check_argument_types()
+
+ super().__init__()
+ # note that eos is the same as sos (equivalent ID)
+ self.vocab_size = vocab_size
+ self.token_list = token_list.copy()
+
+ self.frontend = frontend
+ self.specaug = specaug
+ self.normalize = normalize
+ self.preencoder = preencoder
+ self.postencoder = postencoder
+ self.encoder = encoder
+ self.pooling_layer = pooling_layer
+ self.decoder = decoder
+
+ def forward(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
+ """Frontend + Encoder + Decoder + Calc loss
+
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ text: (Batch, Length)
+ text_lengths: (Batch,)
+ """
+ assert text_lengths.dim() == 1, text_lengths.shape
+ # Check that batch_size is unified
+ assert (
+ speech.shape[0]
+ == speech_lengths.shape[0]
+ == text.shape[0]
+ == text_lengths.shape[0]
+ ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)
+ batch_size = speech.shape[0]
+
+ # for data-parallel
+ text = text[:, : text_lengths.max()]
+
+ # 1. Encoder
+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+ intermediate_outs = None
+ if isinstance(encoder_out, tuple):
+ intermediate_outs = encoder_out[1]
+ encoder_out = encoder_out[0]
+
+ loss_att, acc_att, cer_att, wer_att = None, None, None, None
+ loss_ctc, cer_ctc = None, None
+ loss_transducer, cer_transducer, wer_transducer = None, None, None
+ stats = dict()
+
+ # 1. CTC branch
+ if self.ctc_weight != 0.0:
+ loss_ctc, cer_ctc = self._calc_ctc_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
+
+ # Collect CTC branch stats
+ stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None
+ stats["cer_ctc"] = cer_ctc
+
+ # Intermediate CTC (optional)
+ loss_interctc = 0.0
+ if self.interctc_weight != 0.0 and intermediate_outs is not None:
+ for layer_idx, intermediate_out in intermediate_outs:
+ # we assume intermediate_out has the same length & padding
+ # as those of encoder_out
+ loss_ic, cer_ic = self._calc_ctc_loss(
+ intermediate_out, encoder_out_lens, text, text_lengths
+ )
+ loss_interctc = loss_interctc + loss_ic
+
+ # Collect Intermedaite CTC stats
+ stats["loss_interctc_layer{}".format(layer_idx)] = (
+ loss_ic.detach() if loss_ic is not None else None
+ )
+ stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic
+
+ loss_interctc = loss_interctc / len(intermediate_outs)
+
+ # calculate whole encoder loss
+ loss_ctc = (
+ 1 - self.interctc_weight
+ ) * loss_ctc + self.interctc_weight * loss_interctc
+
+ if self.use_transducer_decoder:
+ # 2a. Transducer decoder branch
+ (
+ loss_transducer,
+ cer_transducer,
+ wer_transducer,
+ ) = self._calc_transducer_loss(
+ encoder_out,
+ encoder_out_lens,
+ text,
+ )
+
+ if loss_ctc is not None:
+ loss = loss_transducer + (self.ctc_weight * loss_ctc)
+ else:
+ loss = loss_transducer
+
+ # Collect Transducer branch stats
+ stats["loss_transducer"] = (
+ loss_transducer.detach() if loss_transducer is not None else None
+ )
+ stats["cer_transducer"] = cer_transducer
+ stats["wer_transducer"] = wer_transducer
+
+ else:
+ # 2b. Attention decoder branch
+ if self.ctc_weight != 1.0:
+ loss_att, acc_att, cer_att, wer_att = self._calc_att_loss(
+ encoder_out, encoder_out_lens, text, text_lengths
+ )
+
+ # 3. CTC-Att loss definition
+ if self.ctc_weight == 0.0:
+ loss = loss_att
+ elif self.ctc_weight == 1.0:
+ loss = loss_ctc
+ else:
+ loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att
+
+ # Collect Attn branch stats
+ stats["loss_att"] = loss_att.detach() if loss_att is not None else None
+ stats["acc"] = acc_att
+ stats["cer"] = cer_att
+ stats["wer"] = wer_att
+
+ # Collect total loss stats
+ stats["loss"] = torch.clone(loss.detach())
+
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
+ return loss, stats, weight
+
+ def collect_feats(
+ self,
+ speech: torch.Tensor,
+ speech_lengths: torch.Tensor,
+ text: torch.Tensor,
+ text_lengths: torch.Tensor,
+ ) -> Dict[str, torch.Tensor]:
+ if self.extract_feats_in_collect_stats:
+ feats, feats_lengths = self._extract_feats(speech, speech_lengths)
+ else:
+ # Generate dummy stats if extract_feats_in_collect_stats is False
+ logging.warning(
+ "Generating dummy stats for feats and feats_lengths, "
+ "because encoder_conf.extract_feats_in_collect_stats is "
+ f"{self.extract_feats_in_collect_stats}"
+ )
+ feats, feats_lengths = speech, speech_lengths
+ return {"feats": feats, "feats_lengths": feats_lengths}
+
+ def encode(
+ self, speech: torch.Tensor, speech_lengths: torch.Tensor
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Frontend + Encoder. Note that this method is used by asr_inference.py
+
+ Args:
+ speech: (Batch, Length, ...)
+ speech_lengths: (Batch, )
+ """
+ with autocast(False):
+ # 1. Extract feats
+ feats, feats_lengths = self._extract_feats(speech, speech_lengths)
+
+ # 2. Data augmentation
+ if self.specaug is not None and self.training:
+ feats, feats_lengths = self.specaug(feats, feats_lengths)
+
+ # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
+ if self.normalize is not None:
+ feats, feats_lengths = self.normalize(feats, feats_lengths)
+
+ # Pre-encoder, e.g. used for raw input data
+ if self.preencoder is not None:
+ feats, feats_lengths = self.preencoder(feats, feats_lengths)
+
+ # 4. Forward encoder
+ # feats: (Batch, Length, Dim) -> (Batch, Channel, Length2, Dim2)
+ encoder_out, encoder_out_lens = self.encoder(feats, feats_lengths)
+
+ # Post-encoder, e.g. NLU
+ if self.postencoder is not None:
+ encoder_out, encoder_out_lens = self.postencoder(
+ encoder_out, encoder_out_lens
+ )
+
+ return encoder_out, encoder_out_lens
+
+ def _extract_feats(
+ self, speech: torch.Tensor, speech_lengths: torch.Tensor
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ assert speech_lengths.dim() == 1, speech_lengths.shape
+
+ # for data-parallel
+ speech = speech[:, : speech_lengths.max()]
+
+ if self.frontend is not None:
+ # Frontend
+ # e.g. STFT and Feature extract
+ # data_loader may send time-domain signal in this case
+ # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim)
+ feats, feats_lengths = self.frontend(speech, speech_lengths)
+ else:
+ # No frontend and no feature extract
+ feats, feats_lengths = speech, speech_lengths
+ return feats, feats_lengths
diff --git a/funasr/models/e2e_uni_asr.py b/funasr/models/e2e_uni_asr.py
index 03fbca9..ad6fe41 100644
--- a/funasr/models/e2e_uni_asr.py
+++ b/funasr/models/e2e_uni_asr.py
@@ -198,16 +198,15 @@
# for data-parallel
text = text[:, : text_lengths.max()]
- speech = speech[:, :speech_lengths.max(), :]
+ speech = speech[:, :speech_lengths.max()]
ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind)
- speech_raw = speech.clone().to(speech.device)
# 1. Encoder
if self.enable_maas_finetune:
with torch.no_grad():
- encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
+ speech_raw, encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
else:
- encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
+ speech_raw, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
intermediate_outs = None
if isinstance(encoder_out, tuple):
@@ -486,7 +485,7 @@
# 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
if self.normalize is not None:
feats, feats_lengths = self.normalize(feats, feats_lengths)
-
+ speech_raw = feats.clone().to(feats.device)
# Pre-encoder, e.g. used for raw input data
if self.preencoder is not None:
feats, feats_lengths = self.preencoder(feats, feats_lengths)
@@ -523,7 +522,7 @@
if intermediate_outs is not None:
return (encoder_out, intermediate_outs), encoder_out_lens
- return encoder_out, encoder_out_lens
+ return speech_raw, encoder_out, encoder_out_lens
def encode2(
self,
diff --git a/funasr/models/e2e_vad.py b/funasr/models/e2e_vad.py
new file mode 100755
index 0000000..35b9260
--- /dev/null
+++ b/funasr/models/e2e_vad.py
@@ -0,0 +1,567 @@
+from enum import Enum
+from typing import List, Tuple, Dict, Any
+
+import torch
+from torch import nn
+import math
+from funasr.models.encoder.fsmn_encoder import FSMN
+# from checkpoint import load_checkpoint
+
+
+class VadStateMachine(Enum):
+ kVadInStateStartPointNotDetected = 1
+ kVadInStateInSpeechSegment = 2
+ kVadInStateEndPointDetected = 3
+
+
+class FrameState(Enum):
+ kFrameStateInvalid = -1
+ kFrameStateSpeech = 1
+ kFrameStateSil = 0
+
+
+# final voice/unvoice state per frame
+class AudioChangeState(Enum):
+ kChangeStateSpeech2Speech = 0
+ kChangeStateSpeech2Sil = 1
+ kChangeStateSil2Sil = 2
+ kChangeStateSil2Speech = 3
+ kChangeStateNoBegin = 4
+ kChangeStateInvalid = 5
+
+
+class VadDetectMode(Enum):
+ kVadSingleUtteranceDetectMode = 0
+ kVadMutipleUtteranceDetectMode = 1
+
+
+class VADXOptions:
+ def __init__(
+ self,
+ sample_rate: int = 16000,
+ detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value,
+ snr_mode: int = 0,
+ max_end_silence_time: int = 800,
+ max_start_silence_time: int = 3000,
+ do_start_point_detection: bool = True,
+ do_end_point_detection: bool = True,
+ window_size_ms: int = 200,
+ sil_to_speech_time_thres: int = 150,
+ speech_to_sil_time_thres: int = 150,
+ speech_2_noise_ratio: float = 1.0,
+ do_extend: int = 1,
+ lookback_time_start_point: int = 200,
+ lookahead_time_end_point: int = 100,
+ max_single_segment_time: int = 60000,
+ nn_eval_block_size: int = 8,
+ dcd_block_size: int = 4,
+ snr_thres: int = -100.0,
+ noise_frame_num_used_for_snr: int = 100,
+ decibel_thres: int = -100.0,
+ speech_noise_thres: float = 0.6,
+ fe_prior_thres: float = 1e-4,
+ silence_pdf_num: int = 1,
+ sil_pdf_ids: List[int] = [0],
+ speech_noise_thresh_low: float = -0.1,
+ speech_noise_thresh_high: float = 0.3,
+ output_frame_probs: bool = False,
+ frame_in_ms: int = 10,
+ frame_length_ms: int = 25,
+ ):
+ self.sample_rate = sample_rate
+ self.detect_mode = detect_mode
+ self.snr_mode = snr_mode
+ self.max_end_silence_time = max_end_silence_time
+ self.max_start_silence_time = max_start_silence_time
+ self.do_start_point_detection = do_start_point_detection
+ self.do_end_point_detection = do_end_point_detection
+ self.window_size_ms = window_size_ms
+ self.sil_to_speech_time_thres = sil_to_speech_time_thres
+ self.speech_to_sil_time_thres = speech_to_sil_time_thres
+ self.speech_2_noise_ratio = speech_2_noise_ratio
+ self.do_extend = do_extend
+ self.lookback_time_start_point = lookback_time_start_point
+ self.lookahead_time_end_point = lookahead_time_end_point
+ self.max_single_segment_time = max_single_segment_time
+ self.nn_eval_block_size = nn_eval_block_size
+ self.dcd_block_size = dcd_block_size
+ self.snr_thres = snr_thres
+ self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr
+ self.decibel_thres = decibel_thres
+ self.speech_noise_thres = speech_noise_thres
+ self.fe_prior_thres = fe_prior_thres
+ self.silence_pdf_num = silence_pdf_num
+ self.sil_pdf_ids = sil_pdf_ids
+ self.speech_noise_thresh_low = speech_noise_thresh_low
+ self.speech_noise_thresh_high = speech_noise_thresh_high
+ self.output_frame_probs = output_frame_probs
+ self.frame_in_ms = frame_in_ms
+ self.frame_length_ms = frame_length_ms
+
+
+class E2EVadSpeechBufWithDoa(object):
+ def __init__(self):
+ self.start_ms = 0
+ self.end_ms = 0
+ self.buffer = []
+ self.contain_seg_start_point = False
+ self.contain_seg_end_point = False
+ self.doa = 0
+
+ def Reset(self):
+ self.start_ms = 0
+ self.end_ms = 0
+ self.buffer = []
+ self.contain_seg_start_point = False
+ self.contain_seg_end_point = False
+ self.doa = 0
+
+
+class E2EVadFrameProb(object):
+ def __init__(self):
+ self.noise_prob = 0.0
+ self.speech_prob = 0.0
+ self.score = 0.0
+ self.frame_id = 0
+ self.frm_state = 0
+
+
+class WindowDetector(object):
+ def __init__(self, window_size_ms: int, sil_to_speech_time: int,
+ speech_to_sil_time: int, frame_size_ms: int):
+ self.window_size_ms = window_size_ms
+ self.sil_to_speech_time = sil_to_speech_time
+ self.speech_to_sil_time = speech_to_sil_time
+ self.frame_size_ms = frame_size_ms
+
+ self.win_size_frame = int(window_size_ms / frame_size_ms)
+ self.win_sum = 0
+ self.win_state = [0 for i in range(0, self.win_size_frame)] # 鍒濆鍖栫獥
+
+ self.cur_win_pos = 0
+ self.pre_frame_state = FrameState.kFrameStateSil
+ self.cur_frame_state = FrameState.kFrameStateSil
+ self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms)
+ self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms)
+
+ self.voice_last_frame_count = 0
+ self.noise_last_frame_count = 0
+ self.hydre_frame_count = 0
+
+ def Reset(self) -> None:
+ self.cur_win_pos = 0
+ self.win_sum = 0
+ self.win_state = [0 for i in range(0, self.win_size_frame)]
+ self.pre_frame_state = FrameState.kFrameStateSil
+ self.cur_frame_state = FrameState.kFrameStateSil
+ self.voice_last_frame_count = 0
+ self.noise_last_frame_count = 0
+ self.hydre_frame_count = 0
+
+ def GetWinSize(self) -> int:
+ return int(self.win_size_frame)
+
+ def DetectOneFrame(self, frameState: FrameState, frame_count: int) -> AudioChangeState:
+ cur_frame_state = FrameState.kFrameStateSil
+ if frameState == FrameState.kFrameStateSpeech:
+ cur_frame_state = 1
+ elif frameState == FrameState.kFrameStateSil:
+ cur_frame_state = 0
+ else:
+ return AudioChangeState.kChangeStateInvalid
+ self.win_sum -= self.win_state[self.cur_win_pos]
+ self.win_sum += cur_frame_state
+ self.win_state[self.cur_win_pos] = cur_frame_state
+ self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame
+
+ if self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres:
+ self.pre_frame_state = FrameState.kFrameStateSpeech
+ return AudioChangeState.kChangeStateSil2Speech
+
+ if self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres:
+ self.pre_frame_state = FrameState.kFrameStateSil
+ return AudioChangeState.kChangeStateSpeech2Sil
+
+ if self.pre_frame_state == FrameState.kFrameStateSil:
+ return AudioChangeState.kChangeStateSil2Sil
+ if self.pre_frame_state == FrameState.kFrameStateSpeech:
+ return AudioChangeState.kChangeStateSpeech2Speech
+ return AudioChangeState.kChangeStateInvalid
+
+ def FrameSizeMs(self) -> int:
+ return int(self.frame_size_ms)
+
+
+class E2EVadModel(torch.nn.Module):
+ def __init__(self, encoder: FSMN, vad_post_args: Dict[str, Any]):
+ super(E2EVadModel, self).__init__()
+ self.vad_opts = VADXOptions(**vad_post_args)
+ self.windows_detector = WindowDetector(self.vad_opts.window_size_ms,
+ self.vad_opts.sil_to_speech_time_thres,
+ self.vad_opts.speech_to_sil_time_thres,
+ self.vad_opts.frame_in_ms)
+ self.encoder = encoder
+ # init variables
+ self.is_final_send = False
+ self.data_buf_start_frame = 0
+ self.frm_cnt = 0
+ self.latest_confirmed_speech_frame = 0
+ self.lastest_confirmed_silence_frame = -1
+ self.continous_silence_frame_count = 0
+ self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected
+ self.confirmed_start_frame = -1
+ self.confirmed_end_frame = -1
+ self.number_end_time_detected = 0
+ self.is_callback_with_sign = False
+ self.sil_frame = 0
+ self.sil_pdf_ids = self.vad_opts.sil_pdf_ids
+ self.noise_average_decibel = -100.0
+ self.pre_end_silence_detected = False
+
+ self.output_data_buf = []
+ self.frame_probs = []
+ self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres
+ self.speech_noise_thres = self.vad_opts.speech_noise_thres
+ self.scores = None
+ self.max_time_out = False
+ self.decibel = []
+ self.data_buf = None
+ self.waveform = None
+ self.ResetDetection()
+
+ def AllResetDetection(self):
+ self.is_final_send = False
+ self.data_buf_start_frame = 0
+ self.frm_cnt = 0
+ self.latest_confirmed_speech_frame = 0
+ self.lastest_confirmed_silence_frame = -1
+ self.continous_silence_frame_count = 0
+ self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected
+ self.confirmed_start_frame = -1
+ self.confirmed_end_frame = -1
+ self.number_end_time_detected = 0
+ self.is_callback_with_sign = False
+ self.sil_frame = 0
+ self.sil_pdf_ids = self.vad_opts.sil_pdf_ids
+ self.noise_average_decibel = -100.0
+ self.pre_end_silence_detected = False
+
+ self.output_data_buf = []
+ self.frame_probs = []
+ self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres
+ self.speech_noise_thres = self.vad_opts.speech_noise_thres
+ self.scores = None
+ self.max_time_out = False
+ self.decibel = []
+ self.data_buf = None
+ self.waveform = None
+ self.ResetDetection()
+
+ def ResetDetection(self):
+ self.continous_silence_frame_count = 0
+ self.latest_confirmed_speech_frame = 0
+ self.lastest_confirmed_silence_frame = -1
+ self.confirmed_start_frame = -1
+ self.confirmed_end_frame = -1
+ self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected
+ self.windows_detector.Reset()
+ self.sil_frame = 0
+ self.frame_probs = []
+
+ def ComputeDecibel(self) -> None:
+ frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000)
+ frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)
+ self.data_buf = self.waveform[0] # 鎸囧悜self.waveform[0]
+ for offset in range(0, self.waveform.shape[1] - frame_sample_length, frame_shift_length):
+ self.decibel.append(
+ 10 * math.log10((self.waveform[0][offset: offset + frame_sample_length]).square().sum() + \
+ 0.000001))
+
+ def ComputeScores(self, feats: torch.Tensor, feats_lengths: int) -> None:
+ self.scores = self.encoder(feats) # return B * T * D
+ self.frm_cnt = feats_lengths # frame
+ # return self.scores
+
+ def PopDataBufTillFrame(self, frame_idx: int) -> None: # need check again
+ while self.data_buf_start_frame < frame_idx:
+ if len(self.data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):
+ self.data_buf_start_frame += 1
+ self.data_buf = self.waveform[0][self.data_buf_start_frame * int(
+ self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):]
+ # for i in range(0, int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)):
+ # self.data_buf.popleft()
+ # self.data_buf_start_frame += 1
+
+ def PopDataToOutputBuf(self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool,
+ last_frm_is_end_point: bool, end_point_is_sent_end: bool) -> None:
+ self.PopDataBufTillFrame(start_frm)
+ expected_sample_number = int(frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000)
+ if last_frm_is_end_point:
+ extra_sample = max(0, int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - \
+ self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000))
+ expected_sample_number += int(extra_sample)
+ if end_point_is_sent_end:
+ # expected_sample_number = max(expected_sample_number, len(self.data_buf))
+ pass
+
+ if len(self.output_data_buf) == 0 or first_frm_is_start_point:
+ self.output_data_buf.append(E2EVadSpeechBufWithDoa())
+ self.output_data_buf[-1].Reset()
+ self.output_data_buf[-1].start_ms = start_frm * self.vad_opts.frame_in_ms
+ self.output_data_buf[-1].end_ms = self.output_data_buf[-1].start_ms
+ self.output_data_buf[-1].doa = 0
+ cur_seg = self.output_data_buf[-1]
+ if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms:
+ print('warning')
+ out_pos = len(cur_seg.buffer) # cur_seg.buff鐜板湪娌″仛浠讳綍鎿嶄綔
+ data_to_pop = 0
+ if end_point_is_sent_end:
+ data_to_pop = expected_sample_number
+ else:
+ data_to_pop = int(frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)
+ # if data_to_pop > len(self.data_buf_)
+ # pass
+ cur_seg.doa = 0
+ for sample_cpy_out in range(0, data_to_pop):
+ # cur_seg.buffer[out_pos ++] = data_buf_.back();
+ out_pos += 1
+ for sample_cpy_out in range(data_to_pop, expected_sample_number):
+ # cur_seg.buffer[out_pos++] = data_buf_.back()
+ out_pos += 1
+ if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms:
+ print('warning')
+ self.data_buf_start_frame += frm_cnt
+ cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms
+ if first_frm_is_start_point:
+ cur_seg.contain_seg_start_point = True
+ if last_frm_is_end_point:
+ cur_seg.contain_seg_end_point = True
+
+ def OnSilenceDetected(self, valid_frame: int):
+ self.lastest_confirmed_silence_frame = valid_frame
+ if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
+ self.PopDataBufTillFrame(valid_frame)
+ # silence_detected_callback_
+ # pass
+
+ def OnVoiceDetected(self, valid_frame: int) -> None:
+ self.latest_confirmed_speech_frame = valid_frame
+ if True: # is_new_api_enable_ = True
+ self.PopDataToOutputBuf(valid_frame, 1, False, False, False)
+
+ def OnVoiceStart(self, start_frame: int, fake_result: bool = False) -> None:
+ if self.vad_opts.do_start_point_detection:
+ pass
+ if self.confirmed_start_frame != -1:
+ print('warning')
+ else:
+ self.confirmed_start_frame = start_frame
+
+ if not fake_result and self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
+ self.PopDataToOutputBuf(self.confirmed_start_frame, 1, True, False, False)
+
+ def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool) -> None:
+ for t in range(self.latest_confirmed_speech_frame + 1, end_frame):
+ self.OnVoiceDetected(t)
+ if self.vad_opts.do_end_point_detection:
+ pass
+ if self.confirmed_end_frame != -1:
+ print('warning')
+ else:
+ self.confirmed_end_frame = end_frame
+ if not fake_result:
+ self.sil_frame = 0
+ self.PopDataToOutputBuf(self.confirmed_end_frame, 1, False, True, is_last_frame)
+ self.number_end_time_detected += 1
+
+ def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int) -> None:
+ if is_final_frame:
+ self.OnVoiceEnd(cur_frm_idx, False, True)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+
+ def GetLatency(self) -> int:
+ return int(self.LatencyFrmNumAtStartPoint() * self.vad_opts.frame_in_ms)
+
+ def LatencyFrmNumAtStartPoint(self) -> int:
+ vad_latency = self.windows_detector.GetWinSize()
+ if self.vad_opts.do_extend:
+ vad_latency += int(self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms)
+ return vad_latency
+
+ def GetFrameState(self, t: int) -> FrameState:
+ frame_state = FrameState.kFrameStateInvalid
+ cur_decibel = self.decibel[t]
+ cur_snr = cur_decibel - self.noise_average_decibel
+ # for each frame, calc log posterior probability of each state
+ if cur_decibel < self.vad_opts.decibel_thres:
+ frame_state = FrameState.kFrameStateSil
+ self.DetectOneFrame(frame_state, t, False)
+ return frame_state
+
+ sum_score = 0.0
+ noise_prob = 0.0
+ assert len(self.sil_pdf_ids) == self.vad_opts.silence_pdf_num
+ if len(self.sil_pdf_ids) > 0:
+ assert len(self.scores) == 1 # 鍙敮鎸乥atch_size = 1鐨勬祴璇�
+ sil_pdf_scores = [self.scores[0][t][sil_pdf_id] for sil_pdf_id in self.sil_pdf_ids]
+ sum_score = sum(sil_pdf_scores)
+ noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio
+ # total_score = sum(self.scores[0][t][:])
+ total_score = 1.0
+ sum_score = total_score - sum_score
+ speech_prob = math.log(sum_score)
+ if self.vad_opts.output_frame_probs:
+ frame_prob = E2EVadFrameProb()
+ frame_prob.noise_prob = noise_prob
+ frame_prob.speech_prob = speech_prob
+ frame_prob.score = sum_score
+ frame_prob.frame_id = t
+ self.frame_probs.append(frame_prob)
+ if math.exp(speech_prob) >= math.exp(noise_prob) + self.speech_noise_thres:
+ if cur_snr >= self.vad_opts.snr_thres and cur_decibel >= self.vad_opts.decibel_thres:
+ frame_state = FrameState.kFrameStateSpeech
+ else:
+ frame_state = FrameState.kFrameStateSil
+ else:
+ frame_state = FrameState.kFrameStateSil
+ if self.noise_average_decibel < -99.9:
+ self.noise_average_decibel = cur_decibel
+ else:
+ self.noise_average_decibel = (cur_decibel + self.noise_average_decibel * (
+ self.vad_opts.noise_frame_num_used_for_snr
+ - 1)) / self.vad_opts.noise_frame_num_used_for_snr
+
+ return frame_state
+
+ def forward(self, feats: torch.Tensor, feats_lengths: int, waveform: torch.tensor) -> List[List[List[int]]]:
+ self.AllResetDetection()
+ self.waveform = waveform # compute decibel for each frame
+ self.ComputeDecibel()
+ self.ComputeScores(feats, feats_lengths)
+ assert len(self.decibel) == len(self.scores[0]) # 淇濊瘉甯ф暟涓�鑷�
+ self.DetectLastFrames()
+ segments = []
+ for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 now
+ segment_batch = []
+ for i in range(0, len(self.output_data_buf)):
+ segment = [self.output_data_buf[i].start_ms, self.output_data_buf[i].end_ms]
+ segment_batch.append(segment)
+ segments.append(segment_batch)
+ return segments
+
+ def DetectLastFrames(self) -> int:
+ if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected:
+ return 0
+ if self.vad_opts.nn_eval_block_size != self.vad_opts.dcd_block_size:
+ frame_state = FrameState.kFrameStateInvalid
+ for t in range(0, self.frm_cnt):
+ frame_state = self.GetFrameState(t)
+ self.DetectOneFrame(frame_state, t, t == self.frm_cnt - 1)
+ else:
+ pass
+ return 0
+
+ def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool) -> None:
+ tmp_cur_frm_state = FrameState.kFrameStateInvalid
+ if cur_frm_state == FrameState.kFrameStateSpeech:
+ if math.fabs(1.0) > self.vad_opts.fe_prior_thres:
+ tmp_cur_frm_state = FrameState.kFrameStateSpeech
+ else:
+ tmp_cur_frm_state = FrameState.kFrameStateSil
+ elif cur_frm_state == FrameState.kFrameStateSil:
+ tmp_cur_frm_state = FrameState.kFrameStateSil
+ state_change = self.windows_detector.DetectOneFrame(tmp_cur_frm_state, cur_frm_idx)
+ frm_shift_in_ms = self.vad_opts.frame_in_ms
+ if AudioChangeState.kChangeStateSil2Speech == state_change:
+ silence_frame_count = self.continous_silence_frame_count
+ self.continous_silence_frame_count = 0
+ self.pre_end_silence_detected = False
+ start_frame = 0
+ if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
+ start_frame = max(self.data_buf_start_frame, cur_frm_idx - self.LatencyFrmNumAtStartPoint())
+ self.OnVoiceStart(start_frame)
+ self.vad_state_machine = VadStateMachine.kVadInStateInSpeechSegment
+ for t in range(start_frame + 1, cur_frm_idx + 1):
+ self.OnVoiceDetected(t)
+ elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+ for t in range(self.latest_confirmed_speech_frame + 1, cur_frm_idx):
+ self.OnVoiceDetected(t)
+ if cur_frm_idx - self.confirmed_start_frame + 1 > \
+ self.vad_opts.max_single_segment_time / frm_shift_in_ms:
+ self.OnVoiceEnd(cur_frm_idx, False, False)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ elif not is_final_frame:
+ self.OnVoiceDetected(cur_frm_idx)
+ else:
+ self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx)
+ else:
+ pass
+ elif AudioChangeState.kChangeStateSpeech2Sil == state_change:
+ self.continous_silence_frame_count = 0
+ if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
+ pass
+ elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+ if cur_frm_idx - self.confirmed_start_frame + 1 > \
+ self.vad_opts.max_single_segment_time / frm_shift_in_ms:
+ self.OnVoiceEnd(cur_frm_idx, False, False)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ elif not is_final_frame:
+ self.OnVoiceDetected(cur_frm_idx)
+ else:
+ self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx)
+ else:
+ pass
+ elif AudioChangeState.kChangeStateSpeech2Speech == state_change:
+ self.continous_silence_frame_count = 0
+ if self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+ if cur_frm_idx - self.confirmed_start_frame + 1 > \
+ self.vad_opts.max_single_segment_time / frm_shift_in_ms:
+ self.max_time_out = True
+ self.OnVoiceEnd(cur_frm_idx, False, False)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ elif not is_final_frame:
+ self.OnVoiceDetected(cur_frm_idx)
+ else:
+ self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx)
+ else:
+ pass
+ elif AudioChangeState.kChangeStateSil2Sil == state_change:
+ self.continous_silence_frame_count += 1
+ if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
+ # silence timeout, return zero length decision
+ if ((self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value) and (
+ self.continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time)) \
+ or (is_final_frame and self.number_end_time_detected == 0):
+ for t in range(self.lastest_confirmed_silence_frame + 1, cur_frm_idx):
+ self.OnSilenceDetected(t)
+ self.OnVoiceStart(0, True)
+ self.OnVoiceEnd(0, True, False);
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ else:
+ if cur_frm_idx >= self.LatencyFrmNumAtStartPoint():
+ self.OnSilenceDetected(cur_frm_idx - self.LatencyFrmNumAtStartPoint())
+ elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+ if self.continous_silence_frame_count * frm_shift_in_ms >= self.max_end_sil_frame_cnt_thresh:
+ lookback_frame = int(self.max_end_sil_frame_cnt_thresh / frm_shift_in_ms)
+ if self.vad_opts.do_extend:
+ lookback_frame -= int(self.vad_opts.lookahead_time_end_point / frm_shift_in_ms)
+ lookback_frame -= 1
+ lookback_frame = max(0, lookback_frame)
+ self.OnVoiceEnd(cur_frm_idx - lookback_frame, False, False)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ elif cur_frm_idx - self.confirmed_start_frame + 1 > \
+ self.vad_opts.max_single_segment_time / frm_shift_in_ms:
+ self.OnVoiceEnd(cur_frm_idx, False, False)
+ self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
+ elif self.vad_opts.do_extend and not is_final_frame:
+ if self.continous_silence_frame_count <= int(
+ self.vad_opts.lookahead_time_end_point / frm_shift_in_ms):
+ self.OnVoiceDetected(cur_frm_idx)
+ else:
+ self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx)
+ else:
+ pass
+
+ if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and \
+ self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value:
+ self.ResetDetection()
diff --git a/funasr/models/encoder/conformer_encoder.py b/funasr/models/encoder/conformer_encoder.py
index 2df2ba6..7c7f661 100644
--- a/funasr/models/encoder/conformer_encoder.py
+++ b/funasr/models/encoder/conformer_encoder.py
@@ -41,7 +41,7 @@
from funasr.modules.subsampling import Conv2dSubsampling8
from funasr.modules.subsampling import TooShortUttError
from funasr.modules.subsampling import check_short_utt
-
+from funasr.modules.subsampling import Conv2dSubsamplingPad
class ConvolutionModule(nn.Module):
"""ConvolutionModule in Conformer model.
@@ -386,6 +386,13 @@
dropout_rate,
pos_enc_class(output_size, positional_dropout_rate),
)
+ elif input_layer == "conv2dpad":
+ self.embed = Conv2dSubsamplingPad(
+ input_size,
+ output_size,
+ dropout_rate,
+ pos_enc_class(output_size, positional_dropout_rate),
+ )
elif input_layer == "conv2d2":
self.embed = Conv2dSubsampling2(
input_size,
@@ -546,6 +553,7 @@
or isinstance(self.embed, Conv2dSubsampling2)
or isinstance(self.embed, Conv2dSubsampling6)
or isinstance(self.embed, Conv2dSubsampling8)
+ or isinstance(self.embed, Conv2dSubsamplingPad)
):
short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1))
if short_status:
diff --git a/funasr/models/encoder/data2vec_encoder.py b/funasr/models/encoder/data2vec_encoder.py
new file mode 100644
index 0000000..fd1796c
--- /dev/null
+++ b/funasr/models/encoder/data2vec_encoder.py
@@ -0,0 +1,577 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import math
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+from typeguard import check_argument_types
+
+from funasr.models.encoder.abs_encoder import AbsEncoder
+from funasr.modules.data2vec.data_utils import compute_mask_indices
+from funasr.modules.data2vec.ema_module import EMAModule
+from funasr.modules.data2vec.grad_multiply import GradMultiply
+from funasr.modules.data2vec.wav2vec2 import (
+ ConvFeatureExtractionModel,
+ TransformerEncoder,
+)
+from funasr.modules.nets_utils import make_pad_mask
+
+
+def get_annealed_rate(start, end, curr_step, total_steps):
+ r = end - start
+ pct_remaining = 1 - curr_step / total_steps
+ return end - r * pct_remaining
+
+
+class Data2VecEncoder(AbsEncoder):
+ def __init__(
+ self,
+ # for ConvFeatureExtractionModel
+ input_size: int = None,
+ extractor_mode: str = None,
+ conv_feature_layers: str = "[(512,2,2)] + [(512,2,2)]",
+ # for Transformer Encoder
+ ## model architecture
+ layer_type: str = "transformer",
+ layer_norm_first: bool = False,
+ encoder_layers: int = 12,
+ encoder_embed_dim: int = 768,
+ encoder_ffn_embed_dim: int = 3072,
+ encoder_attention_heads: int = 12,
+ activation_fn: str = "gelu",
+ ## dropouts
+ dropout: float = 0.1,
+ attention_dropout: float = 0.1,
+ activation_dropout: float = 0.0,
+ encoder_layerdrop: float = 0.0,
+ dropout_input: float = 0.0,
+ dropout_features: float = 0.0,
+ ## grad settings
+ feature_grad_mult: float = 1.0,
+ ## masking
+ mask_prob: float = 0.65,
+ mask_length: int = 10,
+ mask_selection: str = "static",
+ mask_other: int = 0,
+ no_mask_overlap: bool = False,
+ mask_min_space: int = 1,
+ require_same_masks: bool = True, # if set as True, collate_fn should be clipping
+ mask_dropout: float = 0.0,
+ ## channel masking
+ mask_channel_length: int = 10,
+ mask_channel_prob: float = 0.0,
+ mask_channel_before: bool = False,
+ mask_channel_selection: str = "static",
+ mask_channel_other: int = 0,
+ no_mask_channel_overlap: bool = False,
+ mask_channel_min_space: int = 1,
+ ## positional embeddings
+ conv_pos: int = 128,
+ conv_pos_groups: int = 16,
+ pos_conv_depth: int = 1,
+ max_positions: int = 100000,
+ # EMA module
+ average_top_k_layers: int = 8,
+ layer_norm_target_layer: bool = False,
+ instance_norm_target_layer: bool = False,
+ instance_norm_targets: bool = False,
+ layer_norm_targets: bool = False,
+ batch_norm_target_layer: bool = False,
+ group_norm_target_layer: bool = False,
+ ema_decay: float = 0.999,
+ ema_end_decay: float = 0.9999,
+ ema_anneal_end_step: int = 100000,
+ ema_transformer_only: bool = True,
+ ema_layers_only: bool = True,
+ min_target_var: float = 0.1,
+ min_pred_var: float = 0.01,
+ # Loss
+ loss_beta: float = 0.0,
+ loss_scale: float = None,
+ # FP16 optimization
+ required_seq_len_multiple: int = 2,
+ ):
+ assert check_argument_types()
+ super().__init__()
+
+ # ConvFeatureExtractionModel
+ self.conv_feature_layers = conv_feature_layers
+ feature_enc_layers = eval(conv_feature_layers)
+ self.extractor_embed = feature_enc_layers[-1][0]
+ self.feature_extractor = ConvFeatureExtractionModel(
+ conv_layers=feature_enc_layers,
+ dropout=0.0,
+ mode=extractor_mode,
+ in_d=input_size,
+ )
+
+ # Transformer Encoder
+ ## model architecture
+ self.layer_type = layer_type
+ self.layer_norm_first = layer_norm_first
+ self.encoder_layers = encoder_layers
+ self.encoder_embed_dim = encoder_embed_dim
+ self.encoder_ffn_embed_dim = encoder_ffn_embed_dim
+ self.encoder_attention_heads = encoder_attention_heads
+ self.activation_fn = activation_fn
+ ## dropout
+ self.dropout = dropout
+ self.attention_dropout = attention_dropout
+ self.activation_dropout = activation_dropout
+ self.encoder_layerdrop = encoder_layerdrop
+ self.dropout_input = dropout_input
+ self.dropout_features = dropout_features
+ ## grad settings
+ self.feature_grad_mult = feature_grad_mult
+ ## masking
+ self.mask_prob = mask_prob
+ self.mask_length = mask_length
+ self.mask_selection = mask_selection
+ self.mask_other = mask_other
+ self.no_mask_overlap = no_mask_overlap
+ self.mask_min_space = mask_min_space
+ self.require_same_masks = require_same_masks # if set as True, collate_fn should be clipping
+ self.mask_dropout = mask_dropout
+ ## channel masking
+ self.mask_channel_length = mask_channel_length
+ self.mask_channel_prob = mask_channel_prob
+ self.mask_channel_before = mask_channel_before
+ self.mask_channel_selection = mask_channel_selection
+ self.mask_channel_other = mask_channel_other
+ self.no_mask_channel_overlap = no_mask_channel_overlap
+ self.mask_channel_min_space = mask_channel_min_space
+ ## positional embeddings
+ self.conv_pos = conv_pos
+ self.conv_pos_groups = conv_pos_groups
+ self.pos_conv_depth = pos_conv_depth
+ self.max_positions = max_positions
+ self.mask_emb = nn.Parameter(torch.FloatTensor(self.encoder_embed_dim).uniform_())
+ self.encoder = TransformerEncoder(
+ dropout=self.dropout,
+ encoder_embed_dim=self.encoder_embed_dim,
+ required_seq_len_multiple=required_seq_len_multiple,
+ pos_conv_depth=self.pos_conv_depth,
+ conv_pos=self.conv_pos,
+ conv_pos_groups=self.conv_pos_groups,
+ # transformer layers
+ layer_type=self.layer_type,
+ encoder_layers=self.encoder_layers,
+ encoder_ffn_embed_dim=self.encoder_ffn_embed_dim,
+ encoder_attention_heads=self.encoder_attention_heads,
+ attention_dropout=self.attention_dropout,
+ activation_dropout=self.activation_dropout,
+ activation_fn=self.activation_fn,
+ layer_norm_first=self.layer_norm_first,
+ encoder_layerdrop=self.encoder_layerdrop,
+ max_positions=self.max_positions,
+ )
+ ## projections and dropouts
+ self.post_extract_proj = nn.Linear(self.extractor_embed, self.encoder_embed_dim)
+ self.dropout_input = nn.Dropout(self.dropout_input)
+ self.dropout_features = nn.Dropout(self.dropout_features)
+ self.layer_norm = torch.nn.LayerNorm(self.extractor_embed)
+ self.final_proj = nn.Linear(self.encoder_embed_dim, self.encoder_embed_dim)
+
+ # EMA module
+ self.average_top_k_layers = average_top_k_layers
+ self.layer_norm_target_layer = layer_norm_target_layer
+ self.instance_norm_target_layer = instance_norm_target_layer
+ self.instance_norm_targets = instance_norm_targets
+ self.layer_norm_targets = layer_norm_targets
+ self.batch_norm_target_layer = batch_norm_target_layer
+ self.group_norm_target_layer = group_norm_target_layer
+ self.ema_decay = ema_decay
+ self.ema_end_decay = ema_end_decay
+ self.ema_anneal_end_step = ema_anneal_end_step
+ self.ema_transformer_only = ema_transformer_only
+ self.ema_layers_only = ema_layers_only
+ self.min_target_var = min_target_var
+ self.min_pred_var = min_pred_var
+ self.ema = None
+
+ # Loss
+ self.loss_beta = loss_beta
+ self.loss_scale = loss_scale
+
+ # FP16 optimization
+ self.required_seq_len_multiple = required_seq_len_multiple
+
+ self.num_updates = 0
+
+ logging.info("Data2VecEncoder settings: {}".format(self.__dict__))
+
+ def make_ema_teacher(self):
+ skip_keys = set()
+ if self.ema_layers_only:
+ self.ema_transformer_only = True
+ for k, _ in self.encoder.pos_conv.named_parameters():
+ skip_keys.add(f"pos_conv.{k}")
+
+ self.ema = EMAModule(
+ self.encoder if self.ema_transformer_only else self,
+ ema_decay=self.ema_decay,
+ ema_fp32=True,
+ skip_keys=skip_keys,
+ )
+
+ def set_num_updates(self, num_updates):
+ if self.ema is None and self.final_proj is not None:
+ logging.info("Making EMA Teacher")
+ self.make_ema_teacher()
+ elif self.training and self.ema is not None:
+ if self.ema_decay != self.ema_end_decay:
+ if num_updates >= self.ema_anneal_end_step:
+ decay = self.ema_end_decay
+ else:
+ decay = get_annealed_rate(
+ self.ema_decay,
+ self.ema_end_decay,
+ num_updates,
+ self.ema_anneal_end_step,
+ )
+ self.ema.set_decay(decay)
+ if self.ema.get_decay() < 1:
+ self.ema.step(self.encoder if self.ema_transformer_only else self)
+
+ self.num_updates = num_updates
+
+ def apply_mask(
+ self,
+ x,
+ padding_mask,
+ mask_indices=None,
+ mask_channel_indices=None,
+ ):
+ B, T, C = x.shape
+
+ if self.mask_channel_prob > 0 and self.mask_channel_before:
+ mask_channel_indices = compute_mask_indices(
+ (B, C),
+ None,
+ self.mask_channel_prob,
+ self.mask_channel_length,
+ self.mask_channel_selection,
+ self.mask_channel_other,
+ no_overlap=self.no_mask_channel_overlap,
+ min_space=self.mask_channel_min_space,
+ )
+ mask_channel_indices = (
+ torch.from_numpy(mask_channel_indices)
+ .to(x.device)
+ .unsqueeze(1)
+ .expand(-1, T, -1)
+ )
+ x[mask_channel_indices] = 0
+
+ if self.mask_prob > 0:
+ if mask_indices is None:
+ mask_indices = compute_mask_indices(
+ (B, T),
+ padding_mask,
+ self.mask_prob,
+ self.mask_length,
+ self.mask_selection,
+ self.mask_other,
+ min_masks=1,
+ no_overlap=self.no_mask_overlap,
+ min_space=self.mask_min_space,
+ require_same_masks=self.require_same_masks,
+ mask_dropout=self.mask_dropout,
+ )
+ mask_indices = torch.from_numpy(mask_indices).to(x.device)
+ x[mask_indices] = self.mask_emb
+ else:
+ mask_indices = None
+
+ if self.mask_channel_prob > 0 and not self.mask_channel_before:
+ if mask_channel_indices is None:
+ mask_channel_indices = compute_mask_indices(
+ (B, C),
+ None,
+ self.mask_channel_prob,
+ self.mask_channel_length,
+ self.mask_channel_selection,
+ self.mask_channel_other,
+ no_overlap=self.no_mask_channel_overlap,
+ min_space=self.mask_channel_min_space,
+ )
+ mask_channel_indices = (
+ torch.from_numpy(mask_channel_indices)
+ .to(x.device)
+ .unsqueeze(1)
+ .expand(-1, T, -1)
+ )
+ x[mask_channel_indices] = 0
+
+ return x, mask_indices
+
+ def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
+ """
+ Computes the output length of the convolutional layers
+ """
+
+ def _conv_out_length(input_length, kernel_size, stride):
+ return torch.floor((input_length - kernel_size).to(torch.float32) / stride + 1)
+
+ conv_cfg_list = eval(self.conv_feature_layers)
+
+ for i in range(len(conv_cfg_list)):
+ input_lengths = _conv_out_length(
+ input_lengths, conv_cfg_list[i][1], conv_cfg_list[i][2]
+ )
+
+ return input_lengths.to(torch.long)
+
+ def forward(
+ self,
+ xs_pad,
+ ilens=None,
+ mask=False,
+ features_only=True,
+ layer=None,
+ mask_indices=None,
+ mask_channel_indices=None,
+ padding_count=None,
+ ):
+ # create padding_mask by ilens
+ if ilens is not None:
+ padding_mask = make_pad_mask(lengths=ilens).to(xs_pad.device)
+ else:
+ padding_mask = None
+
+ features = xs_pad
+
+ if self.feature_grad_mult > 0:
+ features = self.feature_extractor(features)
+ if self.feature_grad_mult != 1.0:
+ features = GradMultiply.apply(features, self.feature_grad_mult)
+ else:
+ with torch.no_grad():
+ features = self.feature_extractor(features)
+
+ features = features.transpose(1, 2)
+
+ features = self.layer_norm(features)
+
+ orig_padding_mask = padding_mask
+
+ if padding_mask is not None:
+ input_lengths = (1 - padding_mask.long()).sum(-1)
+ # apply conv formula to get real output_lengths
+ output_lengths = self._get_feat_extract_output_lengths(input_lengths)
+
+ padding_mask = torch.zeros(
+ features.shape[:2], dtype=features.dtype, device=features.device
+ )
+ # these two operations makes sure that all values
+ # before the output lengths indices are attended to
+ padding_mask[
+ (
+ torch.arange(padding_mask.shape[0], device=padding_mask.device),
+ output_lengths - 1,
+ )
+ ] = 1
+ padding_mask = (1 - padding_mask.flip([-1]).cumsum(-1).flip([-1])).bool()
+ else:
+ padding_mask = None
+
+ if self.post_extract_proj is not None:
+ features = self.post_extract_proj(features)
+
+ pre_encoder_features = None
+ if self.ema_transformer_only:
+ pre_encoder_features = features.clone()
+
+ features = self.dropout_input(features)
+
+ if mask:
+ x, mask_indices = self.apply_mask(
+ features,
+ padding_mask,
+ mask_indices=mask_indices,
+ mask_channel_indices=mask_channel_indices,
+ )
+ else:
+ x = features
+ mask_indices = None
+
+ x, layer_results = self.encoder(
+ x,
+ padding_mask=padding_mask,
+ layer=layer,
+ )
+
+ if features_only:
+ encoder_out_lens = (1 - padding_mask.long()).sum(1)
+ return x, encoder_out_lens, None
+
+ result = {
+ "losses": {},
+ "padding_mask": padding_mask,
+ "x": x,
+ }
+
+ with torch.no_grad():
+ self.ema.model.eval()
+
+ if self.ema_transformer_only:
+ y, layer_results = self.ema.model.extract_features(
+ pre_encoder_features,
+ padding_mask=padding_mask,
+ min_layer=self.encoder_layers - self.average_top_k_layers,
+ )
+ y = {
+ "x": y,
+ "padding_mask": padding_mask,
+ "layer_results": layer_results,
+ }
+ else:
+ y = self.ema.model.extract_features(
+ source=xs_pad,
+ padding_mask=orig_padding_mask,
+ mask=False,
+ )
+
+ target_layer_results = [l[2] for l in y["layer_results"]]
+
+ permuted = False
+ if self.instance_norm_target_layer or self.batch_norm_target_layer:
+ target_layer_results = [
+ tl.permute(1, 2, 0) for tl in target_layer_results # TBC -> BCT
+ ]
+ permuted = True
+
+ if self.batch_norm_target_layer:
+ target_layer_results = [
+ F.batch_norm(
+ tl.float(), running_mean=None, running_var=None, training=True
+ )
+ for tl in target_layer_results
+ ]
+
+ if self.instance_norm_target_layer:
+ target_layer_results = [
+ F.instance_norm(tl.float()) for tl in target_layer_results
+ ]
+
+ if permuted:
+ target_layer_results = [
+ tl.transpose(1, 2) for tl in target_layer_results # BCT -> BTC
+ ]
+
+ if self.group_norm_target_layer:
+ target_layer_results = [
+ F.layer_norm(tl.float(), tl.shape[-2:])
+ for tl in target_layer_results
+ ]
+
+ if self.layer_norm_target_layer:
+ target_layer_results = [
+ F.layer_norm(tl.float(), tl.shape[-1:])
+ for tl in target_layer_results
+ ]
+
+ y = sum(target_layer_results) / len(target_layer_results)
+
+ if self.layer_norm_targets:
+ y = F.layer_norm(y.float(), y.shape[-1:])
+
+ if self.instance_norm_targets:
+ y = F.instance_norm(y.float().transpose(1, 2)).transpose(1, 2)
+
+ if not permuted:
+ y = y.transpose(0, 1)
+
+ y = y[mask_indices]
+
+ x = x[mask_indices]
+ x = self.final_proj(x)
+
+ sz = x.size(-1)
+
+ if self.loss_beta == 0:
+ loss = F.mse_loss(x.float(), y.float(), reduction="none").sum(dim=-1)
+ else:
+ loss = F.smooth_l1_loss(
+ x.float(), y.float(), reduction="none", beta=self.loss_beta
+ ).sum(dim=-1)
+
+ if self.loss_scale is not None:
+ scale = self.loss_scale
+ else:
+ scale = 1 / math.sqrt(sz)
+
+ result["losses"]["regression"] = loss.sum() * scale
+
+ if "sample_size" not in result:
+ result["sample_size"] = loss.numel()
+
+ with torch.no_grad():
+ result["target_var"] = self.compute_var(y)
+ result["pred_var"] = self.compute_var(x.float())
+
+ if self.num_updates > 5000 and result["target_var"] < self.min_target_var:
+ logging.error(
+ f"target var is {result['target_var'].item()} < {self.min_target_var}, exiting"
+ )
+ raise Exception(
+ f"target var is {result['target_var'].item()} < {self.min_target_var}, exiting"
+ )
+ if self.num_updates > 5000 and result["pred_var"] < self.min_pred_var:
+ logging.error(
+ f"pred var is {result['pred_var'].item()} < {self.min_pred_var}, exiting"
+ )
+ raise Exception(
+ f"pred var is {result['pred_var'].item()} < {self.min_pred_var}, exiting"
+ )
+
+ if self.ema is not None:
+ result["ema_decay"] = self.ema.get_decay() * 1000
+
+ return result
+
+ @staticmethod
+ def compute_var(y):
+ y = y.view(-1, y.size(-1))
+ if dist.is_initialized():
+ zc = torch.tensor(y.size(0)).cuda()
+ zs = y.sum(dim=0)
+ zss = (y ** 2).sum(dim=0)
+
+ dist.all_reduce(zc)
+ dist.all_reduce(zs)
+ dist.all_reduce(zss)
+
+ var = zss / (zc - 1) - (zs ** 2) / (zc * (zc - 1))
+ return torch.sqrt(var + 1e-6).mean()
+ else:
+ return torch.sqrt(y.var(dim=0) + 1e-6).mean()
+
+ def extract_features(
+ self, xs_pad, ilens, mask=False, layer=None
+ ):
+ res = self.forward(
+ xs_pad,
+ ilens,
+ mask=mask,
+ features_only=True,
+ layer=layer,
+ )
+ return res
+
+ def remove_pretraining_modules(self, last_layer=None):
+ self.final_proj = None
+ self.ema = None
+ if last_layer is not None:
+ self.encoder.layers = nn.ModuleList(
+ l for i, l in enumerate(self.encoder.layers) if i <= last_layer
+ )
+
+ def output_size(self) -> int:
+ return self.encoder_embed_dim
diff --git a/funasr/models/encoder/fsmn_encoder.py b/funasr/models/encoder/fsmn_encoder.py
new file mode 100755
index 0000000..643cefc
--- /dev/null
+++ b/funasr/models/encoder/fsmn_encoder.py
@@ -0,0 +1,316 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from typing import Tuple
+
+
+class LinearTransform(nn.Module):
+
+ def __init__(self, input_dim, output_dim, quantize=0):
+ super(LinearTransform, self).__init__()
+ self.input_dim = input_dim
+ self.output_dim = output_dim
+ self.linear = nn.Linear(input_dim, output_dim, bias=False)
+ self.quantize = quantize
+ self.quant = torch.quantization.QuantStub()
+ self.dequant = torch.quantization.DeQuantStub()
+
+ def forward(self, input):
+ if self.quantize:
+ output = self.quant(input)
+ else:
+ output = input
+ output = self.linear(output)
+ if self.quantize:
+ output = self.dequant(output)
+
+ return output
+
+
+class AffineTransform(nn.Module):
+
+ def __init__(self, input_dim, output_dim, quantize=0):
+ super(AffineTransform, self).__init__()
+ self.input_dim = input_dim
+ self.output_dim = output_dim
+ self.quantize = quantize
+ self.linear = nn.Linear(input_dim, output_dim)
+ self.quant = torch.quantization.QuantStub()
+ self.dequant = torch.quantization.DeQuantStub()
+
+ def forward(self, input):
+ if self.quantize:
+ output = self.quant(input)
+ else:
+ output = input
+ output = self.linear(output)
+ if self.quantize:
+ output = self.dequant(output)
+
+ return output
+
+
+class FSMNBlock(nn.Module):
+
+ def __init__(
+ self,
+ input_dim: int,
+ output_dim: int,
+ lorder=None,
+ rorder=None,
+ lstride=1,
+ rstride=1,
+ quantize=0
+ ):
+ super(FSMNBlock, self).__init__()
+
+ self.dim = input_dim
+
+ if lorder is None:
+ return
+
+ self.lorder = lorder
+ self.rorder = rorder
+ self.lstride = lstride
+ self.rstride = rstride
+
+ self.conv_left = nn.Conv2d(
+ self.dim, self.dim, [lorder, 1], dilation=[lstride, 1], groups=self.dim, bias=False)
+
+ if self.rorder > 0:
+ self.conv_right = nn.Conv2d(
+ self.dim, self.dim, [rorder, 1], dilation=[rstride, 1], groups=self.dim, bias=False)
+ else:
+ self.conv_right = None
+ self.quantize = quantize
+ self.quant = torch.quantization.QuantStub()
+ self.dequant = torch.quantization.DeQuantStub()
+
+ def forward(self, input):
+ x = torch.unsqueeze(input, 1)
+ x_per = x.permute(0, 3, 2, 1)
+
+ y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
+ if self.quantize:
+ y_left = self.quant(y_left)
+ y_left = self.conv_left(y_left)
+ if self.quantize:
+ y_left = self.dequant(y_left)
+ out = x_per + y_left
+
+ if self.conv_right is not None:
+ y_right = F.pad(x_per, [0, 0, 0, self.rorder * self.rstride])
+ y_right = y_right[:, :, self.rstride:, :]
+ if self.quantize:
+ y_right = self.quant(y_right)
+ y_right = self.conv_right(y_right)
+ if self.quantize:
+ y_right = self.dequant(y_right)
+ out += y_right
+
+ out_per = out.permute(0, 3, 2, 1)
+ output = out_per.squeeze(1)
+
+ return output
+
+
+class RectifiedLinear(nn.Module):
+
+ def __init__(self, input_dim, output_dim):
+ super(RectifiedLinear, self).__init__()
+ self.dim = input_dim
+ self.relu = nn.ReLU()
+ self.dropout = nn.Dropout(0.1)
+
+ def forward(self, input):
+ out = self.relu(input)
+ # out = self.dropout(out)
+ return out
+
+
+def _build_repeats(
+ fsmn_layers: int,
+ linear_dim: int,
+ proj_dim: int,
+ lorder: int,
+ rorder: int,
+ lstride=1,
+ rstride=1,
+):
+ repeats = [
+ nn.Sequential(
+ LinearTransform(linear_dim, proj_dim),
+ FSMNBlock(proj_dim, proj_dim, lorder, rorder, 1, 1),
+ AffineTransform(proj_dim, linear_dim),
+ RectifiedLinear(linear_dim, linear_dim))
+ for i in range(fsmn_layers)
+ ]
+
+ return nn.Sequential(*repeats)
+
+
+'''
+FSMN net for keyword spotting
+input_dim: input dimension
+linear_dim: fsmn input dimensionll
+proj_dim: fsmn projection dimension
+lorder: fsmn left order
+rorder: fsmn right order
+num_syn: output dimension
+fsmn_layers: no. of sequential fsmn layers
+'''
+
+
+class FSMN(nn.Module):
+ def __init__(
+ self,
+ input_dim: int,
+ input_affine_dim: int,
+ fsmn_layers: int,
+ linear_dim: int,
+ proj_dim: int,
+ lorder: int,
+ rorder: int,
+ lstride: int,
+ rstride: int,
+ output_affine_dim: int,
+ output_dim: int,
+ ):
+ super(FSMN, self).__init__()
+
+ self.input_dim = input_dim
+ self.input_affine_dim = input_affine_dim
+ self.fsmn_layers = fsmn_layers
+ self.linear_dim = linear_dim
+ self.proj_dim = proj_dim
+ self.lorder = lorder
+ self.rorder = rorder
+ self.lstride = lstride
+ self.rstride = rstride
+ self.output_affine_dim = output_affine_dim
+ self.output_dim = output_dim
+
+ self.in_linear1 = AffineTransform(input_dim, input_affine_dim)
+ self.in_linear2 = AffineTransform(input_affine_dim, linear_dim)
+ self.relu = RectifiedLinear(linear_dim, linear_dim)
+
+ self.fsmn = _build_repeats(fsmn_layers,
+ linear_dim,
+ proj_dim,
+ lorder, rorder,
+ lstride, rstride)
+
+ self.out_linear1 = AffineTransform(linear_dim, output_affine_dim)
+ self.out_linear2 = AffineTransform(output_affine_dim, output_dim)
+ self.softmax = nn.Softmax(dim=-1)
+
+ def fuse_modules(self):
+ pass
+
+ def forward(
+ self,
+ input: torch.Tensor,
+ in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float)
+ ) -> torch.Tensor:
+ """
+ Args:
+ input (torch.Tensor): Input tensor (B, T, D)
+ in_cache(torhc.Tensor): (B, D, C), C is the accumulated cache size
+ """
+
+ x1 = self.in_linear1(input)
+ x2 = self.in_linear2(x1)
+ x3 = self.relu(x2)
+ x4 = self.fsmn(x3)
+ x5 = self.out_linear1(x4)
+ x6 = self.out_linear2(x5)
+ x7 = self.softmax(x6)
+
+ return x7
+ # return x6, in_cache
+
+
+'''
+one deep fsmn layer
+dimproj: projection dimension, input and output dimension of memory blocks
+dimlinear: dimension of mapping layer
+lorder: left order
+rorder: right order
+lstride: left stride
+rstride: right stride
+'''
+
+
+class DFSMN(nn.Module):
+
+ def __init__(self, dimproj=64, dimlinear=128, lorder=20, rorder=1, lstride=1, rstride=1):
+ super(DFSMN, self).__init__()
+
+ self.lorder = lorder
+ self.rorder = rorder
+ self.lstride = lstride
+ self.rstride = rstride
+
+ self.expand = AffineTransform(dimproj, dimlinear)
+ self.shrink = LinearTransform(dimlinear, dimproj)
+
+ self.conv_left = nn.Conv2d(
+ dimproj, dimproj, [lorder, 1], dilation=[lstride, 1], groups=dimproj, bias=False)
+
+ if rorder > 0:
+ self.conv_right = nn.Conv2d(
+ dimproj, dimproj, [rorder, 1], dilation=[rstride, 1], groups=dimproj, bias=False)
+ else:
+ self.conv_right = None
+
+ def forward(self, input):
+ f1 = F.relu(self.expand(input))
+ p1 = self.shrink(f1)
+
+ x = torch.unsqueeze(p1, 1)
+ x_per = x.permute(0, 3, 2, 1)
+
+ y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0])
+
+ if self.conv_right is not None:
+ y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride])
+ y_right = y_right[:, :, self.rstride:, :]
+ out = x_per + self.conv_left(y_left) + self.conv_right(y_right)
+ else:
+ out = x_per + self.conv_left(y_left)
+
+ out1 = out.permute(0, 3, 2, 1)
+ output = input + out1.squeeze(1)
+
+ return output
+
+
+'''
+build stacked dfsmn layers
+'''
+
+
+def buildDFSMNRepeats(linear_dim=128, proj_dim=64, lorder=20, rorder=1, fsmn_layers=6):
+ repeats = [
+ nn.Sequential(
+ DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1))
+ for i in range(fsmn_layers)
+ ]
+
+ return nn.Sequential(*repeats)
+
+
+if __name__ == '__main__':
+ fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599)
+ print(fsmn)
+
+ num_params = sum(p.numel() for p in fsmn.parameters())
+ print('the number of model params: {}'.format(num_params))
+ x = torch.zeros(128, 200, 400) # batch-size * time * dim
+ y, _ = fsmn(x) # batch-size * time * dim
+ print('input shape: {}'.format(x.shape))
+ print('output shape: {}'.format(y.shape))
+
+ print(fsmn.to_kaldi_net())
diff --git a/funasr/models/encoder/resnet34_encoder.py b/funasr/models/encoder/resnet34_encoder.py
new file mode 100644
index 0000000..66e446c
--- /dev/null
+++ b/funasr/models/encoder/resnet34_encoder.py
@@ -0,0 +1,144 @@
+import torch
+from torch.nn import functional as F
+from funasr.models.encoder.abs_encoder import AbsEncoder
+from typing import Tuple
+
+
+class BasicLayer(torch.nn.Module):
+
+ def __init__(self, in_filters: int, filters: int, stride: int, bn_momentum: float = 0.5):
+
+ super().__init__()
+ self.stride = stride
+ self.in_filters = in_filters
+ self.filters = filters
+
+ self.bn1 = torch.nn.BatchNorm2d(in_filters, eps=1e-3, momentum=bn_momentum, affine=True)
+ self.relu1 = torch.nn.ReLU()
+ self.conv1 = torch.nn.Conv2d(in_filters, filters, 3, stride, bias=False)
+
+ self.bn2 = torch.nn.BatchNorm2d(filters, eps=1e-3, momentum=bn_momentum, affine=True)
+ self.relu2 = torch.nn.ReLU()
+ self.conv2 = torch.nn.Conv2d(filters, filters, 3, 1, bias=False)
+
+ if in_filters != filters or stride > 1:
+ self.conv_sc = torch.nn.Conv2d(in_filters, filters, 1, stride, bias=False)
+ self.bn_sc = torch.nn.BatchNorm2d(filters, eps=1e-3, momentum=bn_momentum, affine=True)
+
+ def proper_padding(self, x, stride):
+ # align padding mode to tf.layers.conv2d with padding_mod="same"
+ if stride == 1:
+ return F.pad(x, (1, 1, 1, 1), "constant", 0)
+ elif stride == 2:
+ h, w = x.size(2), x.size(3)
+ # (left, right, top, bottom)
+ return F.pad(x, (w % 2, 1, h % 2, 1), "constant", 0)
+
+ def forward(self, xs_pad, ilens):
+ identity = xs_pad
+ if self.in_filters != self.filters or self.stride > 1:
+ identity = self.conv_sc(identity)
+ identity = self.bn_sc(identity)
+
+ xs_pad = self.relu1(self.bn1(xs_pad))
+ xs_pad = self.proper_padding(xs_pad, self.stride)
+ xs_pad = self.conv1(xs_pad)
+
+ xs_pad = self.relu2(self.bn2(xs_pad))
+ xs_pad = self.proper_padding(xs_pad, 1)
+ xs_pad = self.conv2(xs_pad)
+
+ if self.stride == 2:
+ ilens = (ilens + 1) // self.stride
+
+ return xs_pad + identity, ilens
+
+
+class BasicBlock(torch.nn.Module):
+ def __init__(self, in_filters, filters, num_layer, stride, bn_momentum=0.5):
+ super().__init__()
+ self.num_layer = num_layer
+
+ for i in range(num_layer):
+ layer = BasicLayer(in_filters if i == 0 else filters, filters,
+ stride if i == 0 else 1, bn_momentum)
+ self.add_module("layer_{}".format(i), layer)
+
+ def forward(self, xs_pad, ilens):
+
+ for i in range(self.num_layer):
+ xs_pad, ilens = self._modules["layer_{}".format(i)](xs_pad, ilens)
+
+ return xs_pad, ilens
+
+
+class ResNet34(AbsEncoder):
+ def __init__(
+ self,
+ input_size,
+ use_head_conv=True,
+ batchnorm_momentum=0.5,
+ use_head_maxpool=False,
+ num_nodes_pooling_layer=256,
+ layers_in_block=(3, 4, 6, 3),
+ filters_in_block=(32, 64, 128, 256),
+ ):
+ super(ResNet34, self).__init__()
+
+ self.use_head_conv = use_head_conv
+ self.use_head_maxpool = use_head_maxpool
+ self.num_nodes_pooling_layer = num_nodes_pooling_layer
+ self.layers_in_block = layers_in_block
+ self.filters_in_block = filters_in_block
+ self.input_size = input_size
+
+ pre_filters = filters_in_block[0]
+ if use_head_conv:
+ self.pre_conv = torch.nn.Conv2d(1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros")
+ self.pre_conv_bn = torch.nn.BatchNorm2d(pre_filters, eps=1e-3, momentum=batchnorm_momentum)
+
+ if use_head_maxpool:
+ self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1)
+
+ for i in range(len(layers_in_block)):
+ if i == 0:
+ in_filters = pre_filters if self.use_head_conv else 1
+ else:
+ in_filters = filters_in_block[i-1]
+
+ block = BasicBlock(in_filters,
+ filters=filters_in_block[i],
+ num_layer=layers_in_block[i],
+ stride=1 if i == 0 else 2,
+ bn_momentum=batchnorm_momentum)
+ self.add_module("block_{}".format(i), block)
+
+ self.resnet0_dense = torch.nn.Conv2d(filters_in_block[-1], num_nodes_pooling_layer, 1)
+ self.resnet0_bn = torch.nn.BatchNorm2d(num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum)
+
+ def output_size(self) -> int:
+ return self.num_nodes_pooling_layer
+
+ def forward(self, xs_pad: torch.Tensor, ilens: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ features = xs_pad
+ assert features.size(-1) == self.input_size, \
+ "Dimension of features {} doesn't match the input_size {}.".format(features.size(-1), self.input_size)
+ features = torch.unsqueeze(features, dim=1)
+ if self.use_head_conv:
+ features = self.pre_conv(features)
+ features = self.pre_conv_bn(features)
+ features = F.relu(features)
+
+ if self.use_head_maxpool:
+ features = self.head_maxpool(features)
+
+ resnet_outs, resnet_out_lens = features, ilens
+ for i in range(len(self.layers_in_block)):
+ block = self._modules["block_{}".format(i)]
+ resnet_outs, resnet_out_lens = block(resnet_outs, resnet_out_lens)
+
+ features = self.resnet0_dense(resnet_outs)
+ features = F.relu(features)
+ features = self.resnet0_bn(features)
+
+ return features, ilens // 8
diff --git a/funasr/models/encoder/sanm_encoder.py b/funasr/models/encoder/sanm_encoder.py
index 3d8079d..4c4bd7c 100644
--- a/funasr/models/encoder/sanm_encoder.py
+++ b/funasr/models/encoder/sanm_encoder.py
@@ -3,12 +3,12 @@
from typing import Sequence
from typing import Tuple
from typing import Union
-
+import logging
import torch
import torch.nn as nn
from funasr.modules.streaming_utils.chunk_utilis import overlap_chunk
from typeguard import check_argument_types
-
+import numpy as np
from funasr.modules.nets_utils import make_pad_mask
from funasr.modules.attention import MultiHeadedAttention, MultiHeadedAttentionSANM
from funasr.modules.embedding import SinusoidalPositionEncoder
@@ -27,6 +27,7 @@
from funasr.modules.subsampling import check_short_utt
from funasr.models.ctc import CTC
from funasr.models.encoder.abs_encoder import AbsEncoder
+
class EncoderLayerSANM(nn.Module):
def __init__(
@@ -144,6 +145,8 @@
kernel_size : int = 11,
sanm_shfit : int = 0,
selfattention_layer_type: str = "sanm",
+ tf2torch_tensor_name_prefix_torch: str = "encoder",
+ tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",
):
assert check_argument_types()
super().__init__()
@@ -168,7 +171,7 @@
elif input_layer == "embed":
self.embed = torch.nn.Sequential(
torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx),
- pos_enc_class(output_size, positional_dropout_rate),
+ SinusoidalPositionEncoder(),
)
elif input_layer is None:
if input_size == output_size:
@@ -267,6 +270,8 @@
self.interctc_use_conditioning = interctc_use_conditioning
self.conditioning_layer = None
self.dropout = nn.Dropout(dropout_rate)
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
return self._output_size
@@ -342,6 +347,163 @@
return (xs_pad, intermediate_outs), olens, None
return xs_pad, olens, None
+ def gen_tf2torch_map_dict(self):
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ map_dict_local = {
+ ## encoder
+ # cicd
+ "{}.encoders.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.self_attn.linear_q_k_v.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (768,256),(1,256,768)
+ "{}.encoders.layeridx.self_attn.linear_q_k_v.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (768,),(768,)
+ "{}.encoders.layeridx.self_attn.fsmn_block.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/depth_conv_w".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 2, 0),
+ }, # (256,1,31),(1,31,256,1)
+ "{}.encoders.layeridx.self_attn.linear_out.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.encoders.layeridx.self_attn.linear_out.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # ffn
+ "{}.encoders.layeridx.norm2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.norm2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.encoders.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.encoders.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+ "{}.encoders.layeridx.feed_forward.w_2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # out norm
+ "{}.after_norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.after_norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+
+ }
+
+ return map_dict_local
+
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+
+ map_dict = self.gen_tf2torch_map_dict()
+
+ var_dict_torch_update = dict()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ if names[1] == "encoders0":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ name_q = name_q.replace("encoders0", "encoders")
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+ elif names[1] == "encoders":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ layeridx_bias = 1
+ layeridx += layeridx_bias
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "after_norm":
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ return var_dict_torch_update
+
class SANMEncoderChunkOpt(AbsEncoder):
"""
@@ -378,6 +540,8 @@
pad_left: Union[int, Sequence[int]] = (0,),
encoder_att_look_back_factor: Union[int, Sequence[int]] = (1,),
decoder_att_look_back_factor: Union[int, Sequence[int]] = (1,),
+ tf2torch_tensor_name_prefix_torch: str = "encoder",
+ tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",
):
assert check_argument_types()
super().__init__()
@@ -508,6 +672,8 @@
encoder_att_look_back_factor=encoder_att_look_back_factor,
decoder_att_look_back_factor=decoder_att_look_back_factor,
)
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
return self._output_size
@@ -593,3 +759,160 @@
if len(intermediate_outs) > 0:
return (xs_pad, intermediate_outs), olens, None
return xs_pad, olens, None
+
+ def gen_tf2torch_map_dict(self):
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ map_dict_local = {
+ ## encoder
+ # cicd
+ "{}.encoders.layeridx.norm1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.norm1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.self_attn.linear_q_k_v.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (768,256),(1,256,768)
+ "{}.encoders.layeridx.self_attn.linear_q_k_v.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (768,),(768,)
+ "{}.encoders.layeridx.self_attn.fsmn_block.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/depth_conv_w".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 2, 0),
+ }, # (256,1,31),(1,31,256,1)
+ "{}.encoders.layeridx.self_attn.linear_out.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,256),(1,256,256)
+ "{}.encoders.layeridx.self_attn.linear_out.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/multi_head/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # ffn
+ "{}.encoders.layeridx.norm2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.norm2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.encoders.layeridx.feed_forward.w_1.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1024,256),(1,256,1024)
+ "{}.encoders.layeridx.feed_forward.w_1.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1024,),(1024,)
+ "{}.encoders.layeridx.feed_forward.w_2.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (256,1024),(1,1024,256)
+ "{}.encoders.layeridx.feed_forward.w_2.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/layer_layeridx/ffn/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ # out norm
+ "{}.after_norm.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/gamma".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.after_norm.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/LayerNorm/beta".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+
+ }
+
+ return map_dict_local
+
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+
+ map_dict = self.gen_tf2torch_map_dict()
+
+ var_dict_torch_update = dict()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ if names[1] == "encoders0":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+
+ name_q = name_q.replace("encoders0", "encoders")
+ layeridx_bias = 0
+ layeridx += layeridx_bias
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+ elif names[1] == "encoders":
+ layeridx = int(names[2])
+ name_q = name.replace(".{}.".format(layeridx), ".layeridx.")
+ layeridx_bias = 1
+ layeridx += layeridx_bias
+ if name_q in map_dict.keys():
+ name_v = map_dict[name_q]["name"]
+ name_tf = name_v.replace("layeridx", "{}".format(layeridx))
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name_q]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name_q]["squeeze"])
+ if map_dict[name_q]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name_q]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_v,
+ var_dict_tf[name_tf].shape))
+
+ elif names[1] == "after_norm":
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ return var_dict_torch_update
diff --git a/funasr/models/frontend/wav_frontend.py b/funasr/models/frontend/wav_frontend.py
index c0b28ff..57c5976 100644
--- a/funasr/models/frontend/wav_frontend.py
+++ b/funasr/models/frontend/wav_frontend.py
@@ -1,22 +1,43 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
# Part of the implementation is borrowed from espnet/espnet.
-import copy
-from typing import Optional, Tuple, Union
+from typing import Tuple
-import humanfriendly
import numpy as np
import torch
import torchaudio.compliance.kaldi as kaldi
from funasr.models.frontend.abs_frontend import AbsFrontend
-from funasr.layers.log_mel import LogMel
-from funasr.layers.stft import Stft
-from funasr.utils.get_default_kwargs import get_default_kwargs
-from funasr.modules.frontends.frontend import Frontend
from typeguard import check_argument_types
+from torch.nn.utils.rnn import pad_sequence
-def apply_cmvn(inputs, mvn): # noqa
+def load_cmvn(cmvn_file):
+ with open(cmvn_file, 'r', encoding='utf-8') as f:
+ lines = f.readlines()
+ means_list = []
+ vars_list = []
+ for i in range(len(lines)):
+ line_item = lines[i].split()
+ if line_item[0] == '<AddShift>':
+ line_item = lines[i + 1].split()
+ if line_item[0] == '<LearnRateCoef>':
+ add_shift_line = line_item[3:(len(line_item) - 1)]
+ means_list = list(add_shift_line)
+ continue
+ elif line_item[0] == '<Rescale>':
+ line_item = lines[i + 1].split()
+ if line_item[0] == '<LearnRateCoef>':
+ rescale_line = line_item[3:(len(line_item) - 1)]
+ vars_list = list(rescale_line)
+ continue
+ means = np.array(means_list).astype(np.float)
+ vars = np.array(vars_list).astype(np.float)
+ cmvn = np.array([means, vars])
+ cmvn = torch.as_tensor(cmvn)
+ return cmvn
+
+
+def apply_cmvn(inputs, cmvn_file): # noqa
"""
Apply CMVN with mvn data
"""
@@ -25,9 +46,10 @@
dtype = inputs.dtype
frame, dim = inputs.shape
- meams = np.tile(mvn[0:1, :dim], (frame, 1))
- vars = np.tile(mvn[1:2, :dim], (frame, 1))
- inputs += torch.from_numpy(meams).type(dtype).to(device)
+ cmvn = load_cmvn(cmvn_file)
+ means = np.tile(cmvn[0:1, :dim], (frame, 1))
+ vars = np.tile(cmvn[1:2, :dim], (frame, 1))
+ inputs += torch.from_numpy(means).type(dtype).to(device)
inputs *= torch.from_numpy(vars).type(dtype).to(device)
return inputs.type(torch.float32)
@@ -58,98 +80,124 @@
"""
def __init__(
self,
- fs: Union[int, str] = 16000,
- n_fft: int = 512,
- win_length: int = 400,
- hop_length: int = 160,
- window: Optional[str] = 'hamming',
- center: bool = True,
- normalized: bool = False,
- onesided: bool = True,
+ cmvn_file: str = None,
+ fs: int = 16000,
+ window: str = 'hamming',
n_mels: int = 80,
- fmin: int = None,
- fmax: int = None,
+ frame_length: int = 25,
+ frame_shift: int = 10,
+ filter_length_min: int = -1,
+ filter_length_max: int = -1,
lfr_m: int = 1,
lfr_n: int = 1,
- htk: bool = False,
- mvn_data=None,
- frontend_conf: Optional[dict] = get_default_kwargs(Frontend),
- apply_stft: bool = True,
+ dither: float = 1.0
):
assert check_argument_types()
super().__init__()
- if isinstance(fs, str):
- fs = humanfriendly.parse_size(fs)
-
- # Deepcopy (In general, dict shouldn't be used as default arg)
- frontend_conf = copy.deepcopy(frontend_conf)
- self.hop_length = hop_length
- self.win_length = win_length
- self.window = window
self.fs = fs
- self.mvn_data = mvn_data
+ self.window = window
+ self.n_mels = n_mels
+ self.frame_length = frame_length
+ self.frame_shift = frame_shift
+ self.filter_length_min = filter_length_min
+ self.filter_length_max = filter_length_max
self.lfr_m = lfr_m
self.lfr_n = lfr_n
-
- if apply_stft:
- self.stft = Stft(
- n_fft=n_fft,
- win_length=win_length,
- hop_length=hop_length,
- center=center,
- window=window,
- normalized=normalized,
- onesided=onesided,
- )
- else:
- self.stft = None
- self.apply_stft = apply_stft
-
- if frontend_conf is not None:
- self.frontend = Frontend(idim=n_fft // 2 + 1, **frontend_conf)
- else:
- self.frontend = None
-
- self.logmel = LogMel(
- fs=fs,
- n_fft=n_fft,
- n_mels=n_mels,
- fmin=fmin,
- fmax=fmax,
- htk=htk,
- )
- self.n_mels = n_mels
- self.frontend_type = 'default'
+ self.cmvn_file = cmvn_file
+ self.dither = dither
def output_size(self) -> int:
- return self.n_mels
+ return self.n_mels * self.lfr_m
def forward(
- self, input: torch.Tensor,
+ self,
+ input: torch.Tensor,
input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ batch_size = input.size(0)
+ feats = []
+ feats_lens = []
+ for i in range(batch_size):
+ waveform_length = input_lengths[i]
+ waveform = input[i][:waveform_length]
+ waveform = waveform * (1 << 15)
+ waveform = waveform.unsqueeze(0)
+ mat = kaldi.fbank(waveform,
+ num_mel_bins=self.n_mels,
+ frame_length=self.frame_length,
+ frame_shift=self.frame_shift,
+ dither=self.dither,
+ energy_floor=0.0,
+ window_type=self.window,
+ sample_frequency=self.fs)
+
+ if self.lfr_m != 1 or self.lfr_n != 1:
+ mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
+ if self.cmvn_file is not None:
+ mat = apply_cmvn(mat, self.cmvn_file)
+ feat_length = mat.size(0)
+ feats.append(mat)
+ feats_lens.append(feat_length)
- sample_frequency = self.fs
- num_mel_bins = self.n_mels
- frame_length = self.win_length * 1000 / sample_frequency
- frame_shift = self.hop_length * 1000 / sample_frequency
+ feats_lens = torch.as_tensor(feats_lens)
+ feats_pad = pad_sequence(feats,
+ batch_first=True,
+ padding_value=0.0)
+ return feats_pad, feats_lens
- waveform = input * (1 << 15)
+ def forward_fbank(
+ self,
+ input: torch.Tensor,
+ input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ batch_size = input.size(0)
+ feats = []
+ feats_lens = []
+ for i in range(batch_size):
+ waveform_length = input_lengths[i]
+ waveform = input[i][:waveform_length]
+ waveform = waveform * (1 << 15)
+ waveform = waveform.unsqueeze(0)
+ mat = kaldi.fbank(waveform,
+ num_mel_bins=self.n_mels,
+ frame_length=self.frame_length,
+ frame_shift=self.frame_shift,
+ dither=self.dither,
+ energy_floor=0.0,
+ window_type=self.window,
+ sample_frequency=self.fs)
- mat = kaldi.fbank(waveform,
- num_mel_bins=num_mel_bins,
- frame_length=frame_length,
- frame_shift=frame_shift,
- dither=1.0,
- energy_floor=0.0,
- window_type=self.window,
- sample_frequency=sample_frequency)
- if self.lfr_m != 1 or self.lfr_n != 1:
- mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
- if self.mvn_data is not None:
- mat = apply_cmvn(mat, self.mvn_data)
+ # if self.lfr_m != 1 or self.lfr_n != 1:
+ # mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
+ # if self.cmvn_file is not None:
+ # mat = apply_cmvn(mat, self.cmvn_file)
+ feat_length = mat.size(0)
+ feats.append(mat)
+ feats_lens.append(feat_length)
- input_feats = mat[None, :]
- feats_lens = torch.randn(1)
- feats_lens.fill_(input_feats.shape[1])
+ feats_lens = torch.as_tensor(feats_lens)
+ feats_pad = pad_sequence(feats,
+ batch_first=True,
+ padding_value=0.0)
+ return feats_pad, feats_lens
- return input_feats, feats_lens
+ def forward_lfr_cmvn(
+ self,
+ input: torch.Tensor,
+ input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ batch_size = input.size(0)
+ feats = []
+ feats_lens = []
+ for i in range(batch_size):
+ mat = input[i, :input_lengths[i], :]
+ if self.lfr_m != 1 or self.lfr_n != 1:
+ mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
+ if self.cmvn_file is not None:
+ mat = apply_cmvn(mat, self.cmvn_file)
+ feat_length = mat.size(0)
+ feats.append(mat)
+ feats_lens.append(feat_length)
+
+ feats_lens = torch.as_tensor(feats_lens)
+ feats_pad = pad_sequence(feats,
+ batch_first=True,
+ padding_value=0.0)
+ return feats_pad, feats_lens
diff --git a/egs_modelscope/common/utils/__init__.py b/funasr/models/pooling/__init__.py
similarity index 100%
copy from egs_modelscope/common/utils/__init__.py
copy to funasr/models/pooling/__init__.py
diff --git a/funasr/models/pooling/statistic_pooling.py b/funasr/models/pooling/statistic_pooling.py
new file mode 100644
index 0000000..eeaed7d
--- /dev/null
+++ b/funasr/models/pooling/statistic_pooling.py
@@ -0,0 +1,36 @@
+import torch
+from typing import Tuple
+from typing import Union
+from funasr.modules.nets_utils import make_non_pad_mask
+
+
+class StatisticPooling(torch.nn.Module):
+ def __init__(self, pooling_dim: Union[int, Tuple] = 2, eps=1e-12):
+ super(StatisticPooling, self).__init__()
+ if isinstance(pooling_dim, int):
+ pooling_dim = (pooling_dim, )
+ self.pooling_dim = pooling_dim
+ self.eps = eps
+
+ def forward(self, xs_pad, ilens=None):
+ # xs_pad in (Batch, Channel, Time, Frequency)
+
+ if ilens is None:
+ masks = torch.ones_like(xs_pad).to(xs_pad)
+ else:
+ masks = make_non_pad_mask(ilens, xs_pad, length_dim=2).to(xs_pad)
+ mean = (torch.sum(xs_pad, dim=self.pooling_dim, keepdim=True) /
+ torch.sum(masks, dim=self.pooling_dim, keepdim=True))
+ squared_difference = torch.pow(xs_pad - mean, 2.0)
+ variance = (torch.sum(squared_difference, dim=self.pooling_dim, keepdim=True) /
+ torch.sum(masks, dim=self.pooling_dim, keepdim=True))
+ for i in reversed(self.pooling_dim):
+ mean, variance = torch.squeeze(mean, dim=i), torch.squeeze(variance, dim=i)
+
+ mask = torch.less_equal(variance, self.eps).float()
+ variance = (1.0 - mask) * variance + mask * self.eps
+ stddev = torch.sqrt(variance)
+
+ stat_pooling = torch.cat([mean, stddev], dim=1)
+
+ return stat_pooling
diff --git a/funasr/models/predictor/cif.py b/funasr/models/predictor/cif.py
index 1d196a7..60cf902 100644
--- a/funasr/models/predictor/cif.py
+++ b/funasr/models/predictor/cif.py
@@ -1,6 +1,7 @@
import torch
from torch import nn
-
+import logging
+import numpy as np
from funasr.modules.nets_utils import make_pad_mask
from funasr.modules.streaming_utils.utils import sequence_mask
@@ -130,8 +131,19 @@
class CifPredictorV2(nn.Module):
- def __init__(self, idim, l_order, r_order, threshold=1.0, dropout=0.1, smooth_factor=1.0, noise_threshold=0,
- tail_threshold=0.0):
+ def __init__(self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ tail_threshold=0.0,
+ tf2torch_tensor_name_prefix_torch="predictor",
+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",
+ tail_mask=True,
+ ):
super(CifPredictorV2, self).__init__()
self.pad = nn.ConstantPad1d((l_order, r_order), 0)
@@ -142,6 +154,9 @@
self.smooth_factor = smooth_factor
self.noise_threshold = noise_threshold
self.tail_threshold = tail_threshold
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
+ self.tail_mask = tail_mask
def forward(self, hidden, target_label=None, mask=None, ignore_id=-1, mask_chunk_predictor=None,
target_label_length=None):
@@ -171,7 +186,10 @@
if target_length is not None:
alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
elif self.tail_threshold > 0.0:
- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)
+ if self.tail_mask:
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)
+ else:
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=None)
acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
if target_length is None and self.tail_threshold > 0.0:
@@ -179,6 +197,394 @@
acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
return acoustic_embeds, token_num, alphas, cif_peak
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+ if mask is not None:
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, tail_threshold], dim=1)
+ else:
+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
+ tail_threshold = torch.reshape(tail_threshold, (1, 1))
+ if b > 1:
+ alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1)
+ else:
+ alphas = torch.cat([alphas, tail_threshold], dim=1)
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+ def gen_frame_alignments(self,
+ alphas: torch.Tensor = None,
+ encoder_sequence_length: torch.Tensor = None):
+ batch_size, maximum_length = alphas.size()
+ int_type = torch.int32
+
+ is_training = self.training
+ if is_training:
+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
+ else:
+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
+
+ max_token_num = torch.max(token_num).item()
+
+ alphas_cumsum = torch.cumsum(alphas, dim=1)
+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
+
+ index = torch.ones([batch_size, max_token_num], dtype=int_type)
+ index = torch.cumsum(index, dim=1)
+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
+
+ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
+ index_div_bool_zeros = index_div.eq(0)
+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
+ index_div_bool_zeros_count = torch.clamp(index_div_bool_zeros_count, 0, encoder_sequence_length.max())
+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
+ index_div_bool_zeros_count *= token_num_mask
+
+ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(1, 1, maximum_length)
+ ones = torch.ones_like(index_div_bool_zeros_count_tile)
+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
+ ones = torch.cumsum(ones, dim=2)
+ cond = index_div_bool_zeros_count_tile == ones
+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
+
+ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
+ predictor_mask = (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max())).type(
+ int_type).to(encoder_sequence_length.device)
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
+
+ predictor_alignments = index_div_bool_zeros_count_tile_out
+ predictor_alignments_length = predictor_alignments.sum(-1).type(encoder_sequence_length.dtype)
+ return predictor_alignments.detach(), predictor_alignments_length.detach()
+
+ def gen_tf2torch_map_dict(self):
+
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ map_dict_local = {
+ ## predictor
+ "{}.cif_conv1d.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": (2, 1, 0),
+ }, # (256,256,3),(3,256,256)
+ "{}.cif_conv1d.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ "{}.cif_output.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d_1/kernel".format(tensor_name_prefix_tf),
+ "squeeze": 0,
+ "transpose": (1, 0),
+ }, # (1,256),(1,256,1)
+ "{}.cif_output.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d_1/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (1,),(1,)
+ }
+ return map_dict_local
+
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+ map_dict = self.gen_tf2torch_map_dict()
+ var_dict_torch_update = dict()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
+ if map_dict[name]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+ assert var_dict_torch[name].size() == data_tf.size(), "{}, {}, {} != {}".format(name, name_tf,
+ var_dict_torch[
+ name].size(),
+ data_tf.size())
+ var_dict_torch_update[name] = data_tf
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+
+ return var_dict_torch_update
+
+
+class mae_loss(nn.Module):
+
+ def __init__(self, normalize_length=False):
+ super(mae_loss, self).__init__()
+ self.normalize_length = normalize_length
+ self.criterion = torch.nn.L1Loss(reduction='sum')
+
+ def forward(self, token_length, pre_token_length):
+ loss_token_normalizer = token_length.size(0)
+ if self.normalize_length:
+ loss_token_normalizer = token_length.sum().type(torch.float32)
+ loss = self.criterion(token_length, pre_token_length)
+ loss = loss / loss_token_normalizer
+ return loss
+
+
+def cif(hidden, alphas, threshold):
+ batch_size, len_time, hidden_size = hidden.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=hidden.device)
+ frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
+ # intermediate vars along time
+ list_fires = []
+ list_frames = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+ distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(fire_place,
+ integrate - torch.ones([batch_size], device=hidden.device),
+ integrate)
+ cur = torch.where(fire_place,
+ distribution_completion,
+ alpha)
+ remainds = alpha - cur
+
+ frame += cur[:, None] * hidden[:, t, :]
+ list_frames.append(frame)
+ frame = torch.where(fire_place[:, None].repeat(1, hidden_size),
+ remainds[:, None] * hidden[:, t, :],
+ frame)
+
+ fires = torch.stack(list_fires, 1)
+ frames = torch.stack(list_frames, 1)
+ list_ls = []
+ len_labels = torch.round(alphas.sum(-1)).int()
+ max_label_len = len_labels.max()
+ for b in range(batch_size):
+ fire = fires[b, :]
+ l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())
+ pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
+ list_ls.append(torch.cat([l, pad_l], 0))
+ return torch.stack(list_ls, 0), fires
+
+
+def cif_wo_hidden(alphas, threshold):
+ batch_size, len_time = alphas.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=alphas.device)
+ # intermediate vars along time
+ list_fires = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(fire_place,
+ integrate - torch.ones([batch_size], device=alphas.device),
+ integrate)
+
+ fires = torch.stack(list_fires, 1)
+ return fires
+
+
+class CifPredictorV3(nn.Module):
+ def __init__(self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ tail_threshold=0.0,
+ tf2torch_tensor_name_prefix_torch="predictor",
+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",
+ smooth_factor2=1.0,
+ noise_threshold2=0,
+ upsample_times=5,
+ upsample_type="cnn",
+ use_cif1_cnn=True,
+ tail_mask=True,
+ ):
+ super(CifPredictorV3, self).__init__()
+
+ self.pad = nn.ConstantPad1d((l_order, r_order), 0)
+ self.cif_conv1d = nn.Conv1d(idim, idim, l_order + r_order + 1)
+ self.cif_output = nn.Linear(idim, 1)
+ self.dropout = torch.nn.Dropout(p=dropout)
+ self.threshold = threshold
+ self.smooth_factor = smooth_factor
+ self.noise_threshold = noise_threshold
+ self.tail_threshold = tail_threshold
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
+
+ self.upsample_times = upsample_times
+ self.upsample_type = upsample_type
+ self.use_cif1_cnn = use_cif1_cnn
+ if self.upsample_type == 'cnn':
+ self.upsample_cnn = nn.ConvTranspose1d(idim, idim, self.upsample_times, self.upsample_times)
+ self.cif_output2 = nn.Linear(idim, 1)
+ elif self.upsample_type == 'cnn_blstm':
+ self.upsample_cnn = nn.ConvTranspose1d(idim, idim, self.upsample_times, self.upsample_times)
+ self.blstm = nn.LSTM(idim, idim, 1, bias=True, batch_first=True, dropout=0.0, bidirectional=True)
+ self.cif_output2 = nn.Linear(idim*2, 1)
+ elif self.upsample_type == 'cnn_attn':
+ self.upsample_cnn = nn.ConvTranspose1d(idim, idim, self.upsample_times, self.upsample_times)
+ from funasr.models.encoder.transformer_encoder import EncoderLayer as TransformerEncoderLayer
+ from funasr.modules.attention import MultiHeadedAttention
+ from funasr.modules.positionwise_feed_forward import PositionwiseFeedForward
+ positionwise_layer_args = (
+ idim,
+ idim*2,
+ 0.1,
+ )
+ self.self_attn = TransformerEncoderLayer(
+ idim,
+ MultiHeadedAttention(
+ 4, idim, 0.1
+ ),
+ PositionwiseFeedForward(*positionwise_layer_args),
+ 0.1,
+ True, #normalize_before,
+ False, #concat_after,
+ )
+ self.cif_output2 = nn.Linear(idim, 1)
+ self.smooth_factor2 = smooth_factor2
+ self.noise_threshold2 = noise_threshold2
+
+ def forward(self, hidden, target_label=None, mask=None, ignore_id=-1, mask_chunk_predictor=None,
+ target_label_length=None):
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+
+ # alphas2 is an extra head for timestamp prediction
+ if not self.use_cif1_cnn:
+ _output = context
+ else:
+ _output = output
+ if self.upsample_type == 'cnn':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ elif self.upsample_type == 'cnn_blstm':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ output2, (_, _) = self.blstm(output2)
+ elif self.upsample_type == 'cnn_attn':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ output2, _ = self.self_attn(output2, mask)
+ # import pdb; pdb.set_trace()
+ alphas2 = torch.sigmoid(self.cif_output2(output2))
+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
+ # repeat the mask in T demension to match the upsampled length
+ if mask is not None:
+ mask2 = mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)
+ mask2 = mask2.unsqueeze(-1)
+ alphas2 = alphas2 * mask2
+ alphas2 = alphas2.squeeze(-1)
+ token_num2 = alphas2.sum(-1)
+
+ output = output.transpose(1, 2)
+
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ if mask is not None:
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ if mask_chunk_predictor is not None:
+ alphas = alphas * mask_chunk_predictor
+ alphas = alphas.squeeze(-1)
+ mask = mask.squeeze(-1)
+ if target_label_length is not None:
+ target_length = target_label_length
+ elif target_label is not None:
+ target_length = (target_label != ignore_id).float().sum(-1)
+ else:
+ target_length = None
+ token_num = alphas.sum(-1)
+
+ if target_length is not None:
+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
+ elif self.tail_threshold > 0.0:
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)
+
+ acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
+ if target_length is None and self.tail_threshold > 0.0:
+ token_num_int = torch.max(token_num).type(torch.int32).item()
+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
+ return acoustic_embeds, token_num, alphas, cif_peak, token_num2
+
+ def get_upsample_timestamp(self, hidden, target_label=None, mask=None, ignore_id=-1, mask_chunk_predictor=None,
+ target_label_length=None, token_num=None):
+ h = hidden
+ b = hidden.shape[0]
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+
+ # alphas2 is an extra head for timestamp prediction
+ if not self.use_cif1_cnn:
+ _output = context
+ else:
+ _output = output
+ if self.upsample_type == 'cnn':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ elif self.upsample_type == 'cnn_blstm':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ output2, (_, _) = self.blstm(output2)
+ elif self.upsample_type == 'cnn_attn':
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1,2)
+ output2, _ = self.self_attn(output2, mask)
+ alphas2 = torch.sigmoid(self.cif_output2(output2))
+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
+ # repeat the mask in T demension to match the upsampled length
+ if mask is not None:
+ mask2 = mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)
+ mask2 = mask2.unsqueeze(-1)
+ alphas2 = alphas2 * mask2
+ alphas2 = alphas2.squeeze(-1)
+ _token_num = alphas2.sum(-1)
+ if token_num is not None:
+ alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))
+ # re-downsample
+ ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1)
+ ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4)
+ # upsampled alphas and cif_peak
+ us_alphas = alphas2
+ us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4)
+ return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak
def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
b, t, d = hidden.size()
@@ -248,65 +654,4 @@
predictor_alignments = index_div_bool_zeros_count_tile_out
predictor_alignments_length = predictor_alignments.sum(-1).type(encoder_sequence_length.dtype)
- return predictor_alignments.detach(), predictor_alignments_length.detach()
-
-
-class mae_loss(nn.Module):
-
- def __init__(self, normalize_length=False):
- super(mae_loss, self).__init__()
- self.normalize_length = normalize_length
- self.criterion = torch.nn.L1Loss(reduction='sum')
-
- def forward(self, token_length, pre_token_length):
- loss_token_normalizer = token_length.size(0)
- if self.normalize_length:
- loss_token_normalizer = token_length.sum().type(torch.float32)
- loss = self.criterion(token_length, pre_token_length)
- loss = loss / loss_token_normalizer
- return loss
-
-
-def cif(hidden, alphas, threshold):
- batch_size, len_time, hidden_size = hidden.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], device=hidden.device)
- frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
- # intermediate vars along time
- list_fires = []
- list_frames = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
- distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(fire_place,
- integrate - torch.ones([batch_size], device=hidden.device),
- integrate)
- cur = torch.where(fire_place,
- distribution_completion,
- alpha)
- remainds = alpha - cur
-
- frame += cur[:, None] * hidden[:, t, :]
- list_frames.append(frame)
- frame = torch.where(fire_place[:, None].repeat(1, hidden_size),
- remainds[:, None] * hidden[:, t, :],
- frame)
-
- fires = torch.stack(list_fires, 1)
- frames = torch.stack(list_frames, 1)
- list_ls = []
- len_labels = torch.round(alphas.sum(-1)).int()
- max_label_len = len_labels.max()
- for b in range(batch_size):
- fire = fires[b, :]
- l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())
- pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
- list_ls.append(torch.cat([l, pad_l], 0))
- return torch.stack(list_ls, 0), fires
+ return predictor_alignments.detach(), predictor_alignments_length.detach()
\ No newline at end of file
diff --git a/egs_modelscope/common/utils/__init__.py b/funasr/modules/data2vec/__init__.py
similarity index 100%
copy from egs_modelscope/common/utils/__init__.py
copy to funasr/modules/data2vec/__init__.py
diff --git a/funasr/modules/data2vec/data_utils.py b/funasr/modules/data2vec/data_utils.py
new file mode 100644
index 0000000..efb72fa
--- /dev/null
+++ b/funasr/modules/data2vec/data_utils.py
@@ -0,0 +1,147 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from typing import Optional, Tuple
+
+import numpy as np
+import torch
+
+
+def compute_mask_indices(
+ shape: Tuple[int, int],
+ padding_mask: Optional[torch.Tensor],
+ mask_prob: float,
+ mask_length: int,
+ mask_type: str = "static",
+ mask_other: float = 0.0,
+ min_masks: int = 0,
+ no_overlap: bool = False,
+ min_space: int = 0,
+ require_same_masks: bool = True,
+ mask_dropout: float = 0.0,
+) -> np.ndarray:
+ """
+ Computes random mask spans for a given shape
+
+ Args:
+ shape: the the shape for which to compute masks.
+ should be of size 2 where first element is batch size and 2nd is timesteps
+ padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements
+ mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by
+ number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
+ however due to overlaps, the actual number will be smaller (unless no_overlap is True)
+ mask_type: how to compute mask lengths
+ static = fixed size
+ uniform = sample from uniform distribution [mask_other, mask_length*2]
+ normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element
+ poisson = sample from possion distribution with lambda = mask length
+ min_masks: minimum number of masked spans
+ no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping
+ min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans
+ require_same_masks: if true, will randomly drop out masks until same amount of masks remains in each sample
+ mask_dropout: randomly dropout this percentage of masks in each example
+ """
+
+ bsz, all_sz = shape
+ mask = np.full((bsz, all_sz), False)
+
+ all_num_mask = int(
+ # add a random number for probabilistic rounding
+ mask_prob * all_sz / float(mask_length)
+ + np.random.rand()
+ )
+
+ all_num_mask = max(min_masks, all_num_mask)
+
+ mask_idcs = []
+ for i in range(bsz):
+ if padding_mask is not None:
+ sz = all_sz - padding_mask[i].long().sum().item()
+ num_mask = int(
+ # add a random number for probabilistic rounding
+ mask_prob * sz / float(mask_length)
+ + np.random.rand()
+ )
+ num_mask = max(min_masks, num_mask)
+ else:
+ sz = all_sz
+ num_mask = all_num_mask
+
+ if mask_type == "static":
+ lengths = np.full(num_mask, mask_length)
+ elif mask_type == "uniform":
+ lengths = np.random.randint(mask_other, mask_length * 2 + 1, size=num_mask)
+ elif mask_type == "normal":
+ lengths = np.random.normal(mask_length, mask_other, size=num_mask)
+ lengths = [max(1, int(round(x))) for x in lengths]
+ elif mask_type == "poisson":
+ lengths = np.random.poisson(mask_length, size=num_mask)
+ lengths = [int(round(x)) for x in lengths]
+ else:
+ raise Exception("unknown mask selection " + mask_type)
+
+ if sum(lengths) == 0:
+ lengths[0] = min(mask_length, sz - 1)
+
+ if no_overlap:
+ mask_idc = []
+
+ def arrange(s, e, length, keep_length):
+ span_start = np.random.randint(s, e - length)
+ mask_idc.extend(span_start + i for i in range(length))
+
+ new_parts = []
+ if span_start - s - min_space >= keep_length:
+ new_parts.append((s, span_start - min_space + 1))
+ if e - span_start - length - min_space > keep_length:
+ new_parts.append((span_start + length + min_space, e))
+ return new_parts
+
+ parts = [(0, sz)]
+ min_length = min(lengths)
+ for length in sorted(lengths, reverse=True):
+ lens = np.fromiter(
+ (e - s if e - s >= length + min_space else 0 for s, e in parts),
+ np.int,
+ )
+ l_sum = np.sum(lens)
+ if l_sum == 0:
+ break
+ probs = lens / np.sum(lens)
+ c = np.random.choice(len(parts), p=probs)
+ s, e = parts.pop(c)
+ parts.extend(arrange(s, e, length, min_length))
+ mask_idc = np.asarray(mask_idc)
+ else:
+ min_len = min(lengths)
+ if sz - min_len <= num_mask:
+ min_len = sz - num_mask - 1
+
+ mask_idc = np.random.choice(sz - min_len, num_mask, replace=False)
+
+ mask_idc = np.asarray(
+ [
+ mask_idc[j] + offset
+ for j in range(len(mask_idc))
+ for offset in range(lengths[j])
+ ]
+ )
+
+ mask_idcs.append(np.unique(mask_idc[mask_idc < sz]))
+
+ min_len = min([len(m) for m in mask_idcs])
+ for i, mask_idc in enumerate(mask_idcs):
+ if len(mask_idc) > min_len and require_same_masks:
+ mask_idc = np.random.choice(mask_idc, min_len, replace=False)
+ if mask_dropout > 0:
+ num_holes = np.rint(len(mask_idc) * mask_dropout).astype(int)
+ mask_idc = np.random.choice(
+ mask_idc, len(mask_idc) - num_holes, replace=False
+ )
+
+ mask[i, mask_idc] = True
+
+ return mask
diff --git a/funasr/modules/data2vec/ema_module.py b/funasr/modules/data2vec/ema_module.py
new file mode 100644
index 0000000..4e46f50
--- /dev/null
+++ b/funasr/modules/data2vec/ema_module.py
@@ -0,0 +1,132 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Used for EMA tracking a given pytorch module. The user is responsible for calling step()
+and setting the appropriate decay
+"""
+
+import copy
+import logging
+
+import torch
+
+
+class EMAModule:
+ """Exponential Moving Average of Fairseq Models"""
+
+ def __init__(self, model, ema_decay=0.9999, ema_fp32=False, device=None, skip_keys=None):
+ """
+ @param model model to initialize the EMA with
+ @param config EMAConfig object with configuration like
+ ema_decay, ema_update_freq, ema_fp32
+ @param device If provided, copy EMA to this device (e.g. gpu).
+ Otherwise EMA is in the same device as the model.
+ """
+
+ self.decay = ema_decay
+ self.ema_fp32 = ema_fp32
+ self.model = copy.deepcopy(model)
+ self.model.requires_grad_(False)
+ self.skip_keys = skip_keys or set()
+ self.fp32_params = {}
+
+ if device is not None:
+ logging.info(f"Copying EMA model to device {device}")
+ self.model = self.model.to(device=device)
+
+ if self.ema_fp32:
+ self.build_fp32_params()
+
+ self.update_freq_counter = 0
+
+ def build_fp32_params(self, state_dict=None):
+ """
+ Store a copy of the EMA params in fp32.
+ If state dict is passed, the EMA params is copied from
+ the provided state dict. Otherwise, it is copied from the
+ current EMA model parameters.
+ """
+ if not self.ema_fp32:
+ raise RuntimeError(
+ "build_fp32_params should not be called if ema_fp32=False. "
+ "Use ema_fp32=True if this is really intended."
+ )
+
+ if state_dict is None:
+ state_dict = self.model.state_dict()
+
+ def _to_float(t):
+ return t.float() if torch.is_floating_point(t) else t
+
+ for param_key in state_dict:
+ if param_key in self.fp32_params:
+ self.fp32_params[param_key].copy_(state_dict[param_key])
+ else:
+ self.fp32_params[param_key] = _to_float(state_dict[param_key])
+
+ def restore(self, state_dict, build_fp32_params=False):
+ """Load data from a model spec into EMA model"""
+ self.model.load_state_dict(state_dict, strict=False)
+ if build_fp32_params:
+ self.build_fp32_params(state_dict)
+
+ def set_decay(self, decay):
+ self.decay = decay
+
+ def get_decay(self):
+ return self.decay
+
+ def _step_internal(self, new_model):
+ """One update of the EMA model based on new model weights"""
+ decay = self.decay
+
+ ema_state_dict = {}
+ ema_params = (
+ self.fp32_params if self.ema_fp32 else self.model.state_dict()
+ )
+ for key, param in new_model.state_dict().items():
+ if isinstance(param, dict):
+ continue
+ try:
+ ema_param = ema_params[key]
+ except KeyError:
+ ema_param = (
+ param.float().clone() if param.ndim == 1 else copy.deepcopy(param)
+ )
+
+ if param.shape != ema_param.shape:
+ raise ValueError(
+ "incompatible tensor shapes between model param and ema param"
+ + "{} vs. {}".format(param.shape, ema_param.shape)
+ )
+
+ if "version" in key:
+ # Do not decay a model.version pytorch param
+ continue
+
+ if key in self.skip_keys or ("num_batches_tracked" in key and ema_param.dtype == torch.int64):
+ ema_param = param.to(dtype=ema_param.dtype).clone()
+ ema_params[key].copy_(ema_param)
+ else:
+ ema_param.mul_(decay)
+ ema_param.add_(param.to(dtype=ema_param.dtype), alpha=1 - decay)
+ ema_state_dict[key] = ema_param
+ self.restore(ema_state_dict, build_fp32_params=False)
+
+ def step(self, new_model):
+ self._step_internal(new_model)
+
+ def reverse(self, model):
+ """
+ Load the model parameters from EMA model.
+ Useful for inference or fine-tuning from the EMA model.
+ """
+ d = self.model.state_dict()
+ if "_ema" in d:
+ del d["_ema"]
+
+ model.load_state_dict(d, strict=False)
+ return model
diff --git a/funasr/modules/data2vec/grad_multiply.py b/funasr/modules/data2vec/grad_multiply.py
new file mode 100644
index 0000000..08d15f5
--- /dev/null
+++ b/funasr/modules/data2vec/grad_multiply.py
@@ -0,0 +1,18 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+
+
+class GradMultiply(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, scale):
+ ctx.scale = scale
+ res = x.new(x)
+ return res
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad * ctx.scale, None
diff --git a/funasr/modules/data2vec/multihead_attention.py b/funasr/modules/data2vec/multihead_attention.py
new file mode 100644
index 0000000..e5171ed
--- /dev/null
+++ b/funasr/modules/data2vec/multihead_attention.py
@@ -0,0 +1,671 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import math
+from typing import Dict, List, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor, nn
+from torch.nn import Parameter
+
+from funasr.modules.data2vec.quant_noise import quant_noise
+
+
+class FairseqDropout(nn.Module):
+ def __init__(self, p, module_name=None):
+ super().__init__()
+ self.p = p
+ self.module_name = module_name
+ self.apply_during_inference = False
+
+ def forward(self, x, inplace: bool = False):
+ if self.p > 0 and (self.training or self.apply_during_inference):
+ return F.dropout(x, p=self.p, training=True, inplace=inplace)
+ else:
+ return x
+
+ def make_generation_fast_(
+ self,
+ name: str,
+ retain_dropout: bool = False,
+ retain_dropout_modules: Optional[List[str]] = None,
+ **kwargs
+ ):
+ if retain_dropout:
+ if retain_dropout_modules is not None and self.module_name is None:
+ logging.warning(
+ "Cannot enable dropout during inference for module {} "
+ "because module_name was not set".format(name)
+ )
+ elif (
+ retain_dropout_modules is None # if None, apply to all modules
+ or self.module_name in retain_dropout_modules
+ ):
+ logging.info(
+ "Enabling dropout during inference for module: {}".format(name)
+ )
+ self.apply_during_inference = True
+ else:
+ logging.info("Disabling dropout for module: {}".format(name))
+
+
+class MultiheadAttention(nn.Module):
+ """Multi-headed attention.
+
+ See "Attention Is All You Need" for more details.
+ """
+
+ def __init__(
+ self,
+ embed_dim,
+ num_heads,
+ kdim=None,
+ vdim=None,
+ dropout=0.0,
+ bias=True,
+ add_bias_kv=False,
+ add_zero_attn=False,
+ self_attention=False,
+ encoder_decoder_attention=False,
+ q_noise=0.0,
+ qn_block_size=8,
+ ):
+ super().__init__()
+ self.embed_dim = embed_dim
+ self.kdim = kdim if kdim is not None else embed_dim
+ self.vdim = vdim if vdim is not None else embed_dim
+ self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim
+
+ self.num_heads = num_heads
+ self.dropout_module = FairseqDropout(
+ dropout, module_name=self.__class__.__name__
+ )
+
+ self.head_dim = embed_dim // num_heads
+ assert (
+ self.head_dim * num_heads == self.embed_dim
+ ), "embed_dim must be divisible by num_heads"
+ self.scaling = self.head_dim ** -0.5
+
+ self.self_attention = self_attention
+ self.encoder_decoder_attention = encoder_decoder_attention
+
+ assert not self.self_attention or self.qkv_same_dim, (
+ "Self-attention requires query, key and " "value to be of the same size"
+ )
+
+ self.k_proj = quant_noise(
+ nn.Linear(self.kdim, embed_dim, bias=bias), q_noise, qn_block_size
+ )
+ self.v_proj = quant_noise(
+ nn.Linear(self.vdim, embed_dim, bias=bias), q_noise, qn_block_size
+ )
+ self.q_proj = quant_noise(
+ nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size
+ )
+
+ self.out_proj = quant_noise(
+ nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size
+ )
+
+ if add_bias_kv:
+ self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
+ self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
+ else:
+ self.bias_k = self.bias_v = None
+
+ self.add_zero_attn = add_zero_attn
+
+ self.reset_parameters()
+
+ self.onnx_trace = False
+ self.skip_embed_dim_check = False
+
+ def prepare_for_onnx_export_(self):
+ self.onnx_trace = True
+
+ def reset_parameters(self):
+ if self.qkv_same_dim:
+ # Empirically observed the convergence to be much better with
+ # the scaled initialization
+ nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2))
+ nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2))
+ nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2))
+ else:
+ nn.init.xavier_uniform_(self.k_proj.weight)
+ nn.init.xavier_uniform_(self.v_proj.weight)
+ nn.init.xavier_uniform_(self.q_proj.weight)
+
+ nn.init.xavier_uniform_(self.out_proj.weight)
+ if self.out_proj.bias is not None:
+ nn.init.constant_(self.out_proj.bias, 0.0)
+ if self.bias_k is not None:
+ nn.init.xavier_normal_(self.bias_k)
+ if self.bias_v is not None:
+ nn.init.xavier_normal_(self.bias_v)
+
+ def _get_reserve_head_index(self, num_heads_to_keep: int):
+ k_proj_heads_norm = []
+ q_proj_heads_norm = []
+ v_proj_heads_norm = []
+
+ for i in range(self.num_heads):
+ start_idx = i * self.head_dim
+ end_idx = (i + 1) * self.head_dim
+ k_proj_heads_norm.append(
+ torch.sum(
+ torch.abs(
+ self.k_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+ ).tolist()
+ + torch.sum(torch.abs(self.k_proj.bias[start_idx:end_idx])).tolist()
+ )
+ q_proj_heads_norm.append(
+ torch.sum(
+ torch.abs(
+ self.q_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+ ).tolist()
+ + torch.sum(torch.abs(self.q_proj.bias[start_idx:end_idx])).tolist()
+ )
+ v_proj_heads_norm.append(
+ torch.sum(
+ torch.abs(
+ self.v_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+ ).tolist()
+ + torch.sum(torch.abs(self.v_proj.bias[start_idx:end_idx])).tolist()
+ )
+
+ heads_norm = []
+ for i in range(self.num_heads):
+ heads_norm.append(
+ k_proj_heads_norm[i] + q_proj_heads_norm[i] + v_proj_heads_norm[i]
+ )
+
+ sorted_head_index = sorted(
+ range(self.num_heads), key=lambda k: heads_norm[k], reverse=True
+ )
+ reserve_head_index = []
+ for i in range(num_heads_to_keep):
+ start = sorted_head_index[i] * self.head_dim
+ end = (sorted_head_index[i] + 1) * self.head_dim
+ reserve_head_index.append((start, end))
+ return reserve_head_index
+
+ def _adaptive_prune_heads(self, reserve_head_index: List[Tuple[int, int]]):
+ new_q_weight = []
+ new_q_bias = []
+ new_k_weight = []
+ new_k_bias = []
+ new_v_weight = []
+ new_v_bias = []
+ new_out_proj_weight = []
+
+ for ele in reserve_head_index:
+ start_idx, end_idx = ele
+ new_q_weight.append(
+ self.q_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+ new_q_bias.append(self.q_proj.bias[start_idx:end_idx])
+
+ new_k_weight.append(
+ self.k_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+
+ new_k_bias.append(self.k_proj.bias[start_idx:end_idx])
+
+ new_v_weight.append(
+ self.v_proj.weight[
+ start_idx:end_idx,
+ ]
+ )
+ new_v_bias.append(self.v_proj.bias[start_idx:end_idx])
+
+ new_out_proj_weight.append(self.out_proj.weight[:, start_idx:end_idx])
+
+ new_q_weight = torch.cat(new_q_weight).detach()
+ new_k_weight = torch.cat(new_k_weight).detach()
+ new_v_weight = torch.cat(new_v_weight).detach()
+ new_out_proj_weight = torch.cat(new_out_proj_weight, dim=-1).detach()
+ new_q_weight.requires_grad = True
+ new_k_weight.requires_grad = True
+ new_v_weight.requires_grad = True
+ new_out_proj_weight.requires_grad = True
+
+ new_q_bias = torch.cat(new_q_bias).detach()
+ new_q_bias.requires_grad = True
+
+ new_k_bias = torch.cat(new_k_bias).detach()
+ new_k_bias.requires_grad = True
+
+ new_v_bias = torch.cat(new_v_bias).detach()
+ new_v_bias.requires_grad = True
+
+ self.q_proj.weight = torch.nn.Parameter(new_q_weight)
+ self.q_proj.bias = torch.nn.Parameter(new_q_bias)
+
+ self.k_proj.weight = torch.nn.Parameter(new_k_weight)
+ self.k_proj.bias = torch.nn.Parameter(new_k_bias)
+
+ self.v_proj.weight = torch.nn.Parameter(new_v_weight)
+ self.v_proj.bias = torch.nn.Parameter(new_v_bias)
+
+ self.out_proj.weight = torch.nn.Parameter(new_out_proj_weight)
+
+ self.num_heads = len(reserve_head_index)
+ self.embed_dim = self.head_dim * self.num_heads
+ self.q_proj.out_features = self.embed_dim
+ self.k_proj.out_features = self.embed_dim
+ self.v_proj.out_features = self.embed_dim
+
+ def _set_skip_embed_dim_check(self):
+ self.skip_embed_dim_check = True
+
+ def forward(
+ self,
+ query,
+ key: Optional[Tensor],
+ value: Optional[Tensor],
+ key_padding_mask: Optional[Tensor] = None,
+ incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
+ need_weights: bool = True,
+ static_kv: bool = False,
+ attn_mask: Optional[Tensor] = None,
+ before_softmax: bool = False,
+ need_head_weights: bool = False,
+ ) -> Tuple[Tensor, Optional[Tensor]]:
+ """Input shape: Time x Batch x Channel
+
+ Args:
+ key_padding_mask (ByteTensor, optional): mask to exclude
+ keys that are pads, of shape `(batch, src_len)`, where
+ padding elements are indicated by 1s.
+ need_weights (bool, optional): return the attention weights,
+ averaged over heads (default: False).
+ attn_mask (ByteTensor, optional): typically used to
+ implement causal attention, where the mask prevents the
+ attention from looking forward in time (default: None).
+ before_softmax (bool, optional): return the raw attention
+ weights and values before the attention softmax.
+ need_head_weights (bool, optional): return the attention
+ weights for each head. Implies *need_weights*. Default:
+ return the average attention weights over all heads.
+ """
+ if need_head_weights:
+ need_weights = True
+
+ is_tpu = query.device.type == "xla"
+
+ tgt_len, bsz, embed_dim = query.size()
+ src_len = tgt_len
+ if not self.skip_embed_dim_check:
+ assert (
+ embed_dim == self.embed_dim
+ ), f"query dim {embed_dim} != {self.embed_dim}"
+ assert list(query.size()) == [tgt_len, bsz, embed_dim]
+ if key is not None:
+ src_len, key_bsz, _ = key.size()
+ if not torch.jit.is_scripting():
+ assert key_bsz == bsz
+ assert value is not None
+ assert src_len, bsz == value.shape[:2]
+
+ if (
+ not self.onnx_trace
+ and not is_tpu # don't use PyTorch version on TPUs
+ and incremental_state is None
+ and not static_kv
+ # A workaround for quantization to work. Otherwise JIT compilation
+ # treats bias in linear module as method.
+ and not torch.jit.is_scripting()
+ # The Multihead attention implemented in pytorch forces strong dimension check
+ # for input embedding dimention and K,Q,V projection dimension.
+ # Since pruning will break the dimension check and it is not easy to modify the pytorch API,
+ # it is preferred to bypass the pytorch MHA when we need to skip embed_dim_check
+ and not self.skip_embed_dim_check
+ ):
+ assert key is not None and value is not None
+ return F.multi_head_attention_forward(
+ query,
+ key,
+ value,
+ self.embed_dim,
+ self.num_heads,
+ torch.empty([0]),
+ torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)),
+ self.bias_k,
+ self.bias_v,
+ self.add_zero_attn,
+ self.dropout_module.p,
+ self.out_proj.weight,
+ self.out_proj.bias,
+ self.training or self.dropout_module.apply_during_inference,
+ key_padding_mask,
+ need_weights,
+ attn_mask,
+ use_separate_proj_weight=True,
+ q_proj_weight=self.q_proj.weight,
+ k_proj_weight=self.k_proj.weight,
+ v_proj_weight=self.v_proj.weight,
+ )
+
+ if incremental_state is not None:
+ saved_state = self._get_input_buffer(incremental_state)
+ if saved_state is not None and "prev_key" in saved_state:
+ # previous time steps are cached - no need to recompute
+ # key and value if they are static
+ if static_kv:
+ assert self.encoder_decoder_attention and not self.self_attention
+ key = value = None
+ else:
+ saved_state = None
+
+ if self.self_attention:
+ q = self.q_proj(query)
+ k = self.k_proj(query)
+ v = self.v_proj(query)
+ elif self.encoder_decoder_attention:
+ # encoder-decoder attention
+ q = self.q_proj(query)
+ if key is None:
+ assert value is None
+ k = v = None
+ else:
+ k = self.k_proj(key)
+ v = self.v_proj(key)
+
+ else:
+ assert key is not None and value is not None
+ q = self.q_proj(query)
+ k = self.k_proj(key)
+ v = self.v_proj(value)
+ q *= self.scaling
+
+ if self.bias_k is not None:
+ assert self.bias_v is not None
+ k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)])
+ v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)])
+ if attn_mask is not None:
+ attn_mask = torch.cat(
+ [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
+ )
+ if key_padding_mask is not None:
+ key_padding_mask = torch.cat(
+ [
+ key_padding_mask,
+ key_padding_mask.new_zeros(key_padding_mask.size(0), 1),
+ ],
+ dim=1,
+ )
+
+ q = (
+ q.contiguous()
+ .view(tgt_len, bsz * self.num_heads, self.head_dim)
+ .transpose(0, 1)
+ )
+ if k is not None:
+ k = (
+ k.contiguous()
+ .view(-1, bsz * self.num_heads, self.head_dim)
+ .transpose(0, 1)
+ )
+ if v is not None:
+ v = (
+ v.contiguous()
+ .view(-1, bsz * self.num_heads, self.head_dim)
+ .transpose(0, 1)
+ )
+
+ if saved_state is not None:
+ # saved states are stored with shape (bsz, num_heads, seq_len, head_dim)
+ if "prev_key" in saved_state:
+ _prev_key = saved_state["prev_key"]
+ assert _prev_key is not None
+ prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim)
+ if static_kv:
+ k = prev_key
+ else:
+ assert k is not None
+ k = torch.cat([prev_key, k], dim=1)
+ src_len = k.size(1)
+ if "prev_value" in saved_state:
+ _prev_value = saved_state["prev_value"]
+ assert _prev_value is not None
+ prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim)
+ if static_kv:
+ v = prev_value
+ else:
+ assert v is not None
+ v = torch.cat([prev_value, v], dim=1)
+ prev_key_padding_mask: Optional[Tensor] = None
+ if "prev_key_padding_mask" in saved_state:
+ prev_key_padding_mask = saved_state["prev_key_padding_mask"]
+ assert k is not None and v is not None
+ key_padding_mask = MultiheadAttention._append_prev_key_padding_mask(
+ key_padding_mask=key_padding_mask,
+ prev_key_padding_mask=prev_key_padding_mask,
+ batch_size=bsz,
+ src_len=k.size(1),
+ static_kv=static_kv,
+ )
+
+ saved_state["prev_key"] = k.view(bsz, self.num_heads, -1, self.head_dim)
+ saved_state["prev_value"] = v.view(bsz, self.num_heads, -1, self.head_dim)
+ saved_state["prev_key_padding_mask"] = key_padding_mask
+ # In this branch incremental_state is never None
+ assert incremental_state is not None
+ incremental_state = self._set_input_buffer(incremental_state, saved_state)
+ assert k is not None
+ assert k.size(1) == src_len
+
+ # This is part of a workaround to get around fork/join parallelism
+ # not supporting Optional types.
+ if key_padding_mask is not None and key_padding_mask.dim() == 0:
+ key_padding_mask = None
+
+ if key_padding_mask is not None:
+ assert key_padding_mask.size(0) == bsz
+ assert key_padding_mask.size(1) == src_len
+
+ if self.add_zero_attn:
+ assert v is not None
+ src_len += 1
+ k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1)
+ v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1)
+ if attn_mask is not None:
+ attn_mask = torch.cat(
+ [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
+ )
+ if key_padding_mask is not None:
+ key_padding_mask = torch.cat(
+ [
+ key_padding_mask,
+ torch.zeros(key_padding_mask.size(0), 1).type_as(
+ key_padding_mask
+ ),
+ ],
+ dim=1,
+ )
+
+ attn_weights = torch.bmm(q, k.transpose(1, 2))
+ attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz)
+
+ assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len]
+
+ if attn_mask is not None:
+ attn_mask = attn_mask.unsqueeze(0)
+ if self.onnx_trace:
+ attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1)
+ attn_weights += attn_mask
+
+ if key_padding_mask is not None:
+ # don't attend to padding symbols
+ attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+ if not is_tpu:
+ attn_weights = attn_weights.masked_fill(
+ key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool),
+ float("-inf"),
+ )
+ else:
+ attn_weights = attn_weights.transpose(0, 2)
+ attn_weights = attn_weights.masked_fill(key_padding_mask, float("-inf"))
+ attn_weights = attn_weights.transpose(0, 2)
+ attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+ if before_softmax:
+ return attn_weights, v
+
+ attn_weights_float = F.softmax(attn_weights, dim=-1, dtype=torch.float32)
+ attn_weights = attn_weights_float.type_as(attn_weights)
+ attn_probs = self.dropout_module(attn_weights)
+
+ assert v is not None
+ attn = torch.bmm(attn_probs, v)
+ assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim]
+ if self.onnx_trace and attn.size(1) == 1:
+ # when ONNX tracing a single decoder step (sequence length == 1)
+ # the transpose is a no-op copy before view, thus unnecessary
+ attn = attn.contiguous().view(tgt_len, bsz, self.embed_dim)
+ else:
+ attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, self.embed_dim)
+ attn = self.out_proj(attn)
+ attn_weights: Optional[Tensor] = None
+ if need_weights:
+ attn_weights = attn_weights_float.view(
+ bsz, self.num_heads, tgt_len, src_len
+ ).transpose(1, 0)
+ if not need_head_weights:
+ # average attention weights over heads
+ attn_weights = attn_weights.mean(dim=0)
+
+ return attn, attn_weights
+
+ @staticmethod
+ def _append_prev_key_padding_mask(
+ key_padding_mask: Optional[Tensor],
+ prev_key_padding_mask: Optional[Tensor],
+ batch_size: int,
+ src_len: int,
+ static_kv: bool,
+ ) -> Optional[Tensor]:
+ # saved key padding masks have shape (bsz, seq_len)
+ if prev_key_padding_mask is not None and static_kv:
+ new_key_padding_mask = prev_key_padding_mask
+ elif prev_key_padding_mask is not None and key_padding_mask is not None:
+ new_key_padding_mask = torch.cat(
+ [prev_key_padding_mask.float(), key_padding_mask.float()], dim=1
+ )
+ # During incremental decoding, as the padding token enters and
+ # leaves the frame, there will be a time when prev or current
+ # is None
+ elif prev_key_padding_mask is not None:
+ if src_len > prev_key_padding_mask.size(1):
+ filler = torch.zeros(
+ (batch_size, src_len - prev_key_padding_mask.size(1)),
+ device=prev_key_padding_mask.device,
+ )
+ new_key_padding_mask = torch.cat(
+ [prev_key_padding_mask.float(), filler.float()], dim=1
+ )
+ else:
+ new_key_padding_mask = prev_key_padding_mask.float()
+ elif key_padding_mask is not None:
+ if src_len > key_padding_mask.size(1):
+ filler = torch.zeros(
+ (batch_size, src_len - key_padding_mask.size(1)),
+ device=key_padding_mask.device,
+ )
+ new_key_padding_mask = torch.cat(
+ [filler.float(), key_padding_mask.float()], dim=1
+ )
+ else:
+ new_key_padding_mask = key_padding_mask.float()
+ else:
+ new_key_padding_mask = prev_key_padding_mask
+ return new_key_padding_mask
+
+ @torch.jit.export
+ def reorder_incremental_state(
+ self,
+ incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
+ new_order: Tensor,
+ ):
+ """Reorder buffered internal state (for incremental generation)."""
+ input_buffer = self._get_input_buffer(incremental_state)
+ if input_buffer is not None:
+ for k in input_buffer.keys():
+ input_buffer_k = input_buffer[k]
+ if input_buffer_k is not None:
+ if self.encoder_decoder_attention and input_buffer_k.size(
+ 0
+ ) == new_order.size(0):
+ break
+ input_buffer[k] = input_buffer_k.index_select(0, new_order)
+ incremental_state = self._set_input_buffer(incremental_state, input_buffer)
+ return incremental_state
+
+ def _get_input_buffer(
+ self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]]
+ ) -> Dict[str, Optional[Tensor]]:
+ result = self.get_incremental_state(incremental_state, "attn_state")
+ if result is not None:
+ return result
+ else:
+ empty_result: Dict[str, Optional[Tensor]] = {}
+ return empty_result
+
+ def _set_input_buffer(
+ self,
+ incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
+ buffer: Dict[str, Optional[Tensor]],
+ ):
+ return self.set_incremental_state(incremental_state, "attn_state", buffer)
+
+ def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, bsz: int):
+ return attn_weights
+
+ def upgrade_state_dict_named(self, state_dict, name):
+ prefix = name + "." if name != "" else ""
+ items_to_add = {}
+ keys_to_remove = []
+ for k in state_dict.keys():
+ if k.endswith(prefix + "in_proj_weight"):
+ # in_proj_weight used to be q + k + v with same dimensions
+ dim = int(state_dict[k].shape[0] / 3)
+ items_to_add[prefix + "q_proj.weight"] = state_dict[k][:dim]
+ items_to_add[prefix + "k_proj.weight"] = state_dict[k][dim: 2 * dim]
+ items_to_add[prefix + "v_proj.weight"] = state_dict[k][2 * dim:]
+
+ keys_to_remove.append(k)
+
+ k_bias = prefix + "in_proj_bias"
+ if k_bias in state_dict.keys():
+ dim = int(state_dict[k].shape[0] / 3)
+ items_to_add[prefix + "q_proj.bias"] = state_dict[k_bias][:dim]
+ items_to_add[prefix + "k_proj.bias"] = state_dict[k_bias][
+ dim: 2 * dim
+ ]
+ items_to_add[prefix + "v_proj.bias"] = state_dict[k_bias][2 * dim:]
+
+ keys_to_remove.append(prefix + "in_proj_bias")
+
+ for k in keys_to_remove:
+ del state_dict[k]
+
+ for key, value in items_to_add.items():
+ state_dict[key] = value
diff --git a/funasr/modules/data2vec/quant_noise.py b/funasr/modules/data2vec/quant_noise.py
new file mode 100644
index 0000000..11a82b6
--- /dev/null
+++ b/funasr/modules/data2vec/quant_noise.py
@@ -0,0 +1,107 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+import torch.nn as nn
+
+
+def quant_noise(module, p, block_size):
+ """
+ Wraps modules and applies quantization noise to the weights for
+ subsequent quantization with Iterative Product Quantization as
+ described in "Training with Quantization Noise for Extreme Model Compression"
+
+ Args:
+ - module: nn.Module
+ - p: amount of Quantization Noise
+ - block_size: size of the blocks for subsequent quantization with iPQ
+
+ Remarks:
+ - Module weights must have the right sizes wrt the block size
+ - Only Linear, Embedding and Conv2d modules are supported for the moment
+ - For more detail on how to quantize by blocks with convolutional weights,
+ see "And the Bit Goes Down: Revisiting the Quantization of Neural Networks"
+ - We implement the simplest form of noise here as stated in the paper
+ which consists in randomly dropping blocks
+ """
+
+ # if no quantization noise, don't register hook
+ if p <= 0:
+ return module
+
+ # supported modules
+ assert isinstance(module, (nn.Linear, nn.Embedding, nn.Conv2d))
+
+ # test whether module.weight has the right sizes wrt block_size
+ is_conv = module.weight.ndim == 4
+
+ # 2D matrix
+ if not is_conv:
+ assert (
+ module.weight.size(1) % block_size == 0
+ ), "Input features must be a multiple of block sizes"
+
+ # 4D matrix
+ else:
+ # 1x1 convolutions
+ if module.kernel_size == (1, 1):
+ assert (
+ module.in_channels % block_size == 0
+ ), "Input channels must be a multiple of block sizes"
+ # regular convolutions
+ else:
+ k = module.kernel_size[0] * module.kernel_size[1]
+ assert k % block_size == 0, "Kernel size must be a multiple of block size"
+
+ def _forward_pre_hook(mod, input):
+ # no noise for evaluation
+ if mod.training:
+ if not is_conv:
+ # gather weight and sizes
+ weight = mod.weight
+ in_features = weight.size(1)
+ out_features = weight.size(0)
+
+ # split weight matrix into blocks and randomly drop selected blocks
+ mask = torch.zeros(
+ in_features // block_size * out_features, device=weight.device
+ )
+ mask.bernoulli_(p)
+ mask = mask.repeat_interleave(block_size, -1).view(-1, in_features)
+
+ else:
+ # gather weight and sizes
+ weight = mod.weight
+ in_channels = mod.in_channels
+ out_channels = mod.out_channels
+
+ # split weight matrix into blocks and randomly drop selected blocks
+ if mod.kernel_size == (1, 1):
+ mask = torch.zeros(
+ int(in_channels // block_size * out_channels),
+ device=weight.device,
+ )
+ mask.bernoulli_(p)
+ mask = mask.repeat_interleave(block_size, -1).view(-1, in_channels)
+ else:
+ mask = torch.zeros(
+ weight.size(0), weight.size(1), device=weight.device
+ )
+ mask.bernoulli_(p)
+ mask = (
+ mask.unsqueeze(2)
+ .unsqueeze(3)
+ .repeat(1, 1, mod.kernel_size[0], mod.kernel_size[1])
+ )
+
+ # scale weights and apply mask
+ mask = mask.to(
+ torch.bool
+ ) # x.bool() is not currently supported in TorchScript
+ s = 1 / (1 - p)
+ mod.weight.data = s * weight.masked_fill(mask, 0)
+
+ module.register_forward_pre_hook(_forward_pre_hook)
+ return module
diff --git a/funasr/modules/data2vec/utils.py b/funasr/modules/data2vec/utils.py
new file mode 100644
index 0000000..1945ac4
--- /dev/null
+++ b/funasr/modules/data2vec/utils.py
@@ -0,0 +1,156 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from funasr.modules.data2vec.multihead_attention import MultiheadAttention
+
+
+class Fp32LayerNorm(nn.LayerNorm):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def forward(self, input):
+ output = F.layer_norm(
+ input.float(),
+ self.normalized_shape,
+ self.weight.float() if self.weight is not None else None,
+ self.bias.float() if self.bias is not None else None,
+ self.eps,
+ )
+ return output.type_as(input)
+
+
+class Fp32GroupNorm(nn.GroupNorm):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def forward(self, input):
+ output = F.group_norm(
+ input.float(),
+ self.num_groups,
+ self.weight.float() if self.weight is not None else None,
+ self.bias.float() if self.bias is not None else None,
+ self.eps,
+ )
+ return output.type_as(input)
+
+
+class TransposeLast(nn.Module):
+ def __init__(self, deconstruct_idx=None):
+ super().__init__()
+ self.deconstruct_idx = deconstruct_idx
+
+ def forward(self, x):
+ if self.deconstruct_idx is not None:
+ x = x[self.deconstruct_idx]
+ return x.transpose(-2, -1)
+
+
+class SamePad(nn.Module):
+ def __init__(self, kernel_size, causal=False):
+ super().__init__()
+ if causal:
+ self.remove = kernel_size - 1
+ else:
+ self.remove = 1 if kernel_size % 2 == 0 else 0
+
+ def forward(self, x):
+ if self.remove > 0:
+ x = x[:, :, : -self.remove]
+ return x
+
+
+def pad_to_multiple(x, multiple, dim=-1, value=0):
+ # Inspired from https://github.com/lucidrains/local-attention/blob/master/local_attention/local_attention.py#L41
+ if x is None:
+ return None, 0
+ tsz = x.size(dim)
+ m = tsz / multiple
+ remainder = math.ceil(m) * multiple - tsz
+ if m.is_integer():
+ return x, 0
+ pad_offset = (0,) * (-1 - dim) * 2
+
+ return F.pad(x, (*pad_offset, 0, remainder), value=value), remainder
+
+
+def gelu_accurate(x):
+ if not hasattr(gelu_accurate, "_a"):
+ gelu_accurate._a = math.sqrt(2 / math.pi)
+ return (
+ 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3))))
+ )
+
+
+def gelu(x: torch.Tensor) -> torch.Tensor:
+ return torch.nn.functional.gelu(x.float()).type_as(x)
+
+
+def get_available_activation_fns():
+ return [
+ "relu",
+ "gelu",
+ "gelu_fast", # deprecated
+ "gelu_accurate",
+ "tanh",
+ "linear",
+ ]
+
+
+def get_activation_fn(activation: str):
+ """Returns the activation function corresponding to `activation`"""
+
+ if activation == "relu":
+ return F.relu
+ elif activation == "gelu":
+ return gelu
+ elif activation == "gelu_accurate":
+ return gelu_accurate
+ elif activation == "tanh":
+ return torch.tanh
+ elif activation == "linear":
+ return lambda x: x
+ elif activation == "swish":
+ return torch.nn.SiLU
+ else:
+ raise RuntimeError("--activation-fn {} not supported".format(activation))
+
+
+def init_bert_params(module):
+ """
+ Initialize the weights specific to the BERT Model.
+ This overrides the default initializations depending on the specified arguments.
+ 1. If normal_init_linear_weights is set then weights of linear
+ layer will be initialized using the normal distribution and
+ bais will be set to the specified value.
+ 2. If normal_init_embed_weights is set then weights of embedding
+ layer will be initialized using the normal distribution.
+ 3. If normal_init_proj_weights is set then weights of
+ in_project_weight for MultiHeadAttention initialized using
+ the normal distribution (to be validated).
+ """
+
+ def normal_(data):
+ # with FSDP, module params will be on CUDA, so we cast them back to CPU
+ # so that the RNG is consistent with and without FSDP
+ data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device))
+
+ if isinstance(module, nn.Linear):
+ normal_(module.weight.data)
+ if module.bias is not None:
+ module.bias.data.zero_()
+ if isinstance(module, nn.Embedding):
+ normal_(module.weight.data)
+ if module.padding_idx is not None:
+ module.weight.data[module.padding_idx].zero_()
+ if isinstance(module, MultiheadAttention):
+ normal_(module.q_proj.weight.data)
+ normal_(module.k_proj.weight.data)
+ normal_(module.v_proj.weight.data)
diff --git a/funasr/modules/data2vec/wav2vec2.py b/funasr/modules/data2vec/wav2vec2.py
new file mode 100644
index 0000000..234ecbb
--- /dev/null
+++ b/funasr/modules/data2vec/wav2vec2.py
@@ -0,0 +1,407 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import math
+from typing import List, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from funasr.modules.data2vec import utils
+from funasr.modules.data2vec.multihead_attention import MultiheadAttention
+
+
+class ConvFeatureExtractionModel(nn.Module):
+ def __init__(
+ self,
+ conv_layers: List[Tuple[int, int, int]],
+ dropout: float = 0.0,
+ mode: str = "default",
+ conv_bias: bool = False,
+ in_d: int = 1
+ ):
+ super().__init__()
+
+ assert mode in {"default", "layer_norm"}
+
+ def block(
+ n_in,
+ n_out,
+ k,
+ stride,
+ is_layer_norm=False,
+ is_group_norm=False,
+ conv_bias=False,
+ ):
+ def make_conv():
+ conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias)
+ nn.init.kaiming_normal_(conv.weight)
+ return conv
+
+ assert (
+ is_layer_norm and is_group_norm
+ ) == False, "layer norm and group norm are exclusive"
+
+ if is_layer_norm:
+ return nn.Sequential(
+ make_conv(),
+ nn.Dropout(p=dropout),
+ nn.Sequential(
+ utils.TransposeLast(),
+ utils.Fp32LayerNorm(dim, elementwise_affine=True),
+ utils.TransposeLast(),
+ ),
+ nn.GELU(),
+ )
+ elif is_group_norm:
+ return nn.Sequential(
+ make_conv(),
+ nn.Dropout(p=dropout),
+ utils.Fp32GroupNorm(dim, dim, affine=True),
+ nn.GELU(),
+ )
+ else:
+ return nn.Sequential(make_conv(), nn.Dropout(p=dropout), nn.GELU())
+
+ self.conv_layers = nn.ModuleList()
+ for i, cl in enumerate(conv_layers):
+ assert len(cl) == 3, "invalid conv definition: " + str(cl)
+ (dim, k, stride) = cl
+
+ self.conv_layers.append(
+ block(
+ in_d,
+ dim,
+ k,
+ stride,
+ is_layer_norm=mode == "layer_norm",
+ is_group_norm=mode == "default" and i == 0,
+ conv_bias=conv_bias,
+ )
+ )
+ in_d = dim
+
+ def forward(self, x):
+ if len(x.shape) == 2:
+ x = x.unsqueeze(1)
+ else:
+ x = x.transpose(1, 2)
+
+ for conv in self.conv_layers:
+ x = conv(x)
+ return x
+
+
+def make_conv_pos(e, k, g):
+ pos_conv = nn.Conv1d(
+ e,
+ e,
+ kernel_size=k,
+ padding=k // 2,
+ groups=g,
+ )
+ dropout = 0
+ std = math.sqrt((4 * (1.0 - dropout)) / (k * e))
+ nn.init.normal_(pos_conv.weight, mean=0, std=std)
+ nn.init.constant_(pos_conv.bias, 0)
+
+ pos_conv = nn.utils.weight_norm(pos_conv, name="weight", dim=2)
+ pos_conv = nn.Sequential(pos_conv, utils.SamePad(k), nn.GELU())
+
+ return pos_conv
+
+
+class TransformerEncoder(nn.Module):
+ def build_encoder_layer(self):
+ if self.layer_type == "transformer":
+ layer = TransformerSentenceEncoderLayer(
+ embedding_dim=self.embedding_dim,
+ ffn_embedding_dim=self.encoder_ffn_embed_dim,
+ num_attention_heads=self.encoder_attention_heads,
+ dropout=self.dropout,
+ attention_dropout=self.attention_dropout,
+ activation_dropout=self.activation_dropout,
+ activation_fn=self.activation_fn,
+ layer_norm_first=self.layer_norm_first,
+ )
+ else:
+ logging.error("Only transformer is supported for data2vec now")
+ return layer
+
+ def __init__(
+ self,
+ # position
+ dropout,
+ encoder_embed_dim,
+ required_seq_len_multiple,
+ pos_conv_depth,
+ conv_pos,
+ conv_pos_groups,
+ # transformer layers
+ layer_type,
+ encoder_layers,
+ encoder_ffn_embed_dim,
+ encoder_attention_heads,
+ attention_dropout,
+ activation_dropout,
+ activation_fn,
+ layer_norm_first,
+ encoder_layerdrop,
+ max_positions,
+ ):
+ super().__init__()
+
+ # position
+ self.dropout = dropout
+ self.embedding_dim = encoder_embed_dim
+ self.required_seq_len_multiple = required_seq_len_multiple
+ if pos_conv_depth > 1:
+ num_layers = pos_conv_depth
+ k = max(3, conv_pos // num_layers)
+
+ def make_conv_block(e, k, g, l):
+ return nn.Sequential(
+ *[
+ nn.Sequential(
+ nn.Conv1d(
+ e,
+ e,
+ kernel_size=k,
+ padding=k // 2,
+ groups=g,
+ ),
+ utils.SamePad(k),
+ utils.TransposeLast(),
+ torch.nn.LayerNorm(e, elementwise_affine=False),
+ utils.TransposeLast(),
+ nn.GELU(),
+ )
+ for _ in range(l)
+ ]
+ )
+
+ self.pos_conv = make_conv_block(
+ self.embedding_dim, k, conv_pos_groups, num_layers
+ )
+
+ else:
+ self.pos_conv = make_conv_pos(
+ self.embedding_dim,
+ conv_pos,
+ conv_pos_groups,
+ )
+
+ # transformer layers
+ self.layer_type = layer_type
+ self.encoder_ffn_embed_dim = encoder_ffn_embed_dim
+ self.encoder_attention_heads = encoder_attention_heads
+ self.attention_dropout = attention_dropout
+ self.activation_dropout = activation_dropout
+ self.activation_fn = activation_fn
+ self.layer_norm_first = layer_norm_first
+ self.layerdrop = encoder_layerdrop
+ self.max_positions = max_positions
+ self.layers = nn.ModuleList(
+ [self.build_encoder_layer() for _ in range(encoder_layers)]
+ )
+ self.layer_norm = torch.nn.LayerNorm(self.embedding_dim)
+
+ self.apply(utils.init_bert_params)
+
+ def forward(self, x, padding_mask=None, layer=None):
+ x, layer_results = self.extract_features(x, padding_mask, layer)
+
+ if self.layer_norm_first and layer is None:
+ x = self.layer_norm(x)
+
+ return x, layer_results
+
+ def extract_features(
+ self,
+ x,
+ padding_mask=None,
+ tgt_layer=None,
+ min_layer=0,
+ ):
+
+ if padding_mask is not None:
+ x[padding_mask] = 0
+
+ x_conv = self.pos_conv(x.transpose(1, 2))
+ x_conv = x_conv.transpose(1, 2)
+ x = x + x_conv
+
+ if not self.layer_norm_first:
+ x = self.layer_norm(x)
+
+ # pad to the sequence length dimension
+ x, pad_length = utils.pad_to_multiple(
+ x, self.required_seq_len_multiple, dim=-2, value=0
+ )
+ if pad_length > 0 and padding_mask is None:
+ padding_mask = x.new_zeros((x.size(0), x.size(1)), dtype=torch.bool)
+ padding_mask[:, -pad_length:] = True
+ else:
+ padding_mask, _ = utils.pad_to_multiple(
+ padding_mask, self.required_seq_len_multiple, dim=-1, value=True
+ )
+ x = F.dropout(x, p=self.dropout, training=self.training)
+
+ # B x T x C -> T x B x C
+ x = x.transpose(0, 1)
+
+ layer_results = []
+ r = None
+ for i, layer in enumerate(self.layers):
+ dropout_probability = np.random.random() if self.layerdrop > 0 else 1
+ if not self.training or (dropout_probability > self.layerdrop):
+ x, (z, lr) = layer(x, self_attn_padding_mask=padding_mask)
+ if i >= min_layer:
+ layer_results.append((x, z, lr))
+ if i == tgt_layer:
+ r = x
+ break
+
+ if r is not None:
+ x = r
+
+ # T x B x C -> B x T x C
+ x = x.transpose(0, 1)
+
+ # undo paddding
+ if pad_length > 0:
+ x = x[:, :-pad_length]
+
+ def undo_pad(a, b, c):
+ return (
+ a[:-pad_length],
+ b[:-pad_length] if b is not None else b,
+ c[:-pad_length],
+ )
+
+ layer_results = [undo_pad(*u) for u in layer_results]
+
+ return x, layer_results
+
+ def max_positions(self):
+ """Maximum output length supported by the encoder."""
+ return self.max_positions
+
+ def upgrade_state_dict_named(self, state_dict, name):
+ """Upgrade a (possibly old) state dict for new versions of fairseq."""
+ return state_dict
+
+
+class TransformerSentenceEncoderLayer(nn.Module):
+ """
+ Implements a Transformer Encoder Layer used in BERT/XLM style pre-trained
+ models.
+ """
+
+ def __init__(
+ self,
+ embedding_dim: int = 768,
+ ffn_embedding_dim: int = 3072,
+ num_attention_heads: int = 8,
+ dropout: float = 0.1,
+ attention_dropout: float = 0.1,
+ activation_dropout: float = 0.1,
+ activation_fn: str = "relu",
+ layer_norm_first: bool = False,
+ ) -> None:
+
+ super().__init__()
+ # Initialize parameters
+ self.embedding_dim = embedding_dim
+ self.dropout = dropout
+ self.activation_dropout = activation_dropout
+
+ # Initialize blocks
+ self.activation_fn = utils.get_activation_fn(activation_fn)
+ self.self_attn = MultiheadAttention(
+ self.embedding_dim,
+ num_attention_heads,
+ dropout=attention_dropout,
+ self_attention=True,
+ )
+
+ self.dropout1 = nn.Dropout(dropout)
+ self.dropout2 = nn.Dropout(self.activation_dropout)
+ self.dropout3 = nn.Dropout(dropout)
+
+ self.layer_norm_first = layer_norm_first
+
+ # layer norm associated with the self attention layer
+ self.self_attn_layer_norm = torch.nn.LayerNorm(self.embedding_dim)
+ self.fc1 = nn.Linear(self.embedding_dim, ffn_embedding_dim)
+ self.fc2 = nn.Linear(ffn_embedding_dim, self.embedding_dim)
+
+ # layer norm associated with the position wise feed-forward NN
+ self.final_layer_norm = torch.nn.LayerNorm(self.embedding_dim)
+
+ def forward(
+ self,
+ x: torch.Tensor, # (T, B, C)
+ self_attn_mask: torch.Tensor = None,
+ self_attn_padding_mask: torch.Tensor = None,
+ ):
+ """
+ LayerNorm is applied either before or after the self-attention/ffn
+ modules similar to the original Transformer imlementation.
+ """
+ residual = x
+
+ if self.layer_norm_first:
+ x = self.self_attn_layer_norm(x)
+ x, attn = self.self_attn(
+ query=x,
+ key=x,
+ value=x,
+ key_padding_mask=self_attn_padding_mask,
+ attn_mask=self_attn_mask,
+ need_weights=False,
+ )
+ x = self.dropout1(x)
+ x = residual + x
+
+ residual = x
+ x = self.final_layer_norm(x)
+ x = self.activation_fn(self.fc1(x))
+ x = self.dropout2(x)
+ x = self.fc2(x)
+
+ layer_result = x
+
+ x = self.dropout3(x)
+ x = residual + x
+ else:
+ x, attn = self.self_attn(
+ query=x,
+ key=x,
+ value=x,
+ key_padding_mask=self_attn_padding_mask,
+ need_weights=False,
+ )
+
+ x = self.dropout1(x)
+ x = residual + x
+
+ x = self.self_attn_layer_norm(x)
+
+ residual = x
+ x = self.activation_fn(self.fc1(x))
+ x = self.dropout2(x)
+ x = self.fc2(x)
+
+ layer_result = x
+
+ x = self.dropout3(x)
+ x = residual + x
+ x = self.final_layer_norm(x)
+
+ return x, (attn, layer_result)
diff --git a/funasr/modules/streaming_utils/load_fr_tf.py b/funasr/modules/streaming_utils/load_fr_tf.py
new file mode 100644
index 0000000..5c8c275
--- /dev/null
+++ b/funasr/modules/streaming_utils/load_fr_tf.py
@@ -0,0 +1,62 @@
+import numpy as np
+np.set_printoptions(threshold=np.inf)
+import logging
+
+def load_ckpt(checkpoint_path):
+ import tensorflow as tf
+ if tf.__version__.startswith('2'):
+ import tensorflow.compat.v1 as tf
+ tf.disable_v2_behavior()
+ reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_path)
+ else:
+ from tensorflow.python import pywrap_tensorflow
+ reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path)
+ var_to_shape_map = reader.get_variable_to_shape_map()
+
+ var_dict = dict()
+ for var_name in sorted(var_to_shape_map):
+ if "Adam" in var_name:
+ continue
+ tensor = reader.get_tensor(var_name)
+ # print("in ckpt: {}, {}".format(var_name, tensor.shape))
+ # print(tensor)
+ var_dict[var_name] = tensor
+
+ return var_dict
+
+
+
+def load_tf_pb_dict(pb_model):
+ import tensorflow as tf
+ if tf.__version__.startswith('2'):
+ import tensorflow.compat.v1 as tf
+ tf.disable_v2_behavior()
+ # import tensorflow_addons as tfa
+ # from tensorflow_addons.seq2seq.python.ops import beam_search_ops
+ else:
+ from tensorflow.contrib.seq2seq.python.ops import beam_search_ops
+ from tensorflow.python.ops import lookup_ops as lookup
+ from tensorflow.python.framework import tensor_util
+ from tensorflow.python.platform import gfile
+
+ sess = tf.Session()
+ with gfile.FastGFile(pb_model, 'rb') as f:
+ graph_def = tf.GraphDef()
+ graph_def.ParseFromString(f.read())
+ sess.graph.as_default()
+ tf.import_graph_def(graph_def, name='')
+
+ var_dict = dict()
+ for node in sess.graph_def.node:
+ if node.op == 'Const':
+ value = tensor_util.MakeNdarray(node.attr['value'].tensor)
+ if len(value.shape) >= 1:
+ var_dict[node.name] = value
+ return var_dict
+
+def load_tf_dict(pb_model):
+ if "model.ckpt-" in pb_model:
+ var_dict = load_ckpt(pb_model)
+ else:
+ var_dict = load_tf_pb_dict(pb_model)
+ return var_dict
diff --git a/funasr/modules/streaming_utils/utils.py b/funasr/modules/streaming_utils/utils.py
index dd76de9..4bb9d4f 100644
--- a/funasr/modules/streaming_utils/utils.py
+++ b/funasr/modules/streaming_utils/utils.py
@@ -1,6 +1,7 @@
+import os
import torch
from torch.nn import functional as F
-
+import yaml
import numpy as np
def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
@@ -45,3 +46,46 @@
outputs += inputs
return outputs
+
+def proc_tf_vocab(vocab_path):
+ with open(vocab_path, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
+ if '<unk>' not in token_list:
+ token_list.append('<unk>')
+ return token_list
+
+
+def gen_config_for_tfmodel(config_path, vocab_path, output_dir):
+ token_list = proc_tf_vocab(vocab_path)
+ with open(config_path, encoding="utf-8") as f:
+ config = yaml.safe_load(f)
+
+ config['token_list'] = token_list
+
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ with open(os.path.join(output_dir, "config.yaml"), "w", encoding="utf-8") as f:
+ yaml_no_alias_safe_dump(config, f, indent=4, sort_keys=False)
+
+
+class NoAliasSafeDumper(yaml.SafeDumper):
+ # Disable anchor/alias in yaml because looks ugly
+ def ignore_aliases(self, data):
+ return True
+
+
+def yaml_no_alias_safe_dump(data, stream=None, **kwargs):
+ """Safe-dump in yaml with no anchor/alias"""
+ return yaml.dump(
+ data, stream, allow_unicode=True, Dumper=NoAliasSafeDumper, **kwargs
+ )
+
+
+if __name__ == '__main__':
+ import sys
+
+ config_path = sys.argv[1]
+ vocab_path = sys.argv[2]
+ output_dir = sys.argv[3]
+ gen_config_for_tfmodel(config_path, vocab_path, output_dir)
\ No newline at end of file
diff --git a/funasr/modules/subsampling.py b/funasr/modules/subsampling.py
index f9a1c16..d492ccf 100644
--- a/funasr/modules/subsampling.py
+++ b/funasr/modules/subsampling.py
@@ -5,12 +5,12 @@
# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
"""Subsampling layer definition."""
-
+import numpy as np
import torch
import torch.nn.functional as F
from funasr.modules.embedding import PositionalEncoding
-
-
+import logging
+from funasr.modules.streaming_utils.utils import sequence_mask
class TooShortUttError(Exception):
"""Raised when the utt is too short for subsampling.
@@ -87,6 +87,72 @@
if x_mask is None:
return x, None
return x, x_mask[:, :, :-2:2][:, :, :-2:2]
+
+ def __getitem__(self, key):
+ """Get item.
+
+ When reset_parameters() is called, if use_scaled_pos_enc is used,
+ return the positioning encoding.
+
+ """
+ if key != -1:
+ raise NotImplementedError("Support only `-1` (for `reset_parameters`).")
+ return self.out[key]
+
+class Conv2dSubsamplingPad(torch.nn.Module):
+ """Convolutional 2D subsampling (to 1/4 length).
+
+ Args:
+ idim (int): Input dimension.
+ odim (int): Output dimension.
+ dropout_rate (float): Dropout rate.
+ pos_enc (torch.nn.Module): Custom position encoding layer.
+
+ """
+
+ def __init__(self, idim, odim, dropout_rate, pos_enc=None):
+ """Construct an Conv2dSubsampling object."""
+ super(Conv2dSubsamplingPad, self).__init__()
+ self.conv = torch.nn.Sequential(
+ torch.nn.Conv2d(1, odim, 3, 2, padding=(0, 0)),
+ torch.nn.ReLU(),
+ torch.nn.Conv2d(odim, odim, 3, 2, padding=(0, 0)),
+ torch.nn.ReLU(),
+ )
+ self.out = torch.nn.Sequential(
+ torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim),
+ pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate),
+ )
+ self.pad_fn = torch.nn.ConstantPad1d((0, 4), 0.0)
+
+ def forward(self, x, x_mask):
+ """Subsample x.
+
+ Args:
+ x (torch.Tensor): Input tensor (#batch, time, idim).
+ x_mask (torch.Tensor): Input mask (#batch, 1, time).
+
+ Returns:
+ torch.Tensor: Subsampled tensor (#batch, time', odim),
+ where time' = time // 4.
+ torch.Tensor: Subsampled mask (#batch, 1, time'),
+ where time' = time // 4.
+
+ """
+ x = x.transpose(1, 2)
+ x = self.pad_fn(x)
+ x = x.transpose(1, 2)
+ x = x.unsqueeze(1) # (b, c, t, f)
+ x = self.conv(x)
+ b, c, t, f = x.size()
+ x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f))
+ if x_mask is None:
+ return x, None
+ x_len = torch.sum(x_mask[:, 0, :], dim=-1)
+ x_len = (x_len - 1) // 2 + 1
+ x_len = (x_len - 1) // 2 + 1
+ mask = sequence_mask(x_len, None, x_len.dtype, x[0].device)
+ return x, mask[:, None, :]
def __getitem__(self, key):
"""Get item.
@@ -267,12 +333,17 @@
"""
- def __init__(self, idim, odim, kernel_size, stride, pad):
+ def __init__(self, idim, odim, kernel_size, stride, pad,
+ tf2torch_tensor_name_prefix_torch: str = "stride_conv",
+ tf2torch_tensor_name_prefix_tf: str = "seq2seq/proj_encoder/downsampling",
+ ):
super(Conv1dSubsampling, self).__init__()
self.conv = torch.nn.Conv1d(idim, odim, kernel_size, stride)
self.pad_fn = torch.nn.ConstantPad1d(pad, 0.0)
self.stride = stride
self.odim = odim
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
return self.odim
@@ -292,13 +363,47 @@
x_len = (x_len - 1) // self.stride + 1
return x, x_len
- def __getitem__(self, key):
- """Get item.
+ def gen_tf2torch_map_dict(self):
+ tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
+ tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
+ map_dict_local = {
+ ## predictor
+ "{}.conv.weight".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d/kernel".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": (2, 1, 0),
+ }, # (256,256,3),(3,256,256)
+ "{}.conv.bias".format(tensor_name_prefix_torch):
+ {"name": "{}/conv1d/bias".format(tensor_name_prefix_tf),
+ "squeeze": None,
+ "transpose": None,
+ }, # (256,),(256,)
+ }
+ return map_dict_local
- When reset_parameters() is called, if use_scaled_pos_enc is used,
- return the positioning encoding.
+ def convert_tf2torch(self,
+ var_dict_tf,
+ var_dict_torch,
+ ):
+
+ map_dict = self.gen_tf2torch_map_dict()
+
+ var_dict_torch_update = dict()
+ for name in sorted(var_dict_torch.keys(), reverse=False):
+ names = name.split('.')
+ if names[0] == self.tf2torch_tensor_name_prefix_torch:
+ name_tf = map_dict[name]["name"]
+ data_tf = var_dict_tf[name_tf]
+ if map_dict[name]["squeeze"] is not None:
+ data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
+ if map_dict[name]["transpose"] is not None:
+ data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
+ data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
+
+ var_dict_torch_update[name] = data_tf
+
+ logging.info(
+ "torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
+ var_dict_tf[name_tf].shape))
+ return var_dict_torch_update
- """
- if key != -1:
- raise NotImplementedError("Support only `-1` (for `reset_parameters`).")
- return self.out[key]
diff --git a/egs_modelscope/common/utils/__init__.py b/funasr/punctuation/__init__.py
similarity index 100%
copy from egs_modelscope/common/utils/__init__.py
copy to funasr/punctuation/__init__.py
diff --git a/funasr/punctuation/abs_model.py b/funasr/punctuation/abs_model.py
new file mode 100644
index 0000000..3e0b6ce
--- /dev/null
+++ b/funasr/punctuation/abs_model.py
@@ -0,0 +1,29 @@
+from abc import ABC
+from abc import abstractmethod
+from typing import Tuple
+
+import torch
+
+from funasr.modules.scorers.scorer_interface import BatchScorerInterface
+
+
+class AbsPunctuation(torch.nn.Module, BatchScorerInterface, ABC):
+ """The abstract class
+
+ To share the loss calculation way among different models,
+ We uses delegate pattern here:
+ The instance of this class should be passed to "LanguageModel"
+
+ >>> from funasr.punctuation.abs_model import AbsPunctuation
+ >>> punc = AbsPunctuation()
+ >>> model = ESPnetPunctuationModel(punc=punc)
+
+ This "model" is one of mediator objects for "Task" class.
+
+ """
+
+ @abstractmethod
+ def forward(
+ self, input: torch.Tensor, hidden: torch.Tensor
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ raise NotImplementedError
diff --git a/funasr/punctuation/espnet_model.py b/funasr/punctuation/espnet_model.py
new file mode 100644
index 0000000..ded1012
--- /dev/null
+++ b/funasr/punctuation/espnet_model.py
@@ -0,0 +1,152 @@
+from typing import Dict
+from typing import Optional
+from typing import Tuple
+
+import torch
+import torch.nn.functional as F
+from typeguard import check_argument_types
+
+from funasr.modules.nets_utils import make_pad_mask
+from funasr.punctuation.abs_model import AbsPunctuation
+from funasr.torch_utils.device_funcs import force_gatherable
+from funasr.train.abs_espnet_model import AbsESPnetModel
+
+
+class ESPnetPunctuationModel(AbsESPnetModel):
+ def __init__(self, punc_model: AbsPunctuation, vocab_size: int, ignore_id: int = 0):
+ assert check_argument_types()
+ super().__init__()
+ self.punc_model = punc_model
+ self.sos = 1
+ self.eos = 2
+
+ # ignore_id may be assumed as 0, shared with CTC-blank symbol for ASR.
+ self.ignore_id = ignore_id
+
+ def nll(
+ self,
+ text: torch.Tensor,
+ punc: torch.Tensor,
+ text_lengths: torch.Tensor,
+ punc_lengths: torch.Tensor,
+ max_length: Optional[int] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Compute negative log likelihood(nll)
+
+ Normally, this function is called in batchify_nll.
+ Args:
+ text: (Batch, Length)
+ punc: (Batch, Length)
+ text_lengths: (Batch,)
+ max_lengths: int
+ """
+ batch_size = text.size(0)
+ # For data parallel
+ if max_length is None:
+ text = text[:, : text_lengths.max()]
+ punc = punc[:, : text_lengths.max()]
+ else:
+ text = text[:, :max_length]
+ punc = punc[:, :max_length]
+ # 1. Create a sentence pair like '<sos> w1 w2 w3' and 'w1 w2 w3 <eos>'
+ # text: (Batch, Length) -> x, y: (Batch, Length + 1)
+ #x = F.pad(text, [1, 0], "constant", self.eos)
+ #t = F.pad(text, [0, 1], "constant", self.ignore_id)
+ #for i, l in enumerate(text_lengths):
+ # t[i, l] = self.sos
+ #x_lengths = text_lengths + 1
+
+ # 2. Forward Language model
+ # x: (Batch, Length) -> y: (Batch, Length, NVocab)
+ y, _ = self.punc_model(text, text_lengths)
+
+ # 3. Calc negative log likelihood
+ # nll: (BxL,)
+ if self.training == False:
+ _, indices = y.view(-1, y.shape[-1]).topk(1,dim=1)
+ from sklearn.metrics import f1_score
+ f1_score = f1_score(punc.view(-1).detach().cpu().numpy(), indices.squeeze(-1).detach().cpu().numpy(), average='micro')
+ nll = torch.Tensor([f1_score]).repeat(text_lengths.sum())
+ return nll, text_lengths
+ else:
+ nll = F.cross_entropy(y.view(-1, y.shape[-1]), punc.view(-1), reduction="none", ignore_index=self.ignore_id)
+ # nll: (BxL,) -> (BxL,)
+ if max_length is None:
+ nll.masked_fill_(make_pad_mask(text_lengths).to(nll.device).view(-1), 0.0)
+ else:
+ nll.masked_fill_(
+ make_pad_mask(text_lengths, maxlen=max_length + 1).to(nll.device).view(-1),
+ 0.0,
+ )
+ # nll: (BxL,) -> (B, L)
+ nll = nll.view(batch_size, -1)
+ return nll, text_lengths
+
+ def batchify_nll(
+ self,
+ text: torch.Tensor,
+ punc: torch.Tensor,
+ text_lengths: torch.Tensor,
+ punc_lengths: torch.Tensor,
+ batch_size: int = 100
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Compute negative log likelihood(nll) from transformer language model
+
+ To avoid OOM, this fuction seperate the input into batches.
+ Then call nll for each batch and combine and return results.
+ Args:
+ text: (Batch, Length)
+ punc: (Batch, Length)
+ text_lengths: (Batch,)
+ batch_size: int, samples each batch contain when computing nll,
+ you may change this to avoid OOM or increase
+
+ """
+ total_num = text.size(0)
+ if total_num <= batch_size:
+ nll, x_lengths = self.nll(text, punc, text_lengths)
+ else:
+ nlls = []
+ x_lengths = []
+ max_length = text_lengths.max()
+
+ start_idx = 0
+ while True:
+ end_idx = min(start_idx + batch_size, total_num)
+ batch_text = text[start_idx:end_idx, :]
+ batch_punc = punc[start_idx:end_idx, :]
+ batch_text_lengths = text_lengths[start_idx:end_idx]
+ # batch_nll: [B * T]
+ batch_nll, batch_x_lengths = self.nll(
+ batch_text, batch_punc, batch_text_lengths, max_length=max_length
+ )
+ nlls.append(batch_nll)
+ x_lengths.append(batch_x_lengths)
+ start_idx = end_idx
+ if start_idx == total_num:
+ break
+ nll = torch.cat(nlls)
+ x_lengths = torch.cat(x_lengths)
+ assert nll.size(0) == total_num
+ assert x_lengths.size(0) == total_num
+ return nll, x_lengths
+
+ def forward(
+ self, text: torch.Tensor, punc: torch.Tensor, text_lengths: torch.Tensor, punc_lengths: torch.Tensor
+ ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
+ nll, y_lengths = self.nll(text, punc, text_lengths, punc_lengths)
+ ntokens = y_lengths.sum()
+ loss = nll.sum() / ntokens
+ stats = dict(loss=loss.detach())
+
+ # force_gatherable: to-device and to-tensor if scalar for DataParallel
+ loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device)
+ return loss, stats, weight
+
+ def collect_feats(
+ self, text: torch.Tensor, punc: torch.Tensor, text_lengths: torch.Tensor
+ ) -> Dict[str, torch.Tensor]:
+ return {}
+
+ def inference(self, text: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]:
+ return self.punc_model(text, text_lengths)
diff --git a/funasr/punctuation/target_delay_transformer.py b/funasr/punctuation/target_delay_transformer.py
new file mode 100644
index 0000000..66025cb
--- /dev/null
+++ b/funasr/punctuation/target_delay_transformer.py
@@ -0,0 +1,137 @@
+from typing import Any
+from typing import List
+from typing import Tuple
+
+import torch
+import torch.nn as nn
+
+from funasr.modules.embedding import PositionalEncoding
+from funasr.modules.embedding import SinusoidalPositionEncoder
+#from funasr.models.encoder.transformer_encoder import TransformerEncoder as Encoder
+from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder
+#from funasr.modules.mask import subsequent_n_mask
+from funasr.punctuation.abs_model import AbsPunctuation
+
+
+class TargetDelayTransformer(AbsPunctuation):
+ def __init__(
+ self,
+ vocab_size: int,
+ punc_size: int,
+ pos_enc: str = None,
+ embed_unit: int = 128,
+ att_unit: int = 256,
+ head: int = 2,
+ unit: int = 1024,
+ layer: int = 4,
+ dropout_rate: float = 0.5,
+ ):
+ super().__init__()
+ if pos_enc == "sinusoidal":
+# pos_enc_class = PositionalEncoding
+ pos_enc_class = SinusoidalPositionEncoder
+ elif pos_enc is None:
+
+ def pos_enc_class(*args, **kwargs):
+ return nn.Sequential() # indentity
+
+ else:
+ raise ValueError(f"unknown pos-enc option: {pos_enc}")
+
+ self.embed = nn.Embedding(vocab_size, embed_unit)
+ self.encoder = Encoder(
+ input_size=embed_unit,
+ output_size=att_unit,
+ attention_heads=head,
+ linear_units=unit,
+ num_blocks=layer,
+ dropout_rate=dropout_rate,
+ input_layer="pe",
+ # pos_enc_class=pos_enc_class,
+ padding_idx=0,
+ )
+ self.decoder = nn.Linear(att_unit, punc_size)
+
+# def _target_mask(self, ys_in_pad):
+# ys_mask = ys_in_pad != 0
+# m = subsequent_n_mask(ys_mask.size(-1), 5, device=ys_mask.device).unsqueeze(0)
+# return ys_mask.unsqueeze(-2) & m
+
+
+ def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]:
+ """Compute loss value from buffer sequences.
+
+ Args:
+ input (torch.Tensor): Input ids. (batch, len)
+ hidden (torch.Tensor): Target ids. (batch, len)
+
+ """
+ x = self.embed(input)
+ # mask = self._target_mask(input)
+ h, _, _ = self.encoder(x, text_lengths)
+ y = self.decoder(h)
+ return y, None
+
+ def score(
+ self, y: torch.Tensor, state: Any, x: torch.Tensor
+ ) -> Tuple[torch.Tensor, Any]:
+ """Score new token.
+
+ Args:
+ y (torch.Tensor): 1D torch.int64 prefix tokens.
+ state: Scorer state for prefix tokens
+ x (torch.Tensor): encoder feature that generates ys.
+
+ Returns:
+ tuple[torch.Tensor, Any]: Tuple of
+ torch.float32 scores for next token (vocab_size)
+ and next state for ys
+
+ """
+ y = y.unsqueeze(0)
+ h, _, cache = self.encoder.forward_one_step(
+ self.embed(y), self._target_mask(y), cache=state
+ )
+ h = self.decoder(h[:, -1])
+ logp = h.log_softmax(dim=-1).squeeze(0)
+ return logp, cache
+
+ def batch_score(
+ self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor
+ ) -> Tuple[torch.Tensor, List[Any]]:
+ """Score new token batch.
+
+ Args:
+ ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen).
+ states (List[Any]): Scorer states for prefix tokens.
+ xs (torch.Tensor):
+ The encoder feature that generates ys (n_batch, xlen, n_feat).
+
+ Returns:
+ tuple[torch.Tensor, List[Any]]: Tuple of
+ batchfied scores for next token with shape of `(n_batch, vocab_size)`
+ and next state list for ys.
+
+ """
+ # merge states
+ n_batch = len(ys)
+ n_layers = len(self.encoder.encoders)
+ if states[0] is None:
+ batch_state = None
+ else:
+ # transpose state of [batch, layer] into [layer, batch]
+ batch_state = [
+ torch.stack([states[b][i] for b in range(n_batch)])
+ for i in range(n_layers)
+ ]
+
+ # batch decoding
+ h, _, states = self.encoder.forward_one_step(
+ self.embed(ys), self._target_mask(ys), cache=batch_state
+ )
+ h = self.decoder(h[:, -1])
+ logp = h.log_softmax(dim=-1)
+
+ # transpose state of [layer, batch] into [batch, layer]
+ state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)]
+ return logp, state_list
diff --git a/funasr/punctuation/text_preprocessor.py b/funasr/punctuation/text_preprocessor.py
new file mode 100644
index 0000000..3d2c19e
--- /dev/null
+++ b/funasr/punctuation/text_preprocessor.py
@@ -0,0 +1,33 @@
+def split_words(text: str):
+ words = []
+ segs = text.split()
+ for seg in segs:
+ # There is no space in seg.
+ current_word = ""
+ for c in seg:
+ if len(c.encode()) == 1:
+ # This is an ASCII char.
+ current_word += c
+ else:
+ # This is a Chinese char.
+ if len(current_word) > 0:
+ words.append(current_word)
+ current_word = ""
+ words.append(c)
+ if len(current_word) > 0:
+ words.append(current_word)
+ return words
+
+
+def split_to_mini_sentence(words: list, word_limit: int = 20):
+ assert word_limit > 1
+ if len(words) <= word_limit:
+ return [words]
+ sentences = []
+ length = len(words)
+ sentence_len = length // word_limit
+ for i in range(sentence_len):
+ sentences.append(words[i * word_limit:(i + 1) * word_limit])
+ if length % word_limit > 0:
+ sentences.append(words[sentence_len * word_limit:])
+ return sentences
diff --git a/funasr/samplers/build_batch_sampler.py b/funasr/samplers/build_batch_sampler.py
index edda6ba..074b446 100644
--- a/funasr/samplers/build_batch_sampler.py
+++ b/funasr/samplers/build_batch_sampler.py
@@ -1,4 +1,5 @@
from typing import List
+from typing import Dict
from typing import Sequence
from typing import Tuple
from typing import Union
@@ -78,7 +79,7 @@
type: str,
batch_size: int,
batch_bins: int,
- shape_files: Union[Tuple[str, ...], List[str]],
+ shape_files: Union[Tuple[str, ...], List[str], Dict],
sort_in_batch: str = "descending",
sort_batch: str = "ascending",
drop_last: bool = False,
diff --git a/funasr/samplers/length_batch_sampler.py b/funasr/samplers/length_batch_sampler.py
index cdf0e58..8ee8bdc 100644
--- a/funasr/samplers/length_batch_sampler.py
+++ b/funasr/samplers/length_batch_sampler.py
@@ -1,5 +1,6 @@
from typing import Iterator
from typing import List
+from typing import Dict
from typing import Tuple
from typing import Union
@@ -13,7 +14,7 @@
def __init__(
self,
batch_bins: int,
- shape_files: Union[Tuple[str, ...], List[str]],
+ shape_files: Union[Tuple[str, ...], List[str], Dict],
min_batch_size: int = 1,
sort_in_batch: str = "descending",
sort_batch: str = "ascending",
@@ -40,9 +41,12 @@
# utt2shape: (Length, ...)
# uttA 100,...
# uttB 201,...
- utt2shapes = [
- load_num_sequence_text(s, loader_type="csv_int") for s in shape_files
- ]
+ if isinstance(shape_files, dict):
+ utt2shapes = [shape_files]
+ else:
+ utt2shapes = [
+ load_num_sequence_text(s, loader_type="csv_int") for s in shape_files
+ ]
first_utt2shape = utt2shapes[0]
for s, d in zip(shape_files, utt2shapes):
diff --git a/funasr/tasks/abs_task.py b/funasr/tasks/abs_task.py
index d716423..5424f13 100644
--- a/funasr/tasks/abs_task.py
+++ b/funasr/tasks/abs_task.py
@@ -25,6 +25,7 @@
import humanfriendly
import numpy as np
import torch
+import torch.distributed as dist
import torch.multiprocessing
import torch.nn
import torch.optim
@@ -38,7 +39,6 @@
from funasr.datasets.dataset import DATA_TYPES
from funasr.datasets.dataset import ESPnetDataset
from funasr.datasets.iterable_dataset import IterableESPnetDataset
-from funasr.datasets.iterable_dataset_modelscope import IterableESPnetDatasetModelScope, IterableESPnetBytesModelScope
from funasr.iterators.abs_iter_factory import AbsIterFactory
from funasr.iterators.chunk_iter_factory import ChunkIterFactory
from funasr.iterators.multiple_iter_factory import MultipleIterFactory
@@ -68,6 +68,7 @@
from funasr.utils.types import str2triple_str
from funasr.utils.types import str_or_int
from funasr.utils.types import str_or_none
+from funasr.utils.wav_utils import calc_shape, generate_data_list
from funasr.utils.yaml_no_alias_safe_dump import yaml_no_alias_safe_dump
try:
@@ -182,6 +183,7 @@
num_optimizers: int = 1
trainer = Trainer
class_choices_list: List[ClassChoices] = []
+ finetune_args: None
def __init__(self):
raise RuntimeError("This class can't be instantiated.")
@@ -279,7 +281,7 @@
# NOTE(kamo): add_arguments(..., required=True) can't be used
# to provide --print_config mode. Instead of it, do as
- parser.set_defaults(required=["output_dir"])
+ # parser.set_defaults(required=["output_dir"])
group = parser.add_argument_group("Common configuration")
@@ -696,7 +698,7 @@
group.add_argument(
"--batch_type",
type=str,
- default="folded",
+ default="length",
choices=list(BATCH_TYPES),
help=_batch_type_help,
)
@@ -706,6 +708,18 @@
default=None,
choices=list(BATCH_TYPES) + [None],
help="If not given, the value of --batch_type is used",
+ )
+ group.add_argument(
+ "--speech_length_min",
+ type=int,
+ default=-1,
+ help="speech length min",
+ )
+ group.add_argument(
+ "--speech_length_max",
+ type=int,
+ default=-1,
+ help="speech length max",
)
group.add_argument("--fold_length", type=int, action="append", default=[])
group.add_argument(
@@ -878,6 +892,11 @@
help="flag to indicate whether training on PAI",
)
group.add_argument(
+ "--simple_ddp",
+ type=str2bool,
+ default=False,
+ )
+ group.add_argument(
"--num_worker_count",
type=int,
default=1,
@@ -1005,29 +1024,30 @@
@classmethod
def check_required_command_args(cls, args: argparse.Namespace):
assert check_argument_types()
- for k in vars(args):
- if "-" in k:
- raise RuntimeError(f'Use "_" instead of "-": parser.get_parser("{k}")')
+ if hasattr(args, "required"):
+ for k in vars(args):
+ if "-" in k:
+ raise RuntimeError(f'Use "_" instead of "-": parser.get_parser("{k}")')
- required = ", ".join(
- f"--{a}" for a in args.required if getattr(args, a) is None
- )
-
- if len(required) != 0:
- parser = cls.get_parser()
- parser.print_help(file=sys.stderr)
- p = Path(sys.argv[0]).name
- print(file=sys.stderr)
- print(
- f"{p}: error: the following arguments are required: " f"{required}",
- file=sys.stderr,
+ required = ", ".join(
+ f"--{a}" for a in args.required if getattr(args, a) is None
)
- sys.exit(2)
+
+ if len(required) != 0:
+ parser = cls.get_parser()
+ parser.print_help(file=sys.stderr)
+ p = Path(sys.argv[0]).name
+ print(file=sys.stderr)
+ print(
+ f"{p}: error: the following arguments are required: " f"{required}",
+ file=sys.stderr,
+ )
+ sys.exit(2)
@classmethod
def check_task_requirements(
cls,
- dataset: Union[AbsDataset, IterableESPnetDataset, IterableESPnetDatasetModelScope, IterableESPnetBytesModelScope],
+ dataset: Union[AbsDataset, IterableESPnetDataset],
allow_variable_data_keys: bool,
train: bool,
inference: bool = False,
@@ -1087,6 +1107,22 @@
cls.main_worker(args)
@classmethod
+ def run(cls):
+ assert hasattr(cls, "finetune_args")
+ args = cls.finetune_args
+ args.train_shape_file = None
+ if args.distributed:
+ args.simple_ddp = True
+ else:
+ args.simple_ddp = False
+ args.ngpu = 1
+ args.use_pai = False
+ args.batch_type = "length"
+ args.oss_bucket = None
+ args.input_size = None
+ cls.main_worker(args)
+
+ @classmethod
def main_worker(cls, args: argparse.Namespace):
assert check_argument_types()
@@ -1095,8 +1131,40 @@
# Setting distributed_option.dist_rank, etc.
if args.use_pai:
distributed_option.init_options_pai()
- else:
+ elif not args.simple_ddp:
distributed_option.init_options()
+
+ # Invoking torch.distributed.init_process_group
+ if args.use_pai:
+ distributed_option.init_torch_distributed_pai(args)
+ elif not args.simple_ddp:
+ distributed_option.init_torch_distributed(args)
+ elif args.distributed and args.simple_ddp:
+ distributed_option.init_torch_distributed_pai(args)
+ args.ngpu = dist.get_world_size()
+ if args.dataset_type == "small":
+ if args.batch_size is not None:
+ args.batch_size = args.batch_size * args.ngpu
+ if args.batch_bins is not None:
+ args.batch_bins = args.batch_bins * args.ngpu
+
+ if args.train_shape_file is None and args.dataset_type == "small":
+ if not args.simple_ddp or distributed_option.dist_rank == 0:
+ calc_shape(args.data_dir, args.train_set, args.frontend_conf, args.speech_length_min, args.speech_length_max)
+ calc_shape(args.data_dir, args.dev_set, args.frontend_conf, args.speech_length_min, args.speech_length_max)
+ if args.simple_ddp:
+ dist.barrier()
+ args.train_shape_file = [os.path.join(args.data_dir, args.train_set, "speech_shape")]
+ args.valid_shape_file = [os.path.join(args.data_dir, args.dev_set, "speech_shape")]
+
+ if args.train_data_file is None and args.dataset_type == "large":
+ if not args.simple_ddp or distributed_option.dist_rank == 0:
+ generate_data_list(args.data_dir, args.train_set)
+ generate_data_list(args.data_dir, args.dev_set)
+ if args.simple_ddp:
+ dist.barrier()
+ args.train_data_file = os.path.join(args.data_dir, args.train_set, "data.list")
+ args.valid_data_file = os.path.join(args.data_dir, args.dev_set, "data.list")
# NOTE(kamo): Don't use logging before invoking logging.basicConfig()
if not distributed_option.distributed or distributed_option.dist_rank == 0:
@@ -1124,11 +1192,9 @@
format=f"[{os.uname()[1].split('.')[0]}]"
f" %(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
)
- # Invoking torch.distributed.init_process_group
- if args.use_pai:
- distributed_option.init_torch_distributed_pai(args)
- else:
- distributed_option.init_torch_distributed(args)
+ logging.info("world size: {}, rank: {}, local_rank: {}".format(distributed_option.dist_world_size,
+ distributed_option.dist_rank,
+ distributed_option.local_rank))
# 1. Set random-seed
set_all_random_seed(args.seed)
@@ -1222,10 +1288,14 @@
# 7. Build iterator factories
if args.dataset_type == "large":
from funasr.datasets.large_datasets.build_dataloader import ArkDataLoader
- train_iter_factory = ArkDataLoader(args.train_data_file, args.token_list,
- args.config, mode="train")
- valid_iter_factory = ArkDataLoader(args.valid_data_file, args.token_list,
- args.config, mode="eval")
+ train_iter_factory = ArkDataLoader(args.train_data_file, args.token_list, args.dataset_conf,
+ seg_dict_file=args.seg_dict_file if hasattr(args,
+ "seg_dict_file") else None,
+ mode="train")
+ valid_iter_factory = ArkDataLoader(args.valid_data_file, args.token_list, args.dataset_conf,
+ seg_dict_file=args.seg_dict_file if hasattr(args,
+ "seg_dict_file") else None,
+ mode="eval")
elif args.dataset_type == "small":
train_iter_factory = cls.build_iter_factory(
args=args,
@@ -1749,70 +1819,13 @@
**kwargs,
)
- @classmethod
- def build_streaming_iterator_modelscope(
- cls,
- data_path_and_name_and_type,
- preprocess_fn,
- collate_fn,
- key_file: str = None,
- batch_size: int = 1,
- dtype: str = np.float32,
- num_workers: int = 1,
- allow_variable_data_keys: bool = False,
- ngpu: int = 0,
- inference: bool = False,
- sample_rate: Union[dict, int] = 16000
- ) -> DataLoader:
- """Build DataLoader using iterable dataset"""
- assert check_argument_types()
- # For backward compatibility for pytorch DataLoader
- if collate_fn is not None:
- kwargs = dict(collate_fn=collate_fn)
- else:
- kwargs = {}
-
- audio_data = data_path_and_name_and_type[0]
- if isinstance(audio_data, bytes):
- dataset = IterableESPnetBytesModelScope(
- data_path_and_name_and_type,
- float_dtype=dtype,
- preprocess=preprocess_fn,
- key_file=key_file,
- sample_rate=sample_rate
- )
- else:
- dataset = IterableESPnetDatasetModelScope(
- data_path_and_name_and_type,
- float_dtype=dtype,
- preprocess=preprocess_fn,
- key_file=key_file,
- sample_rate=sample_rate
- )
-
- if dataset.apply_utt2category:
- kwargs.update(batch_size=1)
- else:
- kwargs.update(batch_size=batch_size)
-
- cls.check_task_requirements(dataset,
- allow_variable_data_keys,
- train=False,
- inference=inference)
-
- return DataLoader(
- dataset=dataset,
- pin_memory=ngpu > 0,
- num_workers=num_workers,
- **kwargs,
- )
-
# ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~
@classmethod
def build_model_from_file(
cls,
config_file: Union[Path, str] = None,
model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
device: str = "cpu",
) -> Tuple[AbsESPnetModel, argparse.Namespace]:
"""Build model from the files.
@@ -1837,6 +1850,8 @@
with config_file.open("r", encoding="utf-8") as f:
args = yaml.safe_load(f)
+ if cmvn_file is not None:
+ args["cmvn_file"] = cmvn_file
args = argparse.Namespace(**args)
model = cls.build_model(args)
if not isinstance(model, AbsESPnetModel):
@@ -1850,5 +1865,5 @@
# in PyTorch<=1.4
device = f"cuda:{torch.cuda.current_device()}"
model.load_state_dict(torch.load(model_file, map_location=device))
-
+ model.to(device)
return model, args
diff --git a/funasr/tasks/asr.py b/funasr/tasks/asr.py
index 9367ed3..1b7f152 100644
--- a/funasr/tasks/asr.py
+++ b/funasr/tasks/asr.py
@@ -1,54 +1,69 @@
import argparse
import logging
+import os
+from pathlib import Path
from typing import Callable
from typing import Collection
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
+from typing import Union
import numpy as np
import torch
+import yaml
from typeguard import check_argument_types
from typeguard import check_return_type
from funasr.datasets.collate_fn import CommonCollateFn
from funasr.datasets.preprocessor import CommonPreprocessor
+from funasr.layers.abs_normalize import AbsNormalize
+from funasr.layers.global_mvn import GlobalMVN
+from funasr.layers.utterance_mvn import UtteranceMVN
from funasr.models.ctc import CTC
from funasr.models.decoder.abs_decoder import AbsDecoder
from funasr.models.decoder.rnn_decoder import RNNDecoder
+from funasr.models.decoder.sanm_decoder import ParaformerSANMDecoder, FsmnDecoderSCAMAOpt
from funasr.models.decoder.transformer_decoder import (
- DynamicConvolution2DTransformerDecoder, # noqa: H301
+ DynamicConvolution2DTransformerDecoder, # noqa: H301
)
from funasr.models.decoder.transformer_decoder import DynamicConvolutionTransformerDecoder
from funasr.models.decoder.transformer_decoder import (
- LightweightConvolution2DTransformerDecoder, # noqa: H301
+ LightweightConvolution2DTransformerDecoder, # noqa: H301
)
from funasr.models.decoder.transformer_decoder import (
- LightweightConvolutionTransformerDecoder, # noqa: H301
+ LightweightConvolutionTransformerDecoder, # noqa: H301
)
+from funasr.models.decoder.transformer_decoder import ParaformerDecoderSAN
from funasr.models.decoder.transformer_decoder import TransformerDecoder
+from funasr.models.e2e_asr import ESPnetASRModel
+from funasr.models.e2e_asr_paraformer import Paraformer, ParaformerBert, BiCifParaformer
+from funasr.models.e2e_uni_asr import UniASR
from funasr.models.encoder.abs_encoder import AbsEncoder
from funasr.models.encoder.conformer_encoder import ConformerEncoder
+from funasr.models.encoder.data2vec_encoder import Data2VecEncoder
from funasr.models.encoder.rnn_encoder import RNNEncoder
+from funasr.models.encoder.sanm_encoder import SANMEncoder, SANMEncoderChunkOpt
from funasr.models.encoder.transformer_encoder import TransformerEncoder
from funasr.models.frontend.abs_frontend import AbsFrontend
from funasr.models.frontend.default import DefaultFrontend
from funasr.models.frontend.fused import FusedFrontends
from funasr.models.frontend.s3prl import S3prlFrontend
+from funasr.models.frontend.wav_frontend import WavFrontend
from funasr.models.frontend.windowing import SlidingWindow
from funasr.models.postencoder.abs_postencoder import AbsPostEncoder
from funasr.models.postencoder.hugging_face_transformers_postencoder import (
- HuggingFaceTransformersPostEncoder, # noqa: H301
+ HuggingFaceTransformersPostEncoder, # noqa: H301
)
+from funasr.models.predictor.cif import CifPredictor, CifPredictorV2, CifPredictorV3
from funasr.models.preencoder.abs_preencoder import AbsPreEncoder
from funasr.models.preencoder.linear import LinearProjection
from funasr.models.preencoder.sinc import LightweightSincConvs
from funasr.models.specaug.abs_specaug import AbsSpecAug
from funasr.models.specaug.specaug import SpecAug
-from funasr.layers.abs_normalize import AbsNormalize
-from funasr.layers.global_mvn import GlobalMVN
-from funasr.layers.utterance_mvn import UtteranceMVN
+from funasr.models.specaug.specaug import SpecAugLFR
+from funasr.modules.subsampling import Conv1dSubsampling
from funasr.tasks.abs_task import AbsTask
from funasr.text.phoneme_tokenizer import g2p_choices
from funasr.torch_utils.initialize import initialize
@@ -62,818 +77,1026 @@
from funasr.utils.types import str2bool
from funasr.utils.types import str_or_none
-from funasr.models.specaug.specaug import SpecAugLFR
-from funasr.models.predictor.cif import CifPredictor, CifPredictorV2
-from funasr.modules.subsampling import Conv1dSubsampling
-from funasr.models.e2e_asr import ESPnetASRModel
-from funasr.models.e2e_uni_asr import UniASR
-from funasr.models.encoder.sanm_encoder import SANMEncoder, SANMEncoderChunkOpt
-from funasr.models.decoder.sanm_decoder import ParaformerSANMDecoder, FsmnDecoderSCAMAOpt
-from funasr.models.e2e_asr_paraformer import Paraformer, ParaformerBert
-from funasr.models.decoder.transformer_decoder import ParaformerDecoderSAN
-
frontend_choices = ClassChoices(
- name="frontend",
- classes=dict(
- default=DefaultFrontend,
- sliding_window=SlidingWindow,
- s3prl=S3prlFrontend,
- fused=FusedFrontends,
- ),
- type_check=AbsFrontend,
- default="default",
+ name="frontend",
+ classes=dict(
+ default=DefaultFrontend,
+ sliding_window=SlidingWindow,
+ s3prl=S3prlFrontend,
+ fused=FusedFrontends,
+ wav_frontend=WavFrontend,
+ ),
+ type_check=AbsFrontend,
+ default="default",
)
specaug_choices = ClassChoices(
- name="specaug",
- classes=dict(
- specaug=SpecAug,
- specaug_lfr=SpecAugLFR,
- ),
- type_check=AbsSpecAug,
- default=None,
- optional=True,
+ name="specaug",
+ classes=dict(
+ specaug=SpecAug,
+ specaug_lfr=SpecAugLFR,
+ ),
+ type_check=AbsSpecAug,
+ default=None,
+ optional=True,
)
normalize_choices = ClassChoices(
- "normalize",
- classes=dict(
- global_mvn=GlobalMVN,
- utterance_mvn=UtteranceMVN,
- ),
- type_check=AbsNormalize,
- default=None,
- optional=True,
+ "normalize",
+ classes=dict(
+ global_mvn=GlobalMVN,
+ utterance_mvn=UtteranceMVN,
+ ),
+ type_check=AbsNormalize,
+ default=None,
+ optional=True,
)
model_choices = ClassChoices(
- "model",
- classes=dict(
- asr=ESPnetASRModel,
- uniasr=UniASR,
- paraformer=Paraformer,
- paraformer_bert=ParaformerBert,
- ),
- type_check=AbsESPnetModel,
- default="asr",
+ "model",
+ classes=dict(
+ asr=ESPnetASRModel,
+ uniasr=UniASR,
+ paraformer=Paraformer,
+ paraformer_bert=ParaformerBert,
+ bicif_paraformer=BiCifParaformer,
+ ),
+ type_check=AbsESPnetModel,
+ default="asr",
)
preencoder_choices = ClassChoices(
- name="preencoder",
- classes=dict(
- sinc=LightweightSincConvs,
- linear=LinearProjection,
- ),
- type_check=AbsPreEncoder,
- default=None,
- optional=True,
+ name="preencoder",
+ classes=dict(
+ sinc=LightweightSincConvs,
+ linear=LinearProjection,
+ ),
+ type_check=AbsPreEncoder,
+ default=None,
+ optional=True,
)
encoder_choices = ClassChoices(
- "encoder",
- classes=dict(
- conformer=ConformerEncoder,
- transformer=TransformerEncoder,
- rnn=RNNEncoder,
- sanm=SANMEncoder,
- sanm_chunk_opt=SANMEncoderChunkOpt,
- ),
- type_check=AbsEncoder,
- default="rnn",
+ "encoder",
+ classes=dict(
+ conformer=ConformerEncoder,
+ transformer=TransformerEncoder,
+ rnn=RNNEncoder,
+ sanm=SANMEncoder,
+ sanm_chunk_opt=SANMEncoderChunkOpt,
+ data2vec_encoder=Data2VecEncoder,
+ ),
+ type_check=AbsEncoder,
+ default="rnn",
)
encoder_choices2 = ClassChoices(
- "encoder2",
- classes=dict(
- conformer=ConformerEncoder,
- transformer=TransformerEncoder,
- rnn=RNNEncoder,
- sanm=SANMEncoder,
- sanm_chunk_opt=SANMEncoderChunkOpt,
- ),
- type_check=AbsEncoder,
- default="rnn",
+ "encoder2",
+ classes=dict(
+ conformer=ConformerEncoder,
+ transformer=TransformerEncoder,
+ rnn=RNNEncoder,
+ sanm=SANMEncoder,
+ sanm_chunk_opt=SANMEncoderChunkOpt,
+ ),
+ type_check=AbsEncoder,
+ default="rnn",
)
postencoder_choices = ClassChoices(
- name="postencoder",
- classes=dict(
- hugging_face_transformers=HuggingFaceTransformersPostEncoder,
- ),
- type_check=AbsPostEncoder,
- default=None,
- optional=True,
+ name="postencoder",
+ classes=dict(
+ hugging_face_transformers=HuggingFaceTransformersPostEncoder,
+ ),
+ type_check=AbsPostEncoder,
+ default=None,
+ optional=True,
)
decoder_choices = ClassChoices(
- "decoder",
- classes=dict(
- transformer=TransformerDecoder,
- lightweight_conv=LightweightConvolutionTransformerDecoder,
- lightweight_conv2d=LightweightConvolution2DTransformerDecoder,
- dynamic_conv=DynamicConvolutionTransformerDecoder,
- dynamic_conv2d=DynamicConvolution2DTransformerDecoder,
- rnn=RNNDecoder,
- fsmn_scama_opt=FsmnDecoderSCAMAOpt,
- paraformer_decoder_sanm=ParaformerSANMDecoder,
- paraformer_decoder_san=ParaformerDecoderSAN,
- ),
- type_check=AbsDecoder,
- default="rnn",
+ "decoder",
+ classes=dict(
+ transformer=TransformerDecoder,
+ lightweight_conv=LightweightConvolutionTransformerDecoder,
+ lightweight_conv2d=LightweightConvolution2DTransformerDecoder,
+ dynamic_conv=DynamicConvolutionTransformerDecoder,
+ dynamic_conv2d=DynamicConvolution2DTransformerDecoder,
+ rnn=RNNDecoder,
+ fsmn_scama_opt=FsmnDecoderSCAMAOpt,
+ paraformer_decoder_sanm=ParaformerSANMDecoder,
+ paraformer_decoder_san=ParaformerDecoderSAN,
+ ),
+ type_check=AbsDecoder,
+ default="rnn",
)
decoder_choices2 = ClassChoices(
- "decoder2",
- classes=dict(
- transformer=TransformerDecoder,
- lightweight_conv=LightweightConvolutionTransformerDecoder,
- lightweight_conv2d=LightweightConvolution2DTransformerDecoder,
- dynamic_conv=DynamicConvolutionTransformerDecoder,
- dynamic_conv2d=DynamicConvolution2DTransformerDecoder,
- rnn=RNNDecoder,
- fsmn_scama_opt=FsmnDecoderSCAMAOpt,
- paraformer_decoder_sanm=ParaformerSANMDecoder,
- ),
- type_check=AbsDecoder,
- default="rnn",
+ "decoder2",
+ classes=dict(
+ transformer=TransformerDecoder,
+ lightweight_conv=LightweightConvolutionTransformerDecoder,
+ lightweight_conv2d=LightweightConvolution2DTransformerDecoder,
+ dynamic_conv=DynamicConvolutionTransformerDecoder,
+ dynamic_conv2d=DynamicConvolution2DTransformerDecoder,
+ rnn=RNNDecoder,
+ fsmn_scama_opt=FsmnDecoderSCAMAOpt,
+ paraformer_decoder_sanm=ParaformerSANMDecoder,
+ ),
+ type_check=AbsDecoder,
+ default="rnn",
)
predictor_choices = ClassChoices(
- name="predictor",
- classes=dict(
- cif_predictor=CifPredictor,
- ctc_predictor=None,
- cif_predictor_v2=CifPredictorV2,
- ),
- type_check=None,
- default="cif_predictor",
- optional=True,
+ name="predictor",
+ classes=dict(
+ cif_predictor=CifPredictor,
+ ctc_predictor=None,
+ cif_predictor_v2=CifPredictorV2,
+ cif_predictor_v3=CifPredictorV3,
+ ),
+ type_check=None,
+ default="cif_predictor",
+ optional=True,
)
predictor_choices2 = ClassChoices(
- name="predictor2",
- classes=dict(
- cif_predictor=CifPredictor,
- ctc_predictor=None,
- cif_predictor_v2=CifPredictorV2,
- ),
- type_check=None,
- default="cif_predictor",
- optional=True,
+ name="predictor2",
+ classes=dict(
+ cif_predictor=CifPredictor,
+ ctc_predictor=None,
+ cif_predictor_v2=CifPredictorV2,
+ ),
+ type_check=None,
+ default="cif_predictor",
+ optional=True,
)
stride_conv_choices = ClassChoices(
- name="stride_conv",
- classes=dict(
- stride_conv1d=Conv1dSubsampling
- ),
- type_check=None,
- default="stride_conv1d",
- optional=True,
+ name="stride_conv",
+ classes=dict(
+ stride_conv1d=Conv1dSubsampling
+ ),
+ type_check=None,
+ default="stride_conv1d",
+ optional=True,
)
class ASRTask(AbsTask):
- # If you need more than one optimizers, change this value
- num_optimizers: int = 1
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
- # Add variable objects configurations
- class_choices_list = [
- # --frontend and --frontend_conf
- frontend_choices,
- # --specaug and --specaug_conf
- specaug_choices,
- # --normalize and --normalize_conf
- normalize_choices,
- # --model and --model_conf
- model_choices,
- # --preencoder and --preencoder_conf
- preencoder_choices,
- # --encoder and --encoder_conf
- encoder_choices,
- # --postencoder and --postencoder_conf
- postencoder_choices,
- # --decoder and --decoder_conf
- decoder_choices,
- ]
+ # Add variable objects configurations
+ class_choices_list = [
+ # --frontend and --frontend_conf
+ frontend_choices,
+ # --specaug and --specaug_conf
+ specaug_choices,
+ # --normalize and --normalize_conf
+ normalize_choices,
+ # --model and --model_conf
+ model_choices,
+ # --preencoder and --preencoder_conf
+ preencoder_choices,
+ # --encoder and --encoder_conf
+ encoder_choices,
+ # --postencoder and --postencoder_conf
+ postencoder_choices,
+ # --decoder and --decoder_conf
+ decoder_choices,
+ ]
- # If you need to modify train() or eval() procedures, change Trainer class here
- trainer = Trainer
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
- @classmethod
- def add_task_arguments(cls, parser: argparse.ArgumentParser):
- group = parser.add_argument_group(description="Task related")
+ @classmethod
+ def add_task_arguments(cls, parser: argparse.ArgumentParser):
+ group = parser.add_argument_group(description="Task related")
- # NOTE(kamo): add_arguments(..., required=True) can't be used
- # to provide --print_config mode. Instead of it, do as
- required = parser.get_default("required")
- required += ["token_list"]
+ # NOTE(kamo): add_arguments(..., required=True) can't be used
+ # to provide --print_config mode. Instead of it, do as
+ # required = parser.get_default("required")
+ # required += ["token_list"]
- group.add_argument(
- "--token_list",
- type=str_or_none,
- default=None,
- help="A text mapping int-id to token",
- )
- group.add_argument(
- "--split_with_space",
- type=str2bool,
- default=True,
- help="whether to split text using <space>",
- )
- group.add_argument(
- "--init",
- type=lambda x: str_or_none(x.lower()),
- default=None,
- help="The initialization method",
- choices=[
- "chainer",
- "xavier_uniform",
- "xavier_normal",
- "kaiming_uniform",
- "kaiming_normal",
- None,
- ],
- )
+ group.add_argument(
+ "--token_list",
+ type=str_or_none,
+ default=None,
+ help="A text mapping int-id to token",
+ )
+ group.add_argument(
+ "--split_with_space",
+ type=str2bool,
+ default=True,
+ help="whether to split text using <space>",
+ )
+ group.add_argument(
+ "--seg_dict_file",
+ type=str,
+ default=None,
+ help="seg_dict_file for text processing",
+ )
+ group.add_argument(
+ "--init",
+ type=lambda x: str_or_none(x.lower()),
+ default=None,
+ help="The initialization method",
+ choices=[
+ "chainer",
+ "xavier_uniform",
+ "xavier_normal",
+ "kaiming_uniform",
+ "kaiming_normal",
+ None,
+ ],
+ )
- group.add_argument(
- "--input_size",
- type=int_or_none,
- default=None,
- help="The number of input dimension of the feature",
- )
+ group.add_argument(
+ "--input_size",
+ type=int_or_none,
+ default=None,
+ help="The number of input dimension of the feature",
+ )
- group.add_argument(
- "--ctc_conf",
- action=NestedDictAction,
- default=get_default_kwargs(CTC),
- help="The keyword arguments for CTC class.",
- )
- group.add_argument(
- "--joint_net_conf",
- action=NestedDictAction,
- default=None,
- help="The keyword arguments for joint network class.",
- )
+ group.add_argument(
+ "--ctc_conf",
+ action=NestedDictAction,
+ default=get_default_kwargs(CTC),
+ help="The keyword arguments for CTC class.",
+ )
+ group.add_argument(
+ "--joint_net_conf",
+ action=NestedDictAction,
+ default=None,
+ help="The keyword arguments for joint network class.",
+ )
- group = parser.add_argument_group(description="Preprocess related")
- group.add_argument(
- "--use_preprocessor",
- type=str2bool,
- default=True,
- help="Apply preprocessing to data or not",
- )
- group.add_argument(
- "--token_type",
- type=str,
- default="bpe",
- choices=["bpe", "char", "word", "phn"],
- help="The text will be tokenized " "in the specified level token",
- )
- group.add_argument(
- "--bpemodel",
- type=str_or_none,
- default=None,
- help="The model file of sentencepiece",
- )
- parser.add_argument(
- "--non_linguistic_symbols",
- type=str_or_none,
- default=None,
- help="non_linguistic_symbols file path",
- )
- parser.add_argument(
- "--cleaner",
- type=str_or_none,
- choices=[None, "tacotron", "jaconv", "vietnamese"],
- default=None,
- help="Apply text cleaning",
- )
- parser.add_argument(
- "--g2p",
- type=str_or_none,
- choices=g2p_choices,
- default=None,
- help="Specify g2p method if --token_type=phn",
- )
- parser.add_argument(
- "--speech_volume_normalize",
- type=float_or_none,
- default=None,
- help="Scale the maximum amplitude to the given value.",
- )
- parser.add_argument(
- "--rir_scp",
- type=str_or_none,
- default=None,
- help="The file path of rir scp file.",
- )
- parser.add_argument(
- "--rir_apply_prob",
- type=float,
- default=1.0,
- help="THe probability for applying RIR convolution.",
- )
- parser.add_argument(
- "--noise_scp",
- type=str_or_none,
- default=None,
- help="The file path of noise scp file.",
- )
- parser.add_argument(
- "--noise_apply_prob",
- type=float,
- default=1.0,
- help="The probability applying Noise adding.",
- )
- parser.add_argument(
- "--noise_db_range",
- type=str,
- default="13_15",
- help="The range of noise decibel level.",
- )
+ group = parser.add_argument_group(description="Preprocess related")
+ group.add_argument(
+ "--use_preprocessor",
+ type=str2bool,
+ default=True,
+ help="Apply preprocessing to data or not",
+ )
+ group.add_argument(
+ "--token_type",
+ type=str,
+ default="bpe",
+ choices=["bpe", "char", "word", "phn"],
+ help="The text will be tokenized " "in the specified level token",
+ )
+ group.add_argument(
+ "--bpemodel",
+ type=str_or_none,
+ default=None,
+ help="The model file of sentencepiece",
+ )
+ parser.add_argument(
+ "--non_linguistic_symbols",
+ type=str_or_none,
+ default=None,
+ help="non_linguistic_symbols file path",
+ )
+ parser.add_argument(
+ "--cleaner",
+ type=str_or_none,
+ choices=[None, "tacotron", "jaconv", "vietnamese"],
+ default=None,
+ help="Apply text cleaning",
+ )
+ parser.add_argument(
+ "--g2p",
+ type=str_or_none,
+ choices=g2p_choices,
+ default=None,
+ help="Specify g2p method if --token_type=phn",
+ )
+ parser.add_argument(
+ "--speech_volume_normalize",
+ type=float_or_none,
+ default=None,
+ help="Scale the maximum amplitude to the given value.",
+ )
+ parser.add_argument(
+ "--rir_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of rir scp file.",
+ )
+ parser.add_argument(
+ "--rir_apply_prob",
+ type=float,
+ default=1.0,
+ help="THe probability for applying RIR convolution.",
+ )
+ parser.add_argument(
+ "--cmvn_file",
+ type=str_or_none,
+ default=None,
+ help="The file path of noise scp file.",
+ )
+ parser.add_argument(
+ "--noise_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of noise scp file.",
+ )
+ parser.add_argument(
+ "--noise_apply_prob",
+ type=float,
+ default=1.0,
+ help="The probability applying Noise adding.",
+ )
+ parser.add_argument(
+ "--noise_db_range",
+ type=str,
+ default="13_15",
+ help="The range of noise decibel level.",
+ )
- for class_choices in cls.class_choices_list:
- # Append --<name> and --<name>_conf.
- # e.g. --encoder and --encoder_conf
- class_choices.add_arguments(group)
+ for class_choices in cls.class_choices_list:
+ # Append --<name> and --<name>_conf.
+ # e.g. --encoder and --encoder_conf
+ class_choices.add_arguments(group)
- @classmethod
- def build_collate_fn(
- cls, args: argparse.Namespace, train: bool
- ) -> Callable[
- [Collection[Tuple[str, Dict[str, np.ndarray]]]],
- Tuple[List[str], Dict[str, torch.Tensor]],
- ]:
- assert check_argument_types()
- # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol
- return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
+ @classmethod
+ def build_collate_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Callable[
+ [Collection[Tuple[str, Dict[str, np.ndarray]]]],
+ Tuple[List[str], Dict[str, torch.Tensor]],
+ ]:
+ assert check_argument_types()
+ # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol
+ return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
- @classmethod
- def build_preprocess_fn(
- cls, args: argparse.Namespace, train: bool
- ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
- assert check_argument_types()
- if args.use_preprocessor:
- retval = CommonPreprocessor(
- train=train,
- token_type=args.token_type,
- token_list=args.token_list,
- bpemodel=args.bpemodel,
- non_linguistic_symbols=args.non_linguistic_symbols,
- text_cleaner=args.cleaner,
- g2p_type=args.g2p,
- split_with_space=args.split_with_space if hasattr(args, "split_with_space") else False,
- # NOTE(kamo): Check attribute existence for backward compatibility
- rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None,
- rir_apply_prob=args.rir_apply_prob
- if hasattr(args, "rir_apply_prob")
- else 1.0,
- noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None,
- noise_apply_prob=args.noise_apply_prob
- if hasattr(args, "noise_apply_prob")
- else 1.0,
- noise_db_range=args.noise_db_range
- if hasattr(args, "noise_db_range")
- else "13_15",
- speech_volume_normalize=args.speech_volume_normalize
- if hasattr(args, "rir_scp")
- else None,
- )
- else:
- retval = None
- assert check_return_type(retval)
- return retval
+ @classmethod
+ def build_preprocess_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
+ assert check_argument_types()
+ if args.use_preprocessor:
+ retval = CommonPreprocessor(
+ train=train,
+ token_type=args.token_type,
+ token_list=args.token_list,
+ bpemodel=args.bpemodel,
+ non_linguistic_symbols=args.non_linguistic_symbols,
+ text_cleaner=args.cleaner,
+ g2p_type=args.g2p,
+ split_with_space=args.split_with_space if hasattr(args, "split_with_space") else False,
+ seg_dict_file=args.seg_dict_file if hasattr(args, "seg_dict_file") else None,
+ # NOTE(kamo): Check attribute existence for backward compatibility
+ rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None,
+ rir_apply_prob=args.rir_apply_prob
+ if hasattr(args, "rir_apply_prob")
+ else 1.0,
+ noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None,
+ noise_apply_prob=args.noise_apply_prob
+ if hasattr(args, "noise_apply_prob")
+ else 1.0,
+ noise_db_range=args.noise_db_range
+ if hasattr(args, "noise_db_range")
+ else "13_15",
+ speech_volume_normalize=args.speech_volume_normalize
+ if hasattr(args, "rir_scp")
+ else None,
+ )
+ else:
+ retval = None
+ assert check_return_type(retval)
+ return retval
- @classmethod
- def required_data_names(
- cls, train: bool = True, inference: bool = False
- ) -> Tuple[str, ...]:
- if not inference:
- retval = ("speech", "text")
- else:
- # Recognition mode
- retval = ("speech",)
- return retval
+ @classmethod
+ def required_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ if not inference:
+ retval = ("speech", "text")
+ else:
+ # Recognition mode
+ retval = ("speech",)
+ return retval
- @classmethod
- def optional_data_names(
- cls, train: bool = True, inference: bool = False
- ) -> Tuple[str, ...]:
- retval = ()
- assert check_return_type(retval)
- return retval
+ @classmethod
+ def optional_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ retval = ()
+ assert check_return_type(retval)
+ return retval
- @classmethod
- def build_model(cls, args: argparse.Namespace):
- assert check_argument_types()
- if isinstance(args.token_list, str):
- with open(args.token_list, encoding="utf-8") as f:
- token_list = [line.rstrip() for line in f]
+ @classmethod
+ def build_model(cls, args: argparse.Namespace):
+ assert check_argument_types()
+ if isinstance(args.token_list, str):
+ with open(args.token_list, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
- # Overwriting token_list to keep it as "portable".
- args.token_list = list(token_list)
- elif isinstance(args.token_list, (tuple, list)):
- token_list = list(args.token_list)
- else:
- raise RuntimeError("token_list must be str or list")
- vocab_size = len(token_list)
- logging.info(f"Vocabulary size: {vocab_size}")
+ # Overwriting token_list to keep it as "portable".
+ args.token_list = list(token_list)
+ elif isinstance(args.token_list, (tuple, list)):
+ token_list = list(args.token_list)
+ else:
+ raise RuntimeError("token_list must be str or list")
+ vocab_size = len(token_list)
+ logging.info(f"Vocabulary size: {vocab_size}")
- # 1. frontend
- if args.input_size is None:
- # Extract features in the model
- frontend_class = frontend_choices.get_class(args.frontend)
- frontend = frontend_class(**args.frontend_conf)
- input_size = frontend.output_size()
- else:
- # Give features from data-loader
- args.frontend = None
- args.frontend_conf = {}
- frontend = None
- input_size = args.input_size
+ # 1. frontend
+ if args.input_size is None:
+ # Extract features in the model
+ frontend_class = frontend_choices.get_class(args.frontend)
+ if args.frontend == 'wav_frontend':
+ frontend = frontend_class(cmvn_file=args.cmvn_file, **args.frontend_conf)
+ else:
+ frontend = frontend_class(**args.frontend_conf)
+ input_size = frontend.output_size()
+ else:
+ # Give features from data-loader
+ args.frontend = None
+ args.frontend_conf = {}
+ frontend = None
+ input_size = args.input_size
- # 2. Data augmentation for spectrogram
- if args.specaug is not None:
- specaug_class = specaug_choices.get_class(args.specaug)
- specaug = specaug_class(**args.specaug_conf)
- else:
- specaug = None
+ # 2. Data augmentation for spectrogram
+ if args.specaug is not None:
+ specaug_class = specaug_choices.get_class(args.specaug)
+ specaug = specaug_class(**args.specaug_conf)
+ else:
+ specaug = None
- # 3. Normalization layer
- if args.normalize is not None:
- normalize_class = normalize_choices.get_class(args.normalize)
- normalize = normalize_class(**args.normalize_conf)
- else:
- normalize = None
+ # 3. Normalization layer
+ if args.normalize is not None:
+ normalize_class = normalize_choices.get_class(args.normalize)
+ normalize = normalize_class(**args.normalize_conf)
+ else:
+ normalize = None
- # 4. Pre-encoder input block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- if getattr(args, "preencoder", None) is not None:
- preencoder_class = preencoder_choices.get_class(args.preencoder)
- preencoder = preencoder_class(**args.preencoder_conf)
- input_size = preencoder.output_size()
- else:
- preencoder = None
+ # 4. Pre-encoder input block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ if getattr(args, "preencoder", None) is not None:
+ preencoder_class = preencoder_choices.get_class(args.preencoder)
+ preencoder = preencoder_class(**args.preencoder_conf)
+ input_size = preencoder.output_size()
+ else:
+ preencoder = None
- # 5. Encoder
- encoder_class = encoder_choices.get_class(args.encoder)
- encoder = encoder_class(input_size=input_size, **args.encoder_conf)
+ # 5. Encoder
+ encoder_class = encoder_choices.get_class(args.encoder)
+ encoder = encoder_class(input_size=input_size, **args.encoder_conf)
- # 6. Post-encoder block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- encoder_output_size = encoder.output_size()
- if getattr(args, "postencoder", None) is not None:
- postencoder_class = postencoder_choices.get_class(args.postencoder)
- postencoder = postencoder_class(
- input_size=encoder_output_size, **args.postencoder_conf
- )
- encoder_output_size = postencoder.output_size()
- else:
- postencoder = None
+ # 6. Post-encoder block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ encoder_output_size = encoder.output_size()
+ if getattr(args, "postencoder", None) is not None:
+ postencoder_class = postencoder_choices.get_class(args.postencoder)
+ postencoder = postencoder_class(
+ input_size=encoder_output_size, **args.postencoder_conf
+ )
+ encoder_output_size = postencoder.output_size()
+ else:
+ postencoder = None
- # 7. Decoder
- decoder_class = decoder_choices.get_class(args.decoder)
- decoder = decoder_class(
- vocab_size=vocab_size,
- encoder_output_size=encoder_output_size,
- **args.decoder_conf,
- )
+ # 7. Decoder
+ decoder_class = decoder_choices.get_class(args.decoder)
+ decoder = decoder_class(
+ vocab_size=vocab_size,
+ encoder_output_size=encoder_output_size,
+ **args.decoder_conf,
+ )
- # 8. CTC
- ctc = CTC(
- odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
- )
+ # 8. CTC
+ ctc = CTC(
+ odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
+ )
- # 9. Build model
- try:
- model_class = model_choices.get_class(args.model)
- except AttributeError:
- model_class = model_choices.get_class("asr")
- model = model_class(
- vocab_size=vocab_size,
- frontend=frontend,
- specaug=specaug,
- normalize=normalize,
- preencoder=preencoder,
- encoder=encoder,
- postencoder=postencoder,
- decoder=decoder,
- ctc=ctc,
- token_list=token_list,
- **args.model_conf,
- )
+ # 9. Build model
+ try:
+ model_class = model_choices.get_class(args.model)
+ except AttributeError:
+ model_class = model_choices.get_class("asr")
+ model = model_class(
+ vocab_size=vocab_size,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ decoder=decoder,
+ ctc=ctc,
+ token_list=token_list,
+ **args.model_conf,
+ )
- # 10. Initialize
- if args.init is not None:
- initialize(model, args.init)
+ # 10. Initialize
+ if args.init is not None:
+ initialize(model, args.init)
- assert check_return_type(model)
- return model
+ assert check_return_type(model)
+ return model
class ASRTaskUniASR(ASRTask):
- # If you need more than one optimizers, change this value
- num_optimizers: int = 1
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
- # Add variable objects configurations
- class_choices_list = [
- # --frontend and --frontend_conf
- frontend_choices,
- # --specaug and --specaug_conf
- specaug_choices,
- # --normalize and --normalize_conf
- normalize_choices,
- # --model and --model_conf
- model_choices,
- # --preencoder and --preencoder_conf
- preencoder_choices,
- # --encoder and --encoder_conf
- encoder_choices,
- # --postencoder and --postencoder_conf
- postencoder_choices,
- # --decoder and --decoder_conf
- decoder_choices,
- # --predictor and --predictor_conf
- predictor_choices,
- # --encoder2 and --encoder2_conf
- encoder_choices2,
- # --decoder2 and --decoder2_conf
- decoder_choices2,
- # --predictor2 and --predictor2_conf
- predictor_choices2,
- # --stride_conv and --stride_conv_conf
- stride_conv_choices,
- ]
+ # Add variable objects configurations
+ class_choices_list = [
+ # --frontend and --frontend_conf
+ frontend_choices,
+ # --specaug and --specaug_conf
+ specaug_choices,
+ # --normalize and --normalize_conf
+ normalize_choices,
+ # --model and --model_conf
+ model_choices,
+ # --preencoder and --preencoder_conf
+ preencoder_choices,
+ # --encoder and --encoder_conf
+ encoder_choices,
+ # --postencoder and --postencoder_conf
+ postencoder_choices,
+ # --decoder and --decoder_conf
+ decoder_choices,
+ # --predictor and --predictor_conf
+ predictor_choices,
+ # --encoder2 and --encoder2_conf
+ encoder_choices2,
+ # --decoder2 and --decoder2_conf
+ decoder_choices2,
+ # --predictor2 and --predictor2_conf
+ predictor_choices2,
+ # --stride_conv and --stride_conv_conf
+ stride_conv_choices,
+ ]
- # If you need to modify train() or eval() procedures, change Trainer class here
- trainer = Trainer
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
- @classmethod
- def build_model(cls, args: argparse.Namespace):
- assert check_argument_types()
- if isinstance(args.token_list, str):
- with open(args.token_list, encoding="utf-8") as f:
- token_list = [line.rstrip() for line in f]
+ @classmethod
+ def build_model(cls, args: argparse.Namespace):
+ assert check_argument_types()
+ if isinstance(args.token_list, str):
+ with open(args.token_list, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
- # Overwriting token_list to keep it as "portable".
- args.token_list = list(token_list)
- elif isinstance(args.token_list, (tuple, list)):
- token_list = list(args.token_list)
- else:
- raise RuntimeError("token_list must be str or list")
- vocab_size = len(token_list)
- logging.info(f"Vocabulary size: {vocab_size}")
+ # Overwriting token_list to keep it as "portable".
+ args.token_list = list(token_list)
+ elif isinstance(args.token_list, (tuple, list)):
+ token_list = list(args.token_list)
+ else:
+ raise RuntimeError("token_list must be str or list")
+ vocab_size = len(token_list)
+ logging.info(f"Vocabulary size: {vocab_size}")
- # 1. frontend
- if args.input_size is None:
- # Extract features in the model
- frontend_class = frontend_choices.get_class(args.frontend)
- frontend = frontend_class(**args.frontend_conf)
- input_size = frontend.output_size()
- else:
- # Give features from data-loader
- args.frontend = None
- args.frontend_conf = {}
- frontend = None
- input_size = args.input_size
+ # 1. frontend
+ if args.input_size is None:
+ # Extract features in the model
+ frontend_class = frontend_choices.get_class(args.frontend)
+ if args.frontend == 'wav_frontend':
+ frontend = frontend_class(cmvn_file=args.cmvn_file, **args.frontend_conf)
+ else:
+ frontend = frontend_class(**args.frontend_conf)
+ input_size = frontend.output_size()
+ else:
+ # Give features from data-loader
+ args.frontend = None
+ args.frontend_conf = {}
+ frontend = None
+ input_size = args.input_size
- # 2. Data augmentation for spectrogram
- if args.specaug is not None:
- specaug_class = specaug_choices.get_class(args.specaug)
- specaug = specaug_class(**args.specaug_conf)
- else:
- specaug = None
+ # 2. Data augmentation for spectrogram
+ if args.specaug is not None:
+ specaug_class = specaug_choices.get_class(args.specaug)
+ specaug = specaug_class(**args.specaug_conf)
+ else:
+ specaug = None
- # 3. Normalization layer
- if args.normalize is not None:
- normalize_class = normalize_choices.get_class(args.normalize)
- normalize = normalize_class(**args.normalize_conf)
- else:
- normalize = None
+ # 3. Normalization layer
+ if args.normalize is not None:
+ normalize_class = normalize_choices.get_class(args.normalize)
+ normalize = normalize_class(**args.normalize_conf)
+ else:
+ normalize = None
- # 4. Pre-encoder input block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- if getattr(args, "preencoder", None) is not None:
- preencoder_class = preencoder_choices.get_class(args.preencoder)
- preencoder = preencoder_class(**args.preencoder_conf)
- input_size = preencoder.output_size()
- else:
- preencoder = None
+ # 4. Pre-encoder input block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ if getattr(args, "preencoder", None) is not None:
+ preencoder_class = preencoder_choices.get_class(args.preencoder)
+ preencoder = preencoder_class(**args.preencoder_conf)
+ input_size = preencoder.output_size()
+ else:
+ preencoder = None
- # 5. Encoder
- encoder_class = encoder_choices.get_class(args.encoder)
- encoder = encoder_class(input_size=input_size, **args.encoder_conf)
- encoder_output_size = encoder.output_size()
+ # 5. Encoder
+ encoder_class = encoder_choices.get_class(args.encoder)
+ encoder = encoder_class(input_size=input_size, **args.encoder_conf)
+ encoder_output_size = encoder.output_size()
- stride_conv_class = stride_conv_choices.get_class(args.stride_conv)
- stride_conv = stride_conv_class(**args.stride_conv_conf, idim=input_size + encoder_output_size,
- odim=input_size + encoder_output_size)
- stride_conv_output_size = stride_conv.output_size()
+ stride_conv_class = stride_conv_choices.get_class(args.stride_conv)
+ stride_conv = stride_conv_class(**args.stride_conv_conf, idim=input_size + encoder_output_size,
+ odim=input_size + encoder_output_size)
+ stride_conv_output_size = stride_conv.output_size()
- # 6. Encoder2
- encoder_class2 = encoder_choices2.get_class(args.encoder2)
- encoder2 = encoder_class2(input_size=stride_conv_output_size, **args.encoder2_conf)
+ # 6. Encoder2
+ encoder_class2 = encoder_choices2.get_class(args.encoder2)
+ encoder2 = encoder_class2(input_size=stride_conv_output_size, **args.encoder2_conf)
- # 7. Post-encoder block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- encoder_output_size2 = encoder2.output_size()
- if getattr(args, "postencoder", None) is not None:
- postencoder_class = postencoder_choices.get_class(args.postencoder)
- postencoder = postencoder_class(
- input_size=encoder_output_size, **args.postencoder_conf
- )
- encoder_output_size = postencoder.output_size()
- else:
- postencoder = None
+ # 7. Post-encoder block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ encoder_output_size2 = encoder2.output_size()
+ if getattr(args, "postencoder", None) is not None:
+ postencoder_class = postencoder_choices.get_class(args.postencoder)
+ postencoder = postencoder_class(
+ input_size=encoder_output_size, **args.postencoder_conf
+ )
+ encoder_output_size = postencoder.output_size()
+ else:
+ postencoder = None
- # 8. Decoder & Decoder2
- decoder_class = decoder_choices.get_class(args.decoder)
- decoder_class2 = decoder_choices2.get_class(args.decoder2)
- decoder = decoder_class(
- vocab_size=vocab_size,
- encoder_output_size=encoder_output_size,
- **args.decoder_conf,
- )
- decoder2 = decoder_class2(
- vocab_size=vocab_size,
- encoder_output_size=encoder_output_size2,
- **args.decoder2_conf,
- )
+ # 8. Decoder & Decoder2
+ decoder_class = decoder_choices.get_class(args.decoder)
+ decoder_class2 = decoder_choices2.get_class(args.decoder2)
+ decoder = decoder_class(
+ vocab_size=vocab_size,
+ encoder_output_size=encoder_output_size,
+ **args.decoder_conf,
+ )
+ decoder2 = decoder_class2(
+ vocab_size=vocab_size,
+ encoder_output_size=encoder_output_size2,
+ **args.decoder2_conf,
+ )
- # 9. CTC
- ctc = CTC(
- odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
- )
- ctc2 = CTC(
- odim=vocab_size, encoder_output_size=encoder_output_size2, **args.ctc_conf
- )
+ # 9. CTC
+ ctc = CTC(
+ odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
+ )
+ ctc2 = CTC(
+ odim=vocab_size, encoder_output_size=encoder_output_size2, **args.ctc_conf
+ )
- # 10. Predictor
- predictor_class = predictor_choices.get_class(args.predictor)
- predictor = predictor_class(**args.predictor_conf)
+ # 10. Predictor
+ predictor_class = predictor_choices.get_class(args.predictor)
+ predictor = predictor_class(**args.predictor_conf)
- predictor_class = predictor_choices2.get_class(args.predictor2)
- predictor2 = predictor_class(**args.predictor2_conf)
+ predictor_class = predictor_choices2.get_class(args.predictor2)
+ predictor2 = predictor_class(**args.predictor2_conf)
- # 11. Build model
- try:
- model_class = model_choices.get_class(args.model)
- except AttributeError:
- model_class = model_choices.get_class("asr")
- model = model_class(
- vocab_size=vocab_size,
- frontend=frontend,
- specaug=specaug,
- normalize=normalize,
- preencoder=preencoder,
- encoder=encoder,
- postencoder=postencoder,
- decoder=decoder,
- ctc=ctc,
- token_list=token_list,
- predictor=predictor,
- ctc2=ctc2,
- encoder2=encoder2,
- decoder2=decoder2,
- predictor2=predictor2,
- stride_conv=stride_conv,
- **args.model_conf,
- )
+ # 11. Build model
+ try:
+ model_class = model_choices.get_class(args.model)
+ except AttributeError:
+ model_class = model_choices.get_class("asr")
+ model = model_class(
+ vocab_size=vocab_size,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ decoder=decoder,
+ ctc=ctc,
+ token_list=token_list,
+ predictor=predictor,
+ ctc2=ctc2,
+ encoder2=encoder2,
+ decoder2=decoder2,
+ predictor2=predictor2,
+ stride_conv=stride_conv,
+ **args.model_conf,
+ )
- # 12. Initialize
- if args.init is not None:
- initialize(model, args.init)
+ # 12. Initialize
+ if args.init is not None:
+ initialize(model, args.init)
- assert check_return_type(model)
- return model
+ assert check_return_type(model)
+ return model
+
+ # ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~
+ @classmethod
+ def build_model_from_file(
+ cls,
+ config_file: Union[Path, str] = None,
+ model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ device: str = "cpu",
+ ):
+ """Build model from the files.
+
+ This method is used for inference or fine-tuning.
+
+ Args:
+ config_file: The yaml file saved when training.
+ model_file: The model file saved when training.
+ device: Device type, "cpu", "cuda", or "cuda:N".
+
+ """
+ assert check_argument_types()
+ if config_file is None:
+ assert model_file is not None, (
+ "The argument 'model_file' must be provided "
+ "if the argument 'config_file' is not specified."
+ )
+ config_file = Path(model_file).parent / "config.yaml"
+ else:
+ config_file = Path(config_file)
+
+ with config_file.open("r", encoding="utf-8") as f:
+ args = yaml.safe_load(f)
+ if cmvn_file is not None:
+ args["cmvn_file"] = cmvn_file
+ args = argparse.Namespace(**args)
+ model = cls.build_model(args)
+ if not isinstance(model, AbsESPnetModel):
+ raise RuntimeError(
+ f"model must inherit {AbsESPnetModel.__name__}, but got {type(model)}"
+ )
+ model.to(device)
+ model_dict = dict()
+ model_name_pth = None
+ if model_file is not None:
+ logging.info("model_file is {}".format(model_file))
+ if device == "cuda":
+ device = f"cuda:{torch.cuda.current_device()}"
+ model_dir = os.path.dirname(model_file)
+ model_name = os.path.basename(model_file)
+ if "model.ckpt-" in model_name or ".bin" in model_name:
+ model_name_pth = os.path.join(model_dir, model_name.replace('.bin',
+ '.pb')) if ".bin" in model_name else os.path.join(
+ model_dir, "{}.pth".format(model_name))
+ if os.path.exists(model_name_pth):
+ logging.info("model_file is load from pth: {}".format(model_name_pth))
+ model_dict = torch.load(model_name_pth, map_location=device)
+ else:
+ model_dict = cls.convert_tf2torch(model, model_file)
+ model.load_state_dict(model_dict)
+ else:
+ model_dict = torch.load(model_file, map_location=device)
+ model.load_state_dict(model_dict)
+ if model_name_pth is not None and not os.path.exists(model_name_pth):
+ torch.save(model_dict, model_name_pth)
+ logging.info("model_file is saved to pth: {}".format(model_name_pth))
+
+ return model, args
+
+ @classmethod
+ def convert_tf2torch(
+ cls,
+ model,
+ ckpt,
+ ):
+ logging.info("start convert tf model to torch model")
+ from funasr.modules.streaming_utils.load_fr_tf import load_tf_dict
+ var_dict_tf = load_tf_dict(ckpt)
+ var_dict_torch = model.state_dict()
+ var_dict_torch_update = dict()
+ # encoder
+ var_dict_torch_update_local = model.encoder.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # predictor
+ var_dict_torch_update_local = model.predictor.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # decoder
+ var_dict_torch_update_local = model.decoder.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # encoder2
+ var_dict_torch_update_local = model.encoder2.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # predictor2
+ var_dict_torch_update_local = model.predictor2.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # decoder2
+ var_dict_torch_update_local = model.decoder2.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # stride_conv
+ var_dict_torch_update_local = model.stride_conv.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+
+ return var_dict_torch_update
class ASRTaskParaformer(ASRTask):
- # If you need more than one optimizers, change this value
- num_optimizers: int = 1
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
- # Add variable objects configurations
- class_choices_list = [
- # --frontend and --frontend_conf
- frontend_choices,
- # --specaug and --specaug_conf
- specaug_choices,
- # --normalize and --normalize_conf
- normalize_choices,
- # --model and --model_conf
- model_choices,
- # --preencoder and --preencoder_conf
- preencoder_choices,
- # --encoder and --encoder_conf
- encoder_choices,
- # --postencoder and --postencoder_conf
- postencoder_choices,
- # --decoder and --decoder_conf
- decoder_choices,
- # --predictor and --predictor_conf
- predictor_choices,
- ]
+ # Add variable objects configurations
+ class_choices_list = [
+ # --frontend and --frontend_conf
+ frontend_choices,
+ # --specaug and --specaug_conf
+ specaug_choices,
+ # --normalize and --normalize_conf
+ normalize_choices,
+ # --model and --model_conf
+ model_choices,
+ # --preencoder and --preencoder_conf
+ preencoder_choices,
+ # --encoder and --encoder_conf
+ encoder_choices,
+ # --postencoder and --postencoder_conf
+ postencoder_choices,
+ # --decoder and --decoder_conf
+ decoder_choices,
+ # --predictor and --predictor_conf
+ predictor_choices,
+ ]
- # If you need to modify train() or eval() procedures, change Trainer class here
- trainer = Trainer
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
- @classmethod
- def build_model(cls, args: argparse.Namespace):
- assert check_argument_types()
- if isinstance(args.token_list, str):
- with open(args.token_list, encoding="utf-8") as f:
- token_list = [line.rstrip() for line in f]
+ @classmethod
+ def build_model(cls, args: argparse.Namespace):
+ assert check_argument_types()
+ if isinstance(args.token_list, str):
+ with open(args.token_list, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
- # Overwriting token_list to keep it as "portable".
- args.token_list = list(token_list)
- elif isinstance(args.token_list, (tuple, list)):
- token_list = list(args.token_list)
- else:
- raise RuntimeError("token_list must be str or list")
- vocab_size = len(token_list)
- logging.info(f"Vocabulary size: {vocab_size }")
+ # Overwriting token_list to keep it as "portable".
+ args.token_list = list(token_list)
+ elif isinstance(args.token_list, (tuple, list)):
+ token_list = list(args.token_list)
+ else:
+ raise RuntimeError("token_list must be str or list")
+ vocab_size = len(token_list)
+ logging.info(f"Vocabulary size: {vocab_size}")
- # 1. frontend
- if args.input_size is None:
- # Extract features in the model
- frontend_class = frontend_choices.get_class(args.frontend)
- frontend = frontend_class(**args.frontend_conf)
- input_size = frontend.output_size()
- else:
- # Give features from data-loader
- args.frontend = None
- args.frontend_conf = {}
- frontend = None
- input_size = args.input_size
+ # 1. frontend
+ if args.input_size is None:
+ # Extract features in the model
+ frontend_class = frontend_choices.get_class(args.frontend)
+ if args.frontend == 'wav_frontend':
+ frontend = frontend_class(cmvn_file=args.cmvn_file, **args.frontend_conf)
+ else:
+ frontend = frontend_class(**args.frontend_conf)
+ input_size = frontend.output_size()
+ else:
+ # Give features from data-loader
+ args.frontend = None
+ args.frontend_conf = {}
+ frontend = None
+ input_size = args.input_size
- # 2. Data augmentation for spectrogram
- if args.specaug is not None:
- specaug_class = specaug_choices.get_class(args.specaug)
- specaug = specaug_class(**args.specaug_conf)
- else:
- specaug = None
+ # 2. Data augmentation for spectrogram
+ if args.specaug is not None:
+ specaug_class = specaug_choices.get_class(args.specaug)
+ specaug = specaug_class(**args.specaug_conf)
+ else:
+ specaug = None
- # 3. Normalization layer
- if args.normalize is not None:
- normalize_class = normalize_choices.get_class(args.normalize)
- normalize = normalize_class(**args.normalize_conf)
- else:
- normalize = None
+ # 3. Normalization layer
+ if args.normalize is not None:
+ normalize_class = normalize_choices.get_class(args.normalize)
+ normalize = normalize_class(**args.normalize_conf)
+ else:
+ normalize = None
- # 4. Pre-encoder input block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- if getattr(args, "preencoder", None) is not None:
- preencoder_class = preencoder_choices.get_class(args.preencoder)
- preencoder = preencoder_class(**args.preencoder_conf)
- input_size = preencoder.output_size()
- else:
- preencoder = None
+ # 4. Pre-encoder input block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ if getattr(args, "preencoder", None) is not None:
+ preencoder_class = preencoder_choices.get_class(args.preencoder)
+ preencoder = preencoder_class(**args.preencoder_conf)
+ input_size = preencoder.output_size()
+ else:
+ preencoder = None
- # 5. Encoder
- encoder_class = encoder_choices.get_class(args.encoder)
- encoder = encoder_class(input_size=input_size, **args.encoder_conf)
+ # 5. Encoder
+ encoder_class = encoder_choices.get_class(args.encoder)
+ encoder = encoder_class(input_size=input_size, **args.encoder_conf)
- # 6. Post-encoder block
- # NOTE(kan-bayashi): Use getattr to keep the compatibility
- encoder_output_size = encoder.output_size()
- if getattr(args, "postencoder", None) is not None:
- postencoder_class = postencoder_choices.get_class(args.postencoder)
- postencoder = postencoder_class(
- input_size=encoder_output_size, **args.postencoder_conf
- )
- encoder_output_size = postencoder.output_size()
- else:
- postencoder = None
+ # 6. Post-encoder block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ encoder_output_size = encoder.output_size()
+ if getattr(args, "postencoder", None) is not None:
+ postencoder_class = postencoder_choices.get_class(args.postencoder)
+ postencoder = postencoder_class(
+ input_size=encoder_output_size, **args.postencoder_conf
+ )
+ encoder_output_size = postencoder.output_size()
+ else:
+ postencoder = None
- # 7. Decoder
- decoder_class = decoder_choices.get_class(args.decoder)
- decoder = decoder_class(
- vocab_size=vocab_size,
- encoder_output_size=encoder_output_size,
- **args.decoder_conf,
- )
+ # 7. Decoder
+ decoder_class = decoder_choices.get_class(args.decoder)
+ decoder = decoder_class(
+ vocab_size=vocab_size,
+ encoder_output_size=encoder_output_size,
+ **args.decoder_conf,
+ )
- # 8. CTC
- ctc = CTC(
- odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
- )
+ # 8. CTC
+ ctc = CTC(
+ odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
+ )
- # 9. Predictor
- predictor_class = predictor_choices.get_class(args.predictor)
- predictor = predictor_class(**args.predictor_conf)
+ # 9. Predictor
+ predictor_class = predictor_choices.get_class(args.predictor)
+ predictor = predictor_class(**args.predictor_conf)
- # 10. Build model
- try:
- model_class = model_choices.get_class(args.model)
- except AttributeError:
- model_class = model_choices.get_class("asr")
- model = model_class(
- vocab_size=vocab_size,
- frontend=frontend,
- specaug=specaug,
- normalize=normalize,
- preencoder=preencoder,
- encoder=encoder,
- postencoder=postencoder,
- decoder=decoder,
- ctc=ctc,
- token_list=token_list,
- predictor=predictor,
- **args.model_conf,
- )
+ # 10. Build model
+ try:
+ model_class = model_choices.get_class(args.model)
+ except AttributeError:
+ model_class = model_choices.get_class("asr")
+ model = model_class(
+ vocab_size=vocab_size,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ decoder=decoder,
+ ctc=ctc,
+ token_list=token_list,
+ predictor=predictor,
+ **args.model_conf,
+ )
- # 11. Initialize
- if args.init is not None:
- initialize(model, args.init)
+ # 11. Initialize
+ if args.init is not None:
+ initialize(model, args.init)
- assert check_return_type(model)
- return model
+ assert check_return_type(model)
+ return model
+
+ # ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~
+ @classmethod
+ def build_model_from_file(
+ cls,
+ config_file: Union[Path, str] = None,
+ model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ device: str = "cpu",
+ ):
+ """Build model from the files.
+
+ This method is used for inference or fine-tuning.
+
+ Args:
+ config_file: The yaml file saved when training.
+ model_file: The model file saved when training.
+ device: Device type, "cpu", "cuda", or "cuda:N".
+
+ """
+ assert check_argument_types()
+ if config_file is None:
+ assert model_file is not None, (
+ "The argument 'model_file' must be provided "
+ "if the argument 'config_file' is not specified."
+ )
+ config_file = Path(model_file).parent / "config.yaml"
+ else:
+ config_file = Path(config_file)
+
+ with config_file.open("r", encoding="utf-8") as f:
+ args = yaml.safe_load(f)
+ if cmvn_file is not None:
+ args["cmvn_file"] = cmvn_file
+ args = argparse.Namespace(**args)
+ model = cls.build_model(args)
+ if not isinstance(model, AbsESPnetModel):
+ raise RuntimeError(
+ f"model must inherit {AbsESPnetModel.__name__}, but got {type(model)}"
+ )
+ model.to(device)
+ model_dict = dict()
+ model_name_pth = None
+ if model_file is not None:
+ logging.info("model_file is {}".format(model_file))
+ if device == "cuda":
+ device = f"cuda:{torch.cuda.current_device()}"
+ model_dir = os.path.dirname(model_file)
+ model_name = os.path.basename(model_file)
+ if "model.ckpt-" in model_name or ".bin" in model_name:
+ model_name_pth = os.path.join(model_dir, model_name.replace('.bin',
+ '.pb')) if ".bin" in model_name else os.path.join(
+ model_dir, "{}.pth".format(model_name))
+ if os.path.exists(model_name_pth):
+ logging.info("model_file is load from pth: {}".format(model_name_pth))
+ model_dict = torch.load(model_name_pth, map_location=device)
+ else:
+ model_dict = cls.convert_tf2torch(model, model_file)
+ model.load_state_dict(model_dict)
+ else:
+ model_dict = torch.load(model_file, map_location=device)
+ model.load_state_dict(model_dict)
+ if model_name_pth is not None and not os.path.exists(model_name_pth):
+ torch.save(model_dict, model_name_pth)
+ logging.info("model_file is saved to pth: {}".format(model_name_pth))
+ model.to(device)
+ return model, args
+
+ @classmethod
+ def convert_tf2torch(
+ cls,
+ model,
+ ckpt,
+ ):
+ logging.info("start convert tf model to torch model")
+ from funasr.modules.streaming_utils.load_fr_tf import load_tf_dict
+ var_dict_tf = load_tf_dict(ckpt)
+ var_dict_torch = model.state_dict()
+ var_dict_torch_update = dict()
+ # encoder
+ var_dict_torch_update_local = model.encoder.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # predictor
+ var_dict_torch_update_local = model.predictor.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+ # decoder
+ var_dict_torch_update_local = model.decoder.convert_tf2torch(var_dict_tf, var_dict_torch)
+ var_dict_torch_update.update(var_dict_torch_update_local)
+
+ return var_dict_torch_update
diff --git a/funasr/tasks/punctuation.py b/funasr/tasks/punctuation.py
new file mode 100644
index 0000000..1837b2a
--- /dev/null
+++ b/funasr/tasks/punctuation.py
@@ -0,0 +1,228 @@
+import argparse
+import logging
+from typing import Callable
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+import numpy as np
+import torch
+from typeguard import check_argument_types
+from typeguard import check_return_type
+
+from funasr.datasets.collate_fn import CommonCollateFn
+from funasr.datasets.preprocessor import MutliTokenizerCommonPreprocessor
+from funasr.punctuation.abs_model import AbsPunctuation
+from funasr.punctuation.espnet_model import ESPnetPunctuationModel
+from funasr.punctuation.target_delay_transformer import TargetDelayTransformer
+from funasr.tasks.abs_task import AbsTask
+from funasr.text.phoneme_tokenizer import g2p_choices
+from funasr.torch_utils.initialize import initialize
+from funasr.train.class_choices import ClassChoices
+from funasr.train.trainer import Trainer
+from funasr.utils.get_default_kwargs import get_default_kwargs
+from funasr.utils.nested_dict_action import NestedDictAction
+from funasr.utils.types import str2bool
+from funasr.utils.types import str_or_none
+
+punc_choices = ClassChoices(
+ "punctuation",
+ classes=dict(
+ target_delay=TargetDelayTransformer,
+ ),
+ type_check=AbsPunctuation,
+ default="TargetDelayTransformer",
+)
+
+
+class PunctuationTask(AbsTask):
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
+
+ # Add variable objects configurations
+ class_choices_list = [punc_choices]
+
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
+
+ @classmethod
+ def add_task_arguments(cls, parser: argparse.ArgumentParser):
+ # NOTE(kamo): Use '_' instead of '-' to avoid confusion
+ assert check_argument_types()
+ group = parser.add_argument_group(description="Task related")
+
+ # NOTE(kamo): add_arguments(..., required=True) can't be used
+ # to provide --print_config mode. Instead of it, do as
+ required = parser.get_default("required")
+ #import pdb;pdb.set_trace()
+ #required += ["token_list"]
+
+ group.add_argument(
+ "--token_list",
+ type=str_or_none,
+ default=None,
+ help="A text mapping int-id to token",
+ )
+ group.add_argument(
+ "--init",
+ type=lambda x: str_or_none(x.lower()),
+ default=None,
+ help="The initialization method",
+ choices=[
+ "chainer",
+ "xavier_uniform",
+ "xavier_normal",
+ "kaiming_uniform",
+ "kaiming_normal",
+ None,
+ ],
+ )
+ group.add_argument(
+ "--model_conf",
+ action=NestedDictAction,
+ default=get_default_kwargs(ESPnetPunctuationModel),
+ help="The keyword arguments for model class.",
+ )
+
+ group = parser.add_argument_group(description="Preprocess related")
+ group.add_argument(
+ "--use_preprocessor",
+ type=str2bool,
+ default=True,
+ help="Apply preprocessing to data or not",
+ )
+ group.add_argument(
+ "--token_type",
+ type=str,
+ default="bpe",
+ choices=["bpe", "char", "word"],
+ help="",
+ )
+ group.add_argument(
+ "--bpemodel",
+ type=str_or_none,
+ default=None,
+ help="The model file fo sentencepiece",
+ )
+ parser.add_argument(
+ "--non_linguistic_symbols",
+ type=str_or_none,
+ help="non_linguistic_symbols file path",
+ )
+ parser.add_argument(
+ "--cleaner",
+ type=str_or_none,
+ choices=[None, "tacotron", "jaconv", "vietnamese"],
+ default=None,
+ help="Apply text cleaning",
+ )
+ parser.add_argument(
+ "--g2p",
+ type=str_or_none,
+ choices=g2p_choices,
+ default=None,
+ help="Specify g2p method if --token_type=phn",
+ )
+
+ for class_choices in cls.class_choices_list:
+ # Append --<name> and --<name>_conf.
+ # e.g. --encoder and --encoder_conf
+ class_choices.add_arguments(group)
+
+ assert check_return_type(parser)
+ return parser
+
+ @classmethod
+ def build_collate_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Callable[
+ [Collection[Tuple[str, Dict[str, np.ndarray]]]],
+ Tuple[List[str], Dict[str, torch.Tensor]],
+ ]:
+ assert check_argument_types()
+ return CommonCollateFn(int_pad_value=0)
+
+ @classmethod
+ def build_preprocess_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
+ assert check_argument_types()
+ token_types = [args.token_type, args.token_type]
+ token_lists = [args.token_list, args.punc_list]
+ bpemodels = [args.bpemodel, args.bpemodel]
+ text_names = ["text", "punc"]
+ if args.use_preprocessor:
+ retval = MutliTokenizerCommonPreprocessor(
+ train=train,
+ token_type=token_types,
+ token_list=token_lists,
+ bpemodel=bpemodels,
+ text_cleaner=args.cleaner,
+ g2p_type=args.g2p,
+ text_name = text_names,
+ non_linguistic_symbols=args.non_linguistic_symbols,
+ )
+ else:
+ retval = None
+ assert check_return_type(retval)
+ return retval
+
+ @classmethod
+ def required_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ retval = ("text", "punc")
+ if inference:
+ retval = ("text", )
+ return retval
+
+ @classmethod
+ def optional_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ retval = ()
+ return retval
+
+ @classmethod
+ def build_model(cls, args: argparse.Namespace) -> ESPnetPunctuationModel:
+ assert check_argument_types()
+ if isinstance(args.token_list, str):
+ with open(args.token_list, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
+
+ # "args" is saved as it is in a yaml file by BaseTask.main().
+ # Overwriting token_list to keep it as "portable".
+ args.token_list = token_list.copy()
+ if isinstance(args.punc_list, str):
+ with open(args.punc_list, encoding="utf-8") as f2:
+ punc_list = [line.rstrip() for line in f2]
+ args.punc_list = punc_list.copy()
+ elif isinstance(args.punc_list, list):
+ # This is in the inference code path.
+ punc_list = args.punc_list.copy()
+ if isinstance(args.token_list, (tuple, list)):
+ token_list = args.token_list.copy()
+ else:
+ raise RuntimeError("token_list must be str or dict")
+
+ vocab_size = len(token_list)
+ punc_size = len(punc_list)
+ logging.info(f"Vocabulary size: {vocab_size}")
+
+ # 1. Build PUNC model
+ punc_class = punc_choices.get_class(args.punctuation)
+ punc = punc_class(vocab_size=vocab_size, punc_size=punc_size, **args.punctuation_conf)
+
+ # 2. Build ESPnetModel
+ # Assume the last-id is sos_and_eos
+ model = ESPnetPunctuationModel(punc_model=punc, vocab_size=vocab_size, **args.model_conf)
+
+ # FIXME(kamo): Should be done in model?
+ # 3. Initialize
+ if args.init is not None:
+ initialize(model, args.init)
+
+ assert check_return_type(model)
+ return model
diff --git a/funasr/tasks/sv.py b/funasr/tasks/sv.py
new file mode 100644
index 0000000..16384a7
--- /dev/null
+++ b/funasr/tasks/sv.py
@@ -0,0 +1,437 @@
+import argparse
+import logging
+from typing import Callable
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+import numpy as np
+import torch
+from typeguard import check_argument_types
+from typeguard import check_return_type
+
+from funasr.datasets.collate_fn import CommonCollateFn
+from funasr.datasets.preprocessor import CommonPreprocessor
+from funasr.layers.abs_normalize import AbsNormalize
+from funasr.layers.global_mvn import GlobalMVN
+from funasr.layers.utterance_mvn import UtteranceMVN
+from funasr.models.e2e_asr import ESPnetASRModel
+from funasr.models.decoder.abs_decoder import AbsDecoder
+from funasr.models.encoder.abs_encoder import AbsEncoder
+from funasr.models.encoder.rnn_encoder import RNNEncoder
+from funasr.models.encoder.resnet34_encoder import ResNet34
+from funasr.models.pooling.statistic_pooling import StatisticPooling
+from funasr.models.decoder.sv_decoder import DenseDecoder
+from funasr.models.e2e_sv import ESPnetSVModel
+from funasr.models.frontend.abs_frontend import AbsFrontend
+from funasr.models.frontend.default import DefaultFrontend
+from funasr.models.frontend.fused import FusedFrontends
+from funasr.models.frontend.s3prl import S3prlFrontend
+from funasr.models.frontend.windowing import SlidingWindow
+from funasr.models.postencoder.abs_postencoder import AbsPostEncoder
+from funasr.models.postencoder.hugging_face_transformers_postencoder import (
+ HuggingFaceTransformersPostEncoder, # noqa: H301
+)
+from funasr.models.preencoder.abs_preencoder import AbsPreEncoder
+from funasr.models.preencoder.linear import LinearProjection
+from funasr.models.preencoder.sinc import LightweightSincConvs
+from funasr.models.specaug.abs_specaug import AbsSpecAug
+from funasr.models.specaug.specaug import SpecAug
+from funasr.tasks.abs_task import AbsTask
+from funasr.torch_utils.initialize import initialize
+from funasr.train.abs_espnet_model import AbsESPnetModel
+from funasr.train.class_choices import ClassChoices
+from funasr.train.trainer import Trainer
+from funasr.utils.types import float_or_none
+from funasr.utils.types import int_or_none
+from funasr.utils.types import str2bool
+from funasr.utils.types import str_or_none
+from funasr.models.frontend.wav_frontend import WavFrontend
+
+frontend_choices = ClassChoices(
+ name="frontend",
+ classes=dict(
+ default=DefaultFrontend,
+ sliding_window=SlidingWindow,
+ s3prl=S3prlFrontend,
+ fused=FusedFrontends,
+ wav_frontend=WavFrontend,
+ ),
+ type_check=AbsFrontend,
+ default="default",
+)
+specaug_choices = ClassChoices(
+ name="specaug",
+ classes=dict(
+ specaug=SpecAug,
+ ),
+ type_check=AbsSpecAug,
+ default=None,
+ optional=True,
+)
+normalize_choices = ClassChoices(
+ "normalize",
+ classes=dict(
+ global_mvn=GlobalMVN,
+ utterance_mvn=UtteranceMVN,
+ ),
+ type_check=AbsNormalize,
+ default=None,
+ optional=True,
+)
+model_choices = ClassChoices(
+ "model",
+ classes=dict(
+ espnet=ESPnetSVModel,
+ ),
+ type_check=AbsESPnetModel,
+ default="espnet",
+)
+preencoder_choices = ClassChoices(
+ name="preencoder",
+ classes=dict(
+ sinc=LightweightSincConvs,
+ linear=LinearProjection,
+ ),
+ type_check=AbsPreEncoder,
+ default=None,
+ optional=True,
+)
+encoder_choices = ClassChoices(
+ "encoder",
+ classes=dict(
+ resnet34=ResNet34,
+ rnn=RNNEncoder,
+ ),
+ type_check=AbsEncoder,
+ default="resnet34",
+)
+postencoder_choices = ClassChoices(
+ name="postencoder",
+ classes=dict(
+ hugging_face_transformers=HuggingFaceTransformersPostEncoder,
+ ),
+ type_check=AbsPostEncoder,
+ default=None,
+ optional=True,
+)
+pooling_choices = ClassChoices(
+ name="pooling_type",
+ classes=dict(
+ statistic=StatisticPooling,
+ ),
+ type_check=torch.nn.Module,
+ default="statistic",
+)
+decoder_choices = ClassChoices(
+ "decoder",
+ classes=dict(
+ dense=DenseDecoder,
+ ),
+ type_check=AbsDecoder,
+ default="dense",
+)
+
+
+class SVTask(AbsTask):
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
+
+ # Add variable objects configurations
+ class_choices_list = [
+ # --frontend and --frontend_conf
+ frontend_choices,
+ # --specaug and --specaug_conf
+ specaug_choices,
+ # --normalize and --normalize_conf
+ normalize_choices,
+ # --model and --model_conf
+ model_choices,
+ # --preencoder and --preencoder_conf
+ preencoder_choices,
+ # --encoder and --encoder_conf
+ encoder_choices,
+ # --postencoder and --postencoder_conf
+ postencoder_choices,
+ # --pooling and --pooling_conf
+ pooling_choices,
+ # --decoder and --decoder_conf
+ decoder_choices,
+ ]
+
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
+
+ @classmethod
+ def add_task_arguments(cls, parser: argparse.ArgumentParser):
+ group = parser.add_argument_group(description="Task related")
+
+ # NOTE(kamo): add_arguments(..., required=True) can't be used
+ # to provide --print_config mode. Instead of it, do as
+ required = parser.get_default("required")
+ required += ["token_list"]
+
+ group.add_argument(
+ "--token_list",
+ type=str_or_none,
+ default=None,
+ help="A text mapping int-id to speaker name",
+ )
+ group.add_argument(
+ "--init",
+ type=lambda x: str_or_none(x.lower()),
+ default=None,
+ help="The initialization method",
+ choices=[
+ "chainer",
+ "xavier_uniform",
+ "xavier_normal",
+ "kaiming_uniform",
+ "kaiming_normal",
+ None,
+ ],
+ )
+
+ group.add_argument(
+ "--input_size",
+ type=int_or_none,
+ default=None,
+ help="The number of input dimension of the feature",
+ )
+
+ group = parser.add_argument_group(description="Preprocess related")
+ group.add_argument(
+ "--use_preprocessor",
+ type=str2bool,
+ default=True,
+ help="Apply preprocessing to data or not",
+ )
+ parser.add_argument(
+ "--cleaner",
+ type=str_or_none,
+ choices=[None, "tacotron", "jaconv", "vietnamese"],
+ default=None,
+ help="Apply text cleaning",
+ )
+ parser.add_argument(
+ "--speech_volume_normalize",
+ type=float_or_none,
+ default=None,
+ help="Scale the maximum amplitude to the given value.",
+ )
+ parser.add_argument(
+ "--rir_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of rir scp file.",
+ )
+ parser.add_argument(
+ "--rir_apply_prob",
+ type=float,
+ default=1.0,
+ help="THe probability for applying RIR convolution.",
+ )
+ parser.add_argument(
+ "--noise_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of noise scp file.",
+ )
+ parser.add_argument(
+ "--noise_apply_prob",
+ type=float,
+ default=1.0,
+ help="The probability applying Noise adding.",
+ )
+ parser.add_argument(
+ "--noise_db_range",
+ type=str,
+ default="13_15",
+ help="The range of noise decibel level.",
+ )
+
+ for class_choices in cls.class_choices_list:
+ # Append --<name> and --<name>_conf.
+ # e.g. --encoder and --encoder_conf
+ class_choices.add_arguments(group)
+
+ @classmethod
+ def build_collate_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Callable[
+ [Collection[Tuple[str, Dict[str, np.ndarray]]]],
+ Tuple[List[str], Dict[str, torch.Tensor]],
+ ]:
+ assert check_argument_types()
+ # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol
+ return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
+
+ @classmethod
+ def build_preprocess_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
+ assert check_argument_types()
+ if args.use_preprocessor:
+ retval = CommonPreprocessor(
+ train=train,
+ token_type=None,
+ token_list=None,
+ bpemodel=None,
+ non_linguistic_symbols=None,
+ text_cleaner=args.cleaner,
+ g2p_type=None,
+ # NOTE(kamo): Check attribute existence for backward compatibility
+ rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None,
+ rir_apply_prob=args.rir_apply_prob
+ if hasattr(args, "rir_apply_prob")
+ else 1.0,
+ noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None,
+ noise_apply_prob=args.noise_apply_prob
+ if hasattr(args, "noise_apply_prob")
+ else 1.0,
+ noise_db_range=args.noise_db_range
+ if hasattr(args, "noise_db_range")
+ else "13_15",
+ speech_volume_normalize=args.speech_volume_normalize
+ if hasattr(args, "rir_scp")
+ else None,
+ )
+ else:
+ retval = None
+ assert check_return_type(retval)
+ return retval
+
+ @classmethod
+ def required_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ if not inference:
+ retval = ("speech", "text")
+ else:
+ # Recognition mode
+ retval = ("speech",)
+ return retval
+
+ @classmethod
+ def optional_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ retval = ()
+ if inference:
+ retval = ("ref_speech",)
+ assert check_return_type(retval)
+ return retval
+
+ @classmethod
+ def build_model(cls, args: argparse.Namespace) -> ESPnetSVModel:
+ assert check_argument_types()
+ if isinstance(args.token_list, str):
+ with open(args.token_list, encoding="utf-8") as f:
+ token_list = [line.rstrip() for line in f]
+
+ # Overwriting token_list to keep it as "portable".
+ args.token_list = list(token_list)
+ elif isinstance(args.token_list, (tuple, list)):
+ token_list = list(args.token_list)
+ else:
+ raise RuntimeError("token_list must be str or list")
+ vocab_size = len(token_list)
+ logging.info(f"Speaker number: {vocab_size}")
+
+ # 1. frontend
+ if args.input_size is None:
+ # Extract features in the model
+ frontend_class = frontend_choices.get_class(args.frontend)
+ frontend = frontend_class(**args.frontend_conf)
+ input_size = frontend.output_size()
+ else:
+ # Give features from data-loader
+ args.frontend = None
+ args.frontend_conf = {}
+ frontend = None
+ input_size = args.input_size
+
+ # 2. Data augmentation for spectrogram
+ if args.specaug is not None:
+ specaug_class = specaug_choices.get_class(args.specaug)
+ specaug = specaug_class(**args.specaug_conf)
+ else:
+ specaug = None
+
+ # 3. Normalization layer
+ if args.normalize is not None:
+ normalize_class = normalize_choices.get_class(args.normalize)
+ normalize = normalize_class(**args.normalize_conf)
+ else:
+ normalize = None
+
+ # 4. Pre-encoder input block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ if getattr(args, "preencoder", None) is not None:
+ preencoder_class = preencoder_choices.get_class(args.preencoder)
+ preencoder = preencoder_class(**args.preencoder_conf)
+ input_size = preencoder.output_size()
+ else:
+ preencoder = None
+
+ # 5. Encoder
+ encoder_class = encoder_choices.get_class(args.encoder)
+ encoder = encoder_class(input_size=input_size, **args.encoder_conf)
+
+ # 6. Post-encoder block
+ # NOTE(kan-bayashi): Use getattr to keep the compatibility
+ encoder_output_size = encoder.output_size()
+ if getattr(args, "postencoder", None) is not None:
+ postencoder_class = postencoder_choices.get_class(args.postencoder)
+ postencoder = postencoder_class(
+ input_size=encoder_output_size, **args.postencoder_conf
+ )
+ encoder_output_size = postencoder.output_size()
+ else:
+ postencoder = None
+
+ # 7. Pooling layer
+ pooling_class = pooling_choices.get_class(args.pooling_type)
+ pooling_layer = pooling_class(
+ pooling_dim=(2, 3),
+ eps=1e-12,
+ )
+ if args.pooling_type == "statistic":
+ encoder_output_size *= 2
+
+ # 8. Decoder
+ decoder_class = decoder_choices.get_class(args.decoder)
+ decoder = decoder_class(
+ vocab_size=vocab_size,
+ encoder_output_size=encoder_output_size,
+ **args.decoder_conf,
+ )
+
+ # 7. Build model
+ try:
+ model_class = model_choices.get_class(args.model)
+ except AttributeError:
+ model_class = model_choices.get_class("espnet")
+ model = model_class(
+ vocab_size=vocab_size,
+ token_list=token_list,
+ frontend=frontend,
+ specaug=specaug,
+ normalize=normalize,
+ preencoder=preencoder,
+ encoder=encoder,
+ postencoder=postencoder,
+ pooling_layer=pooling_layer,
+ decoder=decoder,
+ **args.model_conf,
+ )
+
+ # FIXME(kamo): Should be done in model?
+ # 8. Initialize
+ if args.init is not None:
+ initialize(model, args.init)
+
+ assert check_return_type(model)
+ return model
diff --git a/funasr/tasks/vad.py b/funasr/tasks/vad.py
new file mode 100644
index 0000000..dfd07c4
--- /dev/null
+++ b/funasr/tasks/vad.py
@@ -0,0 +1,342 @@
+import argparse
+import logging
+from typing import Callable
+from typing import Collection
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+import os
+from pathlib import Path
+from typing import Tuple
+from typing import Union
+import yaml
+import numpy as np
+import torch
+from typeguard import check_argument_types
+from typeguard import check_return_type
+
+from funasr.datasets.collate_fn import CommonCollateFn
+from funasr.datasets.preprocessor import CommonPreprocessor
+from funasr.models.ctc import CTC
+from funasr.models.decoder.abs_decoder import AbsDecoder
+from funasr.models.decoder.rnn_decoder import RNNDecoder
+from funasr.models.decoder.transformer_decoder import (
+ DynamicConvolution2DTransformerDecoder, # noqa: H301
+)
+from funasr.models.decoder.transformer_decoder import DynamicConvolutionTransformerDecoder
+from funasr.models.decoder.transformer_decoder import (
+ LightweightConvolution2DTransformerDecoder, # noqa: H301
+)
+from funasr.models.decoder.transformer_decoder import (
+ LightweightConvolutionTransformerDecoder, # noqa: H301
+)
+from funasr.models.decoder.transformer_decoder import TransformerDecoder
+from funasr.models.encoder.abs_encoder import AbsEncoder
+from funasr.models.encoder.conformer_encoder import ConformerEncoder
+from funasr.models.encoder.data2vec_encoder import Data2VecEncoder
+from funasr.models.encoder.rnn_encoder import RNNEncoder
+from funasr.models.encoder.transformer_encoder import TransformerEncoder
+from funasr.models.frontend.abs_frontend import AbsFrontend
+from funasr.models.frontend.default import DefaultFrontend
+from funasr.models.frontend.fused import FusedFrontends
+from funasr.models.frontend.wav_frontend import WavFrontend
+from funasr.models.frontend.s3prl import S3prlFrontend
+from funasr.models.frontend.windowing import SlidingWindow
+from funasr.models.postencoder.abs_postencoder import AbsPostEncoder
+from funasr.models.postencoder.hugging_face_transformers_postencoder import (
+ HuggingFaceTransformersPostEncoder, # noqa: H301
+)
+from funasr.models.preencoder.abs_preencoder import AbsPreEncoder
+from funasr.models.preencoder.linear import LinearProjection
+from funasr.models.preencoder.sinc import LightweightSincConvs
+from funasr.models.specaug.abs_specaug import AbsSpecAug
+from funasr.models.specaug.specaug import SpecAug
+from funasr.layers.abs_normalize import AbsNormalize
+from funasr.layers.global_mvn import GlobalMVN
+from funasr.layers.utterance_mvn import UtteranceMVN
+from funasr.tasks.abs_task import AbsTask
+from funasr.text.phoneme_tokenizer import g2p_choices
+from funasr.train.abs_espnet_model import AbsESPnetModel
+from funasr.train.class_choices import ClassChoices
+from funasr.train.trainer import Trainer
+from funasr.utils.get_default_kwargs import get_default_kwargs
+from funasr.utils.nested_dict_action import NestedDictAction
+from funasr.utils.types import float_or_none
+from funasr.utils.types import int_or_none
+from funasr.utils.types import str2bool
+from funasr.utils.types import str_or_none
+
+from funasr.models.specaug.specaug import SpecAugLFR
+from funasr.models.predictor.cif import CifPredictor, CifPredictorV2
+from funasr.modules.subsampling import Conv1dSubsampling
+from funasr.models.e2e_vad import E2EVadModel
+from funasr.models.encoder.fsmn_encoder import FSMN
+
+frontend_choices = ClassChoices(
+ name="frontend",
+ classes=dict(
+ default=DefaultFrontend,
+ sliding_window=SlidingWindow,
+ s3prl=S3prlFrontend,
+ fused=FusedFrontends,
+ wav_frontend=WavFrontend,
+ ),
+ type_check=AbsFrontend,
+ default="default",
+)
+specaug_choices = ClassChoices(
+ name="specaug",
+ classes=dict(
+ specaug=SpecAug,
+ specaug_lfr=SpecAugLFR,
+ ),
+ type_check=AbsSpecAug,
+ default=None,
+ optional=True,
+)
+normalize_choices = ClassChoices(
+ "normalize",
+ classes=dict(
+ global_mvn=GlobalMVN,
+ utterance_mvn=UtteranceMVN,
+ ),
+ type_check=AbsNormalize,
+ default=None,
+ optional=True,
+)
+model_choices = ClassChoices(
+ "model",
+ classes=dict(
+ e2evad=E2EVadModel,
+ ),
+ type_check=object,
+ default="e2evad",
+)
+
+encoder_choices = ClassChoices(
+ "encoder",
+ classes=dict(
+ fsmn=FSMN,
+ ),
+ type_check=torch.nn.Module,
+ default="fsmn",
+)
+
+
+class VADTask(AbsTask):
+ # If you need more than one optimizers, change this value
+ num_optimizers: int = 1
+
+ # Add variable objects configurations
+ class_choices_list = [
+ # --frontend and --frontend_conf
+ frontend_choices,
+ # --model and --model_conf
+ model_choices,
+ ]
+
+ # If you need to modify train() or eval() procedures, change Trainer class here
+ trainer = Trainer
+
+ @classmethod
+ def add_task_arguments(cls, parser: argparse.ArgumentParser):
+ group = parser.add_argument_group(description="Task related")
+
+ # NOTE(kamo): add_arguments(..., required=True) can't be used
+ # to provide --print_config mode. Instead of it, do as
+ # required = parser.get_default("required")
+ # required += ["token_list"]
+
+ group.add_argument(
+ "--init",
+ type=lambda x: str_or_none(x.lower()),
+ default=None,
+ help="The initialization method",
+ choices=[
+ "chainer",
+ "xavier_uniform",
+ "xavier_normal",
+ "kaiming_uniform",
+ "kaiming_normal",
+ None,
+ ],
+ )
+
+ group.add_argument(
+ "--input_size",
+ type=int_or_none,
+ default=None,
+ help="The number of input dimension of the feature",
+ )
+
+ group = parser.add_argument_group(description="Preprocess related")
+ parser.add_argument(
+ "--speech_volume_normalize",
+ type=float_or_none,
+ default=None,
+ help="Scale the maximum amplitude to the given value.",
+ )
+ parser.add_argument(
+ "--rir_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of rir scp file.",
+ )
+ parser.add_argument(
+ "--rir_apply_prob",
+ type=float,
+ default=1.0,
+ help="THe probability for applying RIR convolution.",
+ )
+ parser.add_argument(
+ "--cmvn_file",
+ type=str_or_none,
+ default=None,
+ help="The file path of noise scp file.",
+ )
+ parser.add_argument(
+ "--noise_scp",
+ type=str_or_none,
+ default=None,
+ help="The file path of noise scp file.",
+ )
+ parser.add_argument(
+ "--noise_apply_prob",
+ type=float,
+ default=1.0,
+ help="The probability applying Noise adding.",
+ )
+ parser.add_argument(
+ "--noise_db_range",
+ type=str,
+ default="13_15",
+ help="The range of noise decibel level.",
+ )
+
+ for class_choices in cls.class_choices_list:
+ # Append --<name> and --<name>_conf.
+ # e.g. --encoder and --encoder_conf
+ class_choices.add_arguments(group)
+
+ @classmethod
+ def build_collate_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Callable[
+ [Collection[Tuple[str, Dict[str, np.ndarray]]]],
+ Tuple[List[str], Dict[str, torch.Tensor]],
+ ]:
+ assert check_argument_types()
+ # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol
+ return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
+
+ @classmethod
+ def build_preprocess_fn(
+ cls, args: argparse.Namespace, train: bool
+ ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
+ assert check_argument_types()
+ #if args.use_preprocessor:
+ # retval = CommonPreprocessor(
+ # train=train,
+ # # NOTE(kamo): Check attribute existence for backward compatibility
+ # rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None,
+ # rir_apply_prob=args.rir_apply_prob
+ # if hasattr(args, "rir_apply_prob")
+ # else 1.0,
+ # noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None,
+ # noise_apply_prob=args.noise_apply_prob
+ # if hasattr(args, "noise_apply_prob")
+ # else 1.0,
+ # noise_db_range=args.noise_db_range
+ # if hasattr(args, "noise_db_range")
+ # else "13_15",
+ # speech_volume_normalize=args.speech_volume_normalize
+ # if hasattr(args, "rir_scp")
+ # else None,
+ # )
+ #else:
+ # retval = None
+ retval = None
+ assert check_return_type(retval)
+ return retval
+
+ @classmethod
+ def required_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ if not inference:
+ retval = ("speech", "text")
+ else:
+ # Recognition mode
+ retval = ("speech",)
+ return retval
+
+ @classmethod
+ def optional_data_names(
+ cls, train: bool = True, inference: bool = False
+ ) -> Tuple[str, ...]:
+ retval = ()
+ assert check_return_type(retval)
+ return retval
+
+ @classmethod
+ def build_model(cls, args: argparse.Namespace):
+ assert check_argument_types()
+ # 4. Encoder
+ encoder_class = encoder_choices.get_class(args.encoder)
+ encoder = encoder_class(**args.encoder_conf)
+
+ # 5. Build model
+ try:
+ model_class = model_choices.get_class(args.model)
+ except AttributeError:
+ model_class = model_choices.get_class("e2evad")
+ model = model_class(encoder=encoder, vad_post_args=args.vad_post_conf)
+
+ return model
+
+ # ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~
+ @classmethod
+ def build_model_from_file(
+ cls,
+ config_file: Union[Path, str] = None,
+ model_file: Union[Path, str] = None,
+ device: str = "cpu",
+ ):
+ """Build model from the files.
+
+ This method is used for inference or fine-tuning.
+
+ Args:
+ config_file: The yaml file saved when training.
+ model_file: The model file saved when training.
+ device: Device type, "cpu", "cuda", or "cuda:N".
+
+ """
+ assert check_argument_types()
+ if config_file is None:
+ assert model_file is not None, (
+ "The argument 'model_file' must be provided "
+ "if the argument 'config_file' is not specified."
+ )
+ config_file = Path(model_file).parent / "config.yaml"
+ else:
+ config_file = Path(config_file)
+
+ with config_file.open("r", encoding="utf-8") as f:
+ args = yaml.safe_load(f)
+ args = argparse.Namespace(**args)
+ model = cls.build_model(args)
+ model.to(device)
+ model_dict = dict()
+ model_name_pth = None
+ if model_file is not None:
+ logging.info("model_file is {}".format(model_file))
+ if device == "cuda":
+ device = f"cuda:{torch.cuda.current_device()}"
+ model_dir = os.path.dirname(model_file)
+ model_name = os.path.basename(model_file)
+ model_dict = torch.load(model_file, map_location=device)
+ model.encoder.load_state_dict(model_dict)
+
+ return model, args
diff --git a/funasr/train/distributed_utils.py b/funasr/train/distributed_utils.py
index 088203a..c897930 100644
--- a/funasr/train/distributed_utils.py
+++ b/funasr/train/distributed_utils.py
@@ -60,8 +60,6 @@
self.dist_rank = torch.distributed.get_rank()
self.dist_world_size = torch.distributed.get_world_size()
self.local_rank = args.local_rank
- logging.info("world size: {}, rank: {}, local_rank: {}".format(self.dist_world_size, self.dist_rank,
- self.local_rank))
def init_options_pai(self):
if self.distributed:
@@ -119,8 +117,6 @@
self.dist_rank = torch.distributed.get_rank()
self.dist_world_size = torch.distributed.get_world_size()
self.local_rank = args.local_rank
- logging.info("world size: {}, rank: {}, local_rank: {}".format(self.dist_world_size, self.dist_rank,
- self.local_rank))
def resolve_distributed_mode(args):
diff --git a/funasr/utils/asr_utils.py b/funasr/utils/asr_utils.py
index 4258f05..0bf903c 100644
--- a/funasr/utils/asr_utils.py
+++ b/funasr/utils/asr_utils.py
@@ -49,12 +49,20 @@
r_audio_format = 'pcm'
r_recog_type = 'wav'
- if r_recog_type is None:
+ if audio_in is None:
+ # for raw_inputs
+ r_recog_type = 'wav'
+ r_audio_format = 'pcm'
+
+ if r_recog_type is None and audio_in is not None:
# audio_in is wav, recog_type is wav_file
if os.path.isfile(audio_in):
if audio_in.endswith('.wav') or audio_in.endswith('.WAV'):
r_recog_type = 'wav'
r_audio_format = 'wav'
+ elif audio_in.endswith('.scp') or audio_in.endswith('.SCP'):
+ r_recog_type = 'wav'
+ r_audio_format = 'scp'
# recog_type is datasets_file
elif os.path.isdir(audio_in):
diff --git a/funasr/utils/compute_eer.py b/funasr/utils/compute_eer.py
new file mode 100644
index 0000000..e2ea0d8
--- /dev/null
+++ b/funasr/utils/compute_eer.py
@@ -0,0 +1,59 @@
+import numpy as np
+from sklearn.metrics import roc_curve
+import argparse
+
+
+def _compute_eer(label, pred, positive_label=1):
+ """
+ Python compute equal error rate (eer)
+ ONLY tested on binary classification
+
+ :param label: ground-truth label, should be a 1-d list or np.array, each element represents the ground-truth label of one sample
+ :param pred: model prediction, should be a 1-d list or np.array, each element represents the model prediction of one sample
+ :param positive_label: the class that is viewed as positive class when computing EER
+ :return: equal error rate (EER)
+ """
+
+ # all fpr, tpr, fnr, fnr, threshold are lists (in the format of np.array)
+ fpr, tpr, threshold = roc_curve(label, pred, pos_label=positive_label)
+ fnr = 1 - tpr
+
+ # the threshold of fnr == fpr
+ eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
+
+ # theoretically eer from fpr and eer from fnr should be identical but they can be slightly differ in reality
+ eer_1 = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
+ eer_2 = fnr[np.nanargmin(np.absolute((fnr - fpr)))]
+
+ # return the mean of eer from fpr and from fnr
+ eer = (eer_1 + eer_2) / 2
+ return eer, eer_threshold
+
+
+def compute_eer(trials_path, scores_path):
+ labels = []
+ for one_line in open(trials_path, "r"):
+ labels.append(one_line.strip().rsplit(" ", 1)[-1] == "target")
+ labels = np.array(labels, dtype=int)
+
+ scores = []
+ for one_line in open(scores_path, "r"):
+ scores.append(float(one_line.strip().rsplit(" ", 1)[-1]))
+ scores = np.array(scores, dtype=float)
+
+ eer, threshold = _compute_eer(labels, scores)
+ return eer, threshold
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("trials", help="trial list")
+ parser.add_argument("scores", help="score file, normalized to [0, 1]")
+ args = parser.parse_args()
+
+ eer, threshold = compute_eer(args.trials, args.scores)
+ print("EER is {:.4f} at threshold {:.4f}".format(eer * 100.0, threshold))
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/funasr/utils/compute_min_dcf.py b/funasr/utils/compute_min_dcf.py
new file mode 100644
index 0000000..610113a
--- /dev/null
+++ b/funasr/utils/compute_min_dcf.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+# Copyright 2018 David Snyder
+# Apache 2.0
+
+# This script computes the minimum detection cost function, which is a common
+# error metric used in speaker recognition. Compared to equal error-rate,
+# which assigns equal weight to false negatives and false positives, this
+# error-rate is usually used to assess performance in settings where achieving
+# a low false positive rate is more important than achieving a low false
+# negative rate. See the NIST 2016 Speaker Recognition Evaluation Plan at
+# https://www.nist.gov/sites/default/files/documents/2016/10/07/sre16_eval_plan_v1.3.pdf
+# for more details about the metric.
+from __future__ import print_function
+from operator import itemgetter
+import sys, argparse, os
+
+
+def GetArgs():
+ parser = argparse.ArgumentParser(description="Compute the minimum "
+ "detection cost function along with the threshold at which it occurs. "
+ "Usage: sid/compute_min_dcf.py [options...] <scores-file> "
+ "<trials-file> "
+ "E.g., sid/compute_min_dcf.py --p-target 0.01 --c-miss 1 --c-fa 1 "
+ "exp/scores/trials data/test/trials",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--p-target', type=float, dest="p_target",
+ default=0.01,
+ help='The prior probability of the target speaker in a trial.')
+ parser.add_argument('--c-miss', type=float, dest="c_miss", default=1,
+ help='Cost of a missed detection. This is usually not changed.')
+ parser.add_argument('--c-fa', type=float, dest="c_fa", default=1,
+ help='Cost of a spurious detection. This is usually not changed.')
+ parser.add_argument("scores_filename",
+ help="Input scores file, with columns of the form "
+ "<utt1> <utt2> <score>")
+ parser.add_argument("trials_filename",
+ help="Input trials file, with columns of the form "
+ "<utt1> <utt2> <target/nontarget>")
+ sys.stderr.write(' '.join(sys.argv) + "\n")
+ args = parser.parse_args()
+ args = CheckArgs(args)
+ return args
+
+
+def CheckArgs(args):
+ if args.c_fa <= 0:
+ raise Exception("--c-fa must be greater than 0")
+ if args.c_miss <= 0:
+ raise Exception("--c-miss must be greater than 0")
+ if args.p_target <= 0 or args.p_target >= 1:
+ raise Exception("--p-target must be greater than 0 and less than 1")
+ return args
+
+
+# Creates a list of false-negative rates, a list of false-positive rates
+# and a list of decision thresholds that give those error-rates.
+def ComputeErrorRates(scores, labels):
+
+ # Sort the scores from smallest to largest, and also get the corresponding
+ # indexes of the sorted scores. We will treat the sorted scores as the
+ # thresholds at which the the error-rates are evaluated.
+ sorted_indexes, thresholds = zip(*sorted(
+ [(index, threshold) for index, threshold in enumerate(scores)],
+ key=itemgetter(1)))
+ labels = [labels[i] for i in sorted_indexes]
+ fns = []
+ tns = []
+
+ # At the end of this loop, fns[i] is the number of errors made by
+ # incorrectly rejecting scores less than thresholds[i]. And, tns[i]
+ # is the total number of times that we have correctly rejected scores
+ # less than thresholds[i].
+ for i in range(0, len(labels)):
+ if i == 0:
+ fns.append(labels[i])
+ tns.append(1 - labels[i])
+ else:
+ fns.append(fns[i-1] + labels[i])
+ tns.append(tns[i-1] + 1 - labels[i])
+ positives = sum(labels)
+ negatives = len(labels) - positives
+
+ # Now divide the false negatives by the total number of
+ # positives to obtain the false negative rates across
+ # all thresholds
+ fnrs = [fn / float(positives) for fn in fns]
+
+ # Divide the true negatives by the total number of
+ # negatives to get the true negative rate. Subtract these
+ # quantities from 1 to get the false positive rates.
+ fprs = [1 - tn / float(negatives) for tn in tns]
+ return fnrs, fprs, thresholds
+
+
+# Computes the minimum of the detection cost function. The comments refer to
+# equations in Section 3 of the NIST 2016 Speaker Recognition Evaluation Plan.
+def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):
+ min_c_det = float("inf")
+ min_c_det_threshold = thresholds[0]
+ for i in range(0, len(fnrs)):
+ # See Equation (2). it is a weighted sum of false negative
+ # and false positive errors.
+ c_det = c_miss * fnrs[i] * p_target + c_fa * fprs[i] * (1 - p_target)
+ if c_det < min_c_det:
+ min_c_det = c_det
+ min_c_det_threshold = thresholds[i]
+ # See Equations (3) and (4). Now we normalize the cost.
+ c_def = min(c_miss * p_target, c_fa * (1 - p_target))
+ min_dcf = min_c_det / c_def
+ return min_dcf, min_c_det_threshold
+
+
+def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_target=0.01):
+ scores_file = open(scores_filename, 'r').readlines()
+ trials_file = open(trials_filename, 'r').readlines()
+ c_miss = c_miss
+ c_fa = c_fa
+ p_target = p_target
+
+ scores = []
+ labels = []
+
+ trials = {}
+ for line in trials_file:
+ utt1, utt2, target = line.rstrip().split()
+ trial = utt1 + " " + utt2
+ trials[trial] = target
+
+ for line in scores_file:
+ utt1, utt2, score = line.rstrip().split()
+ trial = utt1 + " " + utt2
+ if trial in trials:
+ scores.append(float(score))
+ if trials[trial] == "target":
+ labels.append(1)
+ else:
+ labels.append(0)
+ else:
+ raise Exception("Missing entry for " + utt1 + " and " + utt2
+ + " " + scores_filename)
+
+ fnrs, fprs, thresholds = ComputeErrorRates(scores, labels)
+ mindcf, threshold = ComputeMinDcf(fnrs, fprs, thresholds, p_target,
+ c_miss, c_fa)
+ return mindcf, threshold
+
+
+def main():
+ args = GetArgs()
+ mindcf, threshold = compute_min_dcf(
+ args.scores_filename, args.trials_filename,
+ args.c_miss, args.c_fa, args.p_target
+ )
+ sys.stdout.write("minDCF is {0:.4f} at threshold {1:.4f} (p-target={2}, c-miss={3}, "
+ "c-fa={4})\n".format(mindcf, threshold, args.p_target, args.c_miss, args.c_fa))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/egs_modelscope/common/utils/compute_wer.py b/funasr/utils/compute_wer.py
similarity index 100%
copy from egs_modelscope/common/utils/compute_wer.py
copy to funasr/utils/compute_wer.py
diff --git a/funasr/utils/modelscope_param.py b/funasr/utils/modelscope_param.py
new file mode 100644
index 0000000..5d6bffb
--- /dev/null
+++ b/funasr/utils/modelscope_param.py
@@ -0,0 +1,25 @@
+
+class modelscope_args():
+ def __init__(self,
+ task: str = "",
+ model: str = "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ data_path: str = None,
+ output_dir: str = None,
+ model_revision: str = None,
+ dataset_type: str = "small",
+ batch_bins: int = 2000,
+ max_epoch: int = None,
+ lr: float = None,
+ ):
+ self.task = task
+ self.model = model
+ self.data_path = data_path
+ self.output_dir = output_dir
+ self.model_revision = model_revision
+ self.dataset_type = dataset_type
+ self.batch_bins = batch_bins
+ self.max_epoch = max_epoch
+ self.lr = lr
+
+
+
\ No newline at end of file
diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.py
index b18cd19..4da0d59 100644
--- a/funasr/utils/postprocess_utils.py
+++ b/funasr/utils/postprocess_utils.py
@@ -1,6 +1,7 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import string
+import logging
from typing import Any, List, Union
@@ -47,12 +48,16 @@
return True
-def abbr_dispose(words: List[Any]) -> List[Any]:
+# def abbr_dispose(words: List[Any]) -> List[Any]:
+def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:
words_size = len(words)
word_lists = []
abbr_begin = []
abbr_end = []
last_num = -1
+ ts_lists = []
+ ts_nums = []
+ ts_index = 0
for num in range(words_size):
if num <= last_num:
continue
@@ -83,12 +88,20 @@
else:
break
+ for num in range(words_size):
+ if words[num] == ' ':
+ ts_nums.append(ts_index)
+ else:
+ ts_nums.append(ts_index)
+ ts_index += 1
last_num = -1
for num in range(words_size):
if num <= last_num:
continue
if num in abbr_begin:
+ if time_stamp is not None:
+ begin = time_stamp[ts_nums[num]][0]
word_lists.append(words[num].upper())
num += 1
while num < words_size:
@@ -100,16 +113,28 @@
if words[num].encode('utf-8').isalpha():
word_lists.append(words[num].upper())
num += 1
+ if time_stamp is not None:
+ end = time_stamp[ts_nums[num]][1]
+ ts_lists.append([begin, end])
else:
word_lists.append(words[num])
+ if time_stamp is not None and words[num] != ' ':
+ begin = time_stamp[ts_nums[num]][0]
+ end = time_stamp[ts_nums[num]][1]
+ ts_lists.append([begin, end])
+ begin = end
- return word_lists
+ if time_stamp is not None:
+ return word_lists, ts_lists
+ else:
+ return word_lists
-def sentence_postprocess(words: List[Any]):
+def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
middle_lists = []
word_lists = []
word_item = ''
+ ts_lists = []
# wash words lists
for i in words:
@@ -126,45 +151,86 @@
# all chinese characters
if isAllChinese(middle_lists):
- for ch in middle_lists:
+ for i, ch in enumerate(middle_lists):
word_lists.append(ch.replace(' ', ''))
+ if time_stamp is not None:
+ ts_lists = time_stamp
# all alpha characters
elif isAllAlpha(middle_lists):
- for ch in middle_lists:
+ ts_flag = True
+ for i, ch in enumerate(middle_lists):
+ if ts_flag and time_stamp is not None:
+ begin = time_stamp[i][0]
+ end = time_stamp[i][1]
word = ''
if '@@' in ch:
word = ch.replace('@@', '')
word_item += word
+ if time_stamp is not None:
+ ts_flag = False
+ end = time_stamp[i][1]
else:
word_item += ch
word_lists.append(word_item)
word_lists.append(' ')
word_item = ''
+ if time_stamp is not None:
+ ts_flag = True
+ end = time_stamp[i][1]
+ ts_lists.append([begin, end])
+ begin = end
# mix characters
else:
alpha_blank = False
- for ch in middle_lists:
+ ts_flag = True
+ begin = -1
+ end = -1
+ for i, ch in enumerate(middle_lists):
+ if ts_flag and time_stamp is not None:
+ begin = time_stamp[i][0]
+ end = time_stamp[i][1]
word = ''
if isAllChinese(ch):
if alpha_blank is True:
word_lists.pop()
word_lists.append(ch)
alpha_blank = False
+ if time_stamp is not None:
+ ts_flag = True
+ ts_lists.append([begin, end])
+ begin = end
elif '@@' in ch:
word = ch.replace('@@', '')
word_item += word
alpha_blank = False
+ if time_stamp is not None:
+ ts_flag = False
+ end = time_stamp[i][1]
elif isAllAlpha(ch):
word_item += ch
word_lists.append(word_item)
word_lists.append(' ')
word_item = ''
alpha_blank = True
+ if time_stamp is not None:
+ ts_flag = True
+ end = time_stamp[i][1]
+ ts_lists.append([begin, end])
+ begin = end
else:
raise ValueError('invalid character: {}'.format(ch))
- word_lists = abbr_dispose(word_lists)
- sentence = ''.join(word_lists).strip()
- return sentence
+ if time_stamp is not None:
+ word_lists, ts_lists = abbr_dispose(word_lists, ts_lists)
+ real_word_lists = []
+ for ch in word_lists:
+ if ch != ' ':
+ real_word_lists.append(ch)
+ sentence = ' '.join(real_word_lists).strip()
+ return sentence, ts_lists, real_word_lists
+ else:
+ word_lists = abbr_dispose(word_lists)
+ sentence = ''.join(word_lists).strip()
+ return sentence
diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.py
new file mode 100644
index 0000000..3afaa40
--- /dev/null
+++ b/funasr/utils/timestamp_tools.py
@@ -0,0 +1,99 @@
+import torch
+import copy
+import logging
+import numpy as np
+from typing import Any, List, Tuple, Union
+
+def cut_interval(alphas: torch.Tensor, start: int, end: int, tail: bool):
+ if not tail:
+ if end == start + 1:
+ cut = (end + start) / 2.0
+ else:
+ alpha = alphas[start+1: end].tolist()
+ reverse_steps = 1
+ for reverse_alpha in alpha[::-1]:
+ if reverse_alpha > 0.35:
+ reverse_steps += 1
+ else:
+ break
+ cut = end - reverse_steps
+ else:
+ if end != len(alphas) - 1:
+ cut = end + 1
+ else:
+ cut = start + 1
+ return float(cut)
+
+def time_stamp_lfr6(alphas: torch.Tensor, speech_lengths: torch.Tensor, raw_text: List[str], begin: int = 0, end: int = None):
+ time_stamp_list = []
+ alphas = alphas[0]
+ text = copy.deepcopy(raw_text)
+ if end is None:
+ time = speech_lengths * 60 / 1000
+ sacle_rate = (time / speech_lengths[0]).tolist()
+ else:
+ time = (end - begin) / 1000
+ sacle_rate = (time / speech_lengths[0]).tolist()
+
+ predictor = (alphas > 0.5).int()
+ fire_places = torch.nonzero(predictor == 1).squeeze(1).tolist()
+
+ cuts = []
+ npeak = int(predictor.sum())
+ nchar = len(raw_text)
+ if npeak - 1 == nchar:
+ fire_places = torch.where((alphas > 0.5) == 1)[0].tolist()
+ for i in range(len(fire_places)):
+ if fire_places[i] < len(alphas) - 1:
+ if 0.05 < alphas[fire_places[i]+1] < 0.5:
+ fire_places[i] += 1
+ elif npeak < nchar:
+ lost_num = nchar - npeak
+ lost_fire = speech_lengths[0].tolist() - fire_places[-1]
+ interval_distance = lost_fire // (lost_num + 1)
+ for i in range(1, lost_num + 1):
+ fire_places.append(fire_places[-1] + interval_distance)
+ elif npeak - 1 > nchar:
+ redundance_num = npeak - 1 - nchar
+ for i in range(redundance_num):
+ fire_places.pop()
+
+ cuts.append(0)
+ start_sil = True
+ if start_sil:
+ text.insert(0, '<sil>')
+
+ for i in range(len(fire_places)-1):
+ cuts.append(cut_interval(alphas, fire_places[i], fire_places[i+1], tail=(i==len(fire_places)-2)))
+
+ for i in range(2, len(fire_places)-2):
+ if fire_places[i-2] == fire_places[i-1] - 1 and fire_places[i-1] != fire_places[i] - 1:
+ cuts[i-1] += 1
+
+ if cuts[-1] != len(alphas) - 1:
+ text.append('<sil>')
+ cuts.append(speech_lengths[0].tolist())
+ cuts.insert(-1, (cuts[-1] + cuts[-2]) * 0.5)
+ sec_fire_places = np.array(cuts) * sacle_rate
+ for i in range(1, len(sec_fire_places) - 1):
+ start, end = sec_fire_places[i], sec_fire_places[i+1]
+ if i == len(sec_fire_places) - 2:
+ end = time
+ time_stamp_list.append([int(round(start, 2) * 1000) + begin, int(round(end, 2) * 1000) + begin])
+ text = text[1:]
+ if npeak - 1 == nchar or npeak > nchar:
+ return time_stamp_list[:-1]
+ else:
+ return time_stamp_list
+
+
+def time_stamp_lfr6_advance(tst: List, text: str):
+ # advanced timestamp prediction for BiCIF_Paraformer using upsampled alphas
+ ds_alphas, ds_cif_peak, us_alphas, us_cif_peak = tst
+ if text.endswith('</s>'):
+ text = text[:-4]
+ else:
+ text = text[:-1]
+ logging.warning("found text does not end with </s>")
+ assert int(ds_alphas.sum() + 1e-4) - 1 == len(text)
+
diff --git a/funasr/utils/wav_utils.py b/funasr/utils/wav_utils.py
index d8564f2..c86c7d2 100644
--- a/funasr/utils/wav_utils.py
+++ b/funasr/utils/wav_utils.py
@@ -2,6 +2,8 @@
import math
import os
+import shutil
+from multiprocessing import Pool
from typing import Any, Dict, Union
import kaldiio
@@ -152,7 +154,7 @@
raise TypeError("'dtype' must be a floating point type")
i = np.iinfo(middle_data.dtype)
- abs_max = 2**(i.bits - 1)
+ abs_max = 2 ** (i.bits - 1)
offset = i.min + abs_max
waveform = np.frombuffer(
(middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32)
@@ -176,3 +178,111 @@
input_feats = mat
return input_feats
+
+
+def wav2num_frame(wav_path, frontend_conf):
+ waveform, sampling_rate = torchaudio.load(wav_path)
+ speech_length = (waveform.shape[1] / sampling_rate) * 1000.
+ n_frames = (waveform.shape[1] * 1000.0) / (sampling_rate * frontend_conf["frame_shift"] * frontend_conf["lfr_n"])
+ feature_dim = frontend_conf["n_mels"] * frontend_conf["lfr_m"]
+ return n_frames, feature_dim, speech_length
+
+
+def calc_shape_core(root_path, frontend_conf, speech_length_min, speech_length_max, idx):
+ wav_scp_file = os.path.join(root_path, "wav.scp.{}".format(idx))
+ shape_file = os.path.join(root_path, "speech_shape.{}".format(idx))
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ with open(shape_file, "w") as f:
+ for line in lines:
+ sample_name, wav_path = line.strip().split()
+ n_frames, feature_dim, speech_length = wav2num_frame(wav_path, frontend_conf)
+ write_flag = True
+ if speech_length_min > 0 and speech_length < speech_length_min:
+ write_flag = False
+ if speech_length_max > 0 and speech_length > speech_length_max:
+ write_flag = False
+ if write_flag:
+ f.write("{} {},{}\n".format(sample_name, str(int(np.ceil(n_frames))), str(int(feature_dim))))
+
+
+def calc_shape(data_dir, dataset, frontend_conf, speech_length_min=-1, speech_length_max=-1, nj=32):
+ shape_path = os.path.join(data_dir, dataset, "shape_files")
+ if os.path.exists(shape_path):
+ assert os.path.exists(os.path.join(data_dir, dataset, "speech_shape"))
+ print('Shape file for small dataset already exists.')
+ return
+ os.makedirs(shape_path, exist_ok=True)
+
+ # split
+ wav_scp_file = os.path.join(data_dir, dataset, "wav.scp")
+ with open(wav_scp_file) as f:
+ lines = f.readlines()
+ num_lines = len(lines)
+ num_job_lines = num_lines // nj
+ start = 0
+ for i in range(nj):
+ end = start + num_job_lines
+ file = os.path.join(shape_path, "wav.scp.{}".format(str(i + 1)))
+ with open(file, "w") as f:
+ if i == nj - 1:
+ f.writelines(lines[start:])
+ else:
+ f.writelines(lines[start:end])
+ start = end
+
+ p = Pool(nj)
+ for i in range(nj):
+ p.apply_async(calc_shape_core,
+ args=(shape_path, frontend_conf, speech_length_min, speech_length_max, str(i + 1)))
+ print('Generating shape files, please wait a few minutes...')
+ p.close()
+ p.join()
+
+ # combine
+ file = os.path.join(data_dir, dataset, "speech_shape")
+ with open(file, "w") as f:
+ for i in range(nj):
+ job_file = os.path.join(shape_path, "speech_shape.{}".format(str(i + 1)))
+ with open(job_file) as job_f:
+ lines = job_f.readlines()
+ f.writelines(lines)
+ print('Generating shape files done.')
+
+
+def generate_data_list(data_dir, dataset, nj=100):
+ split_dir = os.path.join(data_dir, dataset, "split")
+ if os.path.exists(split_dir):
+ assert os.path.exists(os.path.join(data_dir, dataset, "data.list"))
+ print('Data list for large dataset already exists.')
+ return
+ os.makedirs(split_dir, exist_ok=True)
+
+ with open(os.path.join(data_dir, dataset, "wav.scp")) as f_wav:
+ wav_lines = f_wav.readlines()
+ with open(os.path.join(data_dir, dataset, "text")) as f_text:
+ text_lines = f_text.readlines()
+ total_num_lines = len(wav_lines)
+ num_lines = total_num_lines // nj
+ start_num = 0
+ for i in range(nj):
+ end_num = start_num + num_lines
+ split_dir_nj = os.path.join(split_dir, str(i + 1))
+ os.mkdir(split_dir_nj)
+ wav_file = os.path.join(split_dir_nj, 'wav.scp')
+ text_file = os.path.join(split_dir_nj, "text")
+ with open(wav_file, "w") as fw, open(text_file, "w") as ft:
+ if i == nj - 1:
+ fw.writelines(wav_lines[start_num:])
+ ft.writelines(text_lines[start_num:])
+ else:
+ fw.writelines(wav_lines[start_num:end_num])
+ ft.writelines(text_lines[start_num:end_num])
+ start_num = end_num
+
+ data_list_file = os.path.join(data_dir, dataset, "data.list")
+ with open(data_list_file, "w") as f_data:
+ for i in range(nj):
+ wav_path = os.path.join(split_dir, str(i + 1), "wav.scp")
+ text_path = os.path.join(split_dir, str(i + 1), "text")
+ f_data.write(wav_path + " " + text_path + "\n")
diff --git a/funasr/version.txt b/funasr/version.txt
index 845639e..c946ee6 100644
--- a/funasr/version.txt
+++ b/funasr/version.txt
@@ -1 +1 @@
-0.1.4
+0.1.6
diff --git a/setup.py b/setup.py
index 767b3da..39272e3 100644
--- a/setup.py
+++ b/setup.py
@@ -38,8 +38,8 @@
"editdistance==0.5.2",
"tensorboard>=1.14",
"g2p",
- # ITN
- "pynini==2.1.5",
+ # PAI
+ "oss2"
],
# train: The modules invoked when training only.
"train": [
@@ -126,7 +126,7 @@
long_description=open(os.path.join(dirname, "README.md"), encoding="utf-8").read(),
long_description_content_type="text/markdown",
license="The MIT License",
- packages=find_packages(include=["funasr*", "fun_text_processing*"]),
+ packages=find_packages(include=["funasr*"]),
package_data={"funasr": ["version.txt"]},
install_requires=install_requires,
setup_requires=setup_requires,
--
Gitblit v1.9.1