.gitignore
@@ -24,3 +24,4 @@ outputs* emotion2vec* GPT-SoVITS* modelscope_models README.md
@@ -95,7 +95,7 @@ ### Command-line usage ```shell funasr +model=paraformer-zh +vad_model="fsmn-vad" +punc_model="ct-punc" +input=asr_example_zh.wav funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++input=asr_example_zh.wav ``` Notes: Support recognition of single audio file, as well as file list in Kaldi-style wav.scp format: `wav_id wav_pat` README_zh.md
@@ -91,7 +91,7 @@ ### 可执行命令行 ```shell funasr +model=paraformer-zh +vad_model="fsmn-vad" +punc_model="ct-punc" +input=asr_example_zh.wav funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++input=asr_example_zh.wav ``` 注：支持单条音频文件识别，也支持文件列表，列表为kaldi风格wav.scp：`wav_id wav_path` examples/aishell/branchformer/demo_infer.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_infer.sh examples/aishell/branchformer/demo_train_or_finetune.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_train_or_finetune.sh examples/aishell/conformer/demo_infer.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_infer.sh examples/aishell/conformer/demo_train_or_finetune.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_train_or_finetune.sh examples/aishell/e_branchformer/demo_infer.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_infer.sh examples/aishell/e_branchformer/demo_train_or_finetune.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_train_or_finetune.sh examples/aishell/e_branchformer/infer.sh
File was deleted examples/aishell/paraformer/demo_infer.sh
File was renamed from examples/aishell/conformer/infer.sh @@ -1,3 +1,6 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) python -m funasr.bin.inference \ examples/aishell/paraformer/demo_train_or_finetune.sh
New file @@ -0,0 +1,51 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}') # data dir, which contains: train.json, val.json, tokens.jsonl/tokens.txt, am.mvn data_dir="/Users/zhifu/funasr1.0/data/list" ## generate jsonl from wav.scp and text.txt #python -m funasr.datasets.audio_datasets.scp2jsonl \ #++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \ #++data_type_list='["source", "target"]' \ #++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl train_data="${data_dir}/train.jsonl" val_data="${data_dir}/val.jsonl" tokens="${data_dir}/tokens.json" cmvn_file="${data_dir}/am.mvn" # exp output dir output_dir="/Users/zhifu/exp" log_file="${output_dir}/log.txt" workspace=`pwd` config="paraformer_conformer_12e_6d_2048_256.yaml" init_param="${output_dir}/model.pt" mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ --config-path "${workspace}/conf" \ --config-name "${config}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++tokenizer_conf.token_list="${tokens}" \ ++frontend_conf.cmvn_file="${cmvn_file}" \ ++dataset_conf.batch_size=32 \ ++dataset_conf.batch_type="example" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=150 \ ++optim_conf.lr=0.0002 \ ++init_param="${init_param}" \ ++output_dir="${output_dir}" &> ${log_file} examples/aishell/paraformer/infer.sh
File was deleted examples/aishell/transformer/demo_infer.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_infer.sh examples/aishell/transformer/demo_train_or_finetune.sh
New file @@ -0,0 +1 @@ ../paraformer/demo_train_or_finetune.sh examples/aishell/transformer/infer.sh
File was deleted examples/industrial_data_pretraining/bicif_paraformer/demo.sh
examples/industrial_data_pretraining/conformer/demo.sh
examples/industrial_data_pretraining/contextual_paraformer/demo.sh
examples/industrial_data_pretraining/ct_transformer/demo.sh
examples/industrial_data_pretraining/ct_transformer_streaming/demo.sh
examples/industrial_data_pretraining/emotion2vec/infer.sh
File was deleted examples/industrial_data_pretraining/fsmn_vad_streaming/demo.sh
examples/industrial_data_pretraining/monotonic_aligner/demo.sh
examples/industrial_data_pretraining/paraformer-zh-spk/demo.sh
examples/industrial_data_pretraining/paraformer/demo.sh
File was deleted examples/industrial_data_pretraining/paraformer/finetune.sh
@@ -1,9 +1,14 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) ## download model #local_path_root=../modelscope_models #mkdir -p ${local_path_root} #local_path=${local_path_root}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch #git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git ${local_path} # method1, finetune from model hub # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}') # data dir, which contains: train.json, val.json data_dir="/Users/zhifu/funasr1.0/data/list" ## generate jsonl from wav.scp and text.txt #python -m funasr.datasets.audio_datasets.scp2jsonl \ @@ -11,17 +16,29 @@ #++data_type_list='["source", "target"]' \ #++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl train_data="${data_dir}/train.jsonl" val_data="${data_dir}/val.jsonl" # torchrun \ # --nnodes 1 \ # --nproc_per_node 1 \ python funasr/bin/train.py \ +model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \ +model_revision="v2.0.4" \ +train_data_set_list="/Users/zhifu/funasr_github/test_local/aishell2_dev_ios/asr_task_debug_len_10.jsonl" \ +valid_data_set_list="/Users/zhifu/funasr_github/test_local/aishell2_dev_ios/asr_task_debug_len_10.jsonl" \ ++dataset_conf.batch_size=64 \ # exp output dir output_dir="/Users/zhifu/exp" log_file="${output_dir}/log.txt" mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ funasr/bin/train.py \ ++model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \ ++model_revision="v2.0.4" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=32 \ ++dataset_conf.batch_type="example" \ ++train_conf.max_epoch=2 \ ++dataset_conf.num_workers=4 \ +output_dir="outputs/debug/ckpt/funasr2/exp2" ++train_conf.max_epoch=20 \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/paraformer/finetune_from_local.sh
New file @@ -0,0 +1,61 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) # method2, finetune from local model workspace=`pwd` # download model local_path_root=${workspace}/modelscope_models mkdir -p ${local_path_root} local_path=${local_path_root}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git ${local_path} # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}') # data dir, which contains: train.json, val.json data_dir="/Users/zhifu/funasr1.0/data/list" ## generate jsonl from wav.scp and text.txt #python -m funasr.datasets.audio_datasets.scp2jsonl \ #++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \ #++data_type_list='["source", "target"]' \ #++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl train_data="${data_dir}/train.jsonl" val_data="${data_dir}/val.jsonl" tokens="${local_path}/tokens.json" cmvn_file="${local_path}/am.mvn" # exp output dir output_dir="/Users/zhifu/exp" log_file="${output_dir}/log.txt" config="config.yaml" init_param="${local_path}/model.pt" mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ --config-path "${local_path}" \ --config-name "${config}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++tokenizer_conf.token_list="${tokens}" \ ++frontend_conf.cmvn_file="${cmvn_file}" \ ++dataset_conf.batch_size=32 \ ++dataset_conf.batch_type="example" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=20 \ ++optim_conf.lr=0.0002 \ ++init_param="${init_param}" \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/paraformer/infer.sh
New file @@ -0,0 +1,21 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) # method1, inference from model hub # for more input type, please ref to readme.md input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" output_dir="./outputs/debug" model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" model_revision="v2.0.4" device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu" python -m funasr.bin.inference \ ++model=${model} \ ++model_revision=${model_revision} \ ++input="${input}" \ ++output_dir="${output_dir}" \ ++device="${device}" \ examples/industrial_data_pretraining/paraformer/infer_after_finetune.sh
File was deleted examples/industrial_data_pretraining/paraformer/infer_from_local.sh
New file @@ -0,0 +1,39 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) # method2, inference from local model # for more input type, please ref to readme.md input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" output_dir="./outputs/debug" workspace=`pwd` # download model local_path_root=${workspace}/modelscope_models mkdir -p ${local_path_root} local_path=${local_path_root}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git ${local_path} device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu" tokens="${local_path}/tokens.json" cmvn_file="${local_path}/am.mvn" config="config.yaml" init_param="${local_path}/model.pt" python -m funasr.bin.inference \ --config-path "${local_path}" \ --config-name "${config}" \ ++init_param="${init_param}" \ ++tokenizer_conf.token_list="${tokens}" \ ++frontend_conf.cmvn_file="${cmvn_file}" \ ++input="${input}" \ ++output_dir="${output_dir}" \ ++device="${device}" \
examples/industrial_data_pretraining/scama/demo.sh
examples/industrial_data_pretraining/seaco_paraformer/demo.sh
examples/industrial_data_pretraining/transducer/demo.py
New file @@ -0,0 +1,15 @@ #!/usr/bin/env python3 # -*- encoding: utf-8 -*- # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) from funasr import AutoModel # Transducer, BAT and RWKV_BAT models are just same to use, use the correct model_revision # https://modelscope.cn/models?name=transducer&page=1&tasks=auto-speech-recognition&type=audio model = AutoModel(model="iic/speech_bat_asr-zh-cn-16k-aishell1-vocab4234-pytorch", model_revision="v2.0.2", ) res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav") print(res) examples/industrial_data_pretraining/uniasr/demo.sh
funasr/bin/train.py
@@ -96,15 +96,18 @@ init_param = (init_param,) logging.info("init_param is not None: %s", init_param) for p in init_param: logging.info(f"Loading pretrained params from {p}") load_pretrained_model( model=model, path=p, ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True), oss_bucket=kwargs.get("oss_bucket", None), scope_map=kwargs.get("scope_map", None), excludes=kwargs.get("excludes", None), ) if os.path.exists(p): logging.info(f"Loading pretrained params from {p}") load_pretrained_model( model=model, path=p, ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True), oss_bucket=kwargs.get("oss_bucket", None), scope_map=kwargs.get("scope_map", None), excludes=kwargs.get("excludes", None), ) else: logging.info(f"Checkpoint does not exist, init randomly: {p}") else: initialize(model, kwargs.get("init", "kaiming_normal"))
@@ -13,6 +13,7 @@ from funasr.register import tables from funasr.utils import postprocess_utils from funasr.utils.datadir_writer import DatadirWriter from funasr.models.transducer.model import Transducer from funasr.train_utils.device_funcs import force_gatherable from funasr.models.transformer.scorers.ctc import CTCPrefixScorer from funasr.losses.label_smoothing_loss import LabelSmoothingLoss @@ -32,488 +33,5 @@ @tables.register("model_classes", "BAT") # TODO: BAT training class BAT(torch.nn.Module): def __init__( self, frontend: Optional[str] = None, frontend_conf: Optional[Dict] = None, specaug: Optional[str] = None, specaug_conf: Optional[Dict] = None, normalize: str = None, normalize_conf: Optional[Dict] = None, encoder: str = None, encoder_conf: Optional[Dict] = None, decoder: str = None, decoder_conf: Optional[Dict] = None, joint_network: str = None, joint_network_conf: Optional[Dict] = None, transducer_weight: float = 1.0, fastemit_lambda: float = 0.0, auxiliary_ctc_weight: float = 0.0, auxiliary_ctc_dropout_rate: float = 0.0, auxiliary_lm_loss_weight: float = 0.0, auxiliary_lm_loss_smoothing: float = 0.0, input_size: int = 80, vocab_size: int = -1, ignore_id: int = -1, blank_id: int = 0, sos: int = 1, eos: int = 2, lsm_weight: float = 0.0, length_normalized_loss: bool = False, # report_cer: bool = True, # report_wer: bool = True, # sym_space: str = "<space>", # sym_blank: str = "<blank>", # extract_feats_in_collect_stats: bool = True, share_embedding: bool = False, # preencoder: Optional[AbsPreEncoder] = None, # postencoder: Optional[AbsPostEncoder] = None, **kwargs, ): super().__init__() if specaug is not None: specaug_class = tables.specaug_classes.get(specaug) specaug = specaug_class(**specaug_conf) if normalize is not None: normalize_class = tables.normalize_classes.get(normalize) normalize = normalize_class(**normalize_conf) encoder_class = tables.encoder_classes.get(encoder) encoder = encoder_class(input_size=input_size, **encoder_conf) encoder_output_size = encoder.output_size() decoder_class = tables.decoder_classes.get(decoder) decoder = decoder_class( vocab_size=vocab_size, **decoder_conf, ) decoder_output_size = decoder.output_size joint_network_class = tables.joint_network_classes.get(joint_network) joint_network = joint_network_class( vocab_size, encoder_output_size, decoder_output_size, **joint_network_conf, ) self.criterion_transducer = None self.error_calculator = None self.use_auxiliary_ctc = auxiliary_ctc_weight > 0 self.use_auxiliary_lm_loss = auxiliary_lm_loss_weight > 0 if self.use_auxiliary_ctc: self.ctc_lin = torch.nn.Linear(encoder.output_size(), vocab_size) self.ctc_dropout_rate = auxiliary_ctc_dropout_rate if self.use_auxiliary_lm_loss: self.lm_lin = torch.nn.Linear(decoder.output_size, vocab_size) self.lm_loss_smoothing = auxiliary_lm_loss_smoothing self.transducer_weight = transducer_weight self.fastemit_lambda = fastemit_lambda self.auxiliary_ctc_weight = auxiliary_ctc_weight self.auxiliary_lm_loss_weight = auxiliary_lm_loss_weight self.blank_id = blank_id self.sos = sos if sos is not None else vocab_size - 1 self.eos = eos if eos is not None else vocab_size - 1 self.vocab_size = vocab_size self.ignore_id = ignore_id self.frontend = frontend self.specaug = specaug self.normalize = normalize self.encoder = encoder self.decoder = decoder self.joint_network = joint_network self.criterion_att = LabelSmoothingLoss( size=vocab_size, padding_idx=ignore_id, smoothing=lsm_weight, normalize_length=length_normalized_loss, ) self.length_normalized_loss = length_normalized_loss self.beam_search = None self.ctc = None self.ctc_weight = 0.0 def forward( self, speech: torch.Tensor, speech_lengths: torch.Tensor, text: torch.Tensor, text_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: """Encoder + Decoder + Calc loss Args: speech: (Batch, Length, ...) speech_lengths: (Batch, ) text: (Batch, Length) text_lengths: (Batch,) """ if len(text_lengths.size()) > 1: text_lengths = text_lengths[:, 0] if len(speech_lengths.size()) > 1: speech_lengths = speech_lengths[:, 0] batch_size = speech.shape[0] # 1. Encoder encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) if hasattr(self.encoder, 'overlap_chunk_cls') and self.encoder.overlap_chunk_cls is not None: encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens, chunk_outs=None) # 2. Transducer-related I/O preparation decoder_in, target, t_len, u_len = get_transducer_task_io( text, encoder_out_lens, ignore_id=self.ignore_id, ) # 3. Decoder self.decoder.set_device(encoder_out.device) decoder_out = self.decoder(decoder_in, u_len) # 4. Joint Network joint_out = self.joint_network( encoder_out.unsqueeze(2), decoder_out.unsqueeze(1) ) # 5. Losses loss_trans, cer_trans, wer_trans = self._calc_transducer_loss( encoder_out, joint_out, target, t_len, u_len, ) loss_ctc, loss_lm = 0.0, 0.0 if self.use_auxiliary_ctc: loss_ctc = self._calc_ctc_loss( encoder_out, target, t_len, u_len, ) if self.use_auxiliary_lm_loss: loss_lm = self._calc_lm_loss(decoder_out, target) loss = ( self.transducer_weight * loss_trans + self.auxiliary_ctc_weight * loss_ctc + self.auxiliary_lm_loss_weight * loss_lm ) stats = dict( loss=loss.detach(), loss_transducer=loss_trans.detach(), aux_ctc_loss=loss_ctc.detach() if loss_ctc > 0.0 else None, aux_lm_loss=loss_lm.detach() if loss_lm > 0.0 else None, cer_transducer=cer_trans, wer_transducer=wer_trans, ) # force_gatherable: to-device and to-tensor if scalar for DataParallel loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) return loss, stats, weight def encode( self, speech: torch.Tensor, speech_lengths: torch.Tensor, **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: """Frontend + Encoder. Note that this method is used by asr_inference.py Args: speech: (Batch, Length, ...) speech_lengths: (Batch, ) ind: int """ with autocast(False): # Data augmentation if self.specaug is not None and self.training: speech, speech_lengths = self.specaug(speech, speech_lengths) # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN if self.normalize is not None: speech, speech_lengths = self.normalize(speech, speech_lengths) # Forward encoder # feats: (Batch, Length, Dim) # -> encoder_out: (Batch, Length2, Dim2) encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) intermediate_outs = None if isinstance(encoder_out, tuple): intermediate_outs = encoder_out[1] encoder_out = encoder_out[0] if intermediate_outs is not None: return (encoder_out, intermediate_outs), encoder_out_lens return encoder_out, encoder_out_lens def _calc_transducer_loss( self, encoder_out: torch.Tensor, joint_out: torch.Tensor, target: torch.Tensor, t_len: torch.Tensor, u_len: torch.Tensor, ) -> Tuple[torch.Tensor, Optional[float], Optional[float]]: """Compute Transducer loss. Args: encoder_out: Encoder output sequences. (B, T, D_enc) joint_out: Joint Network output sequences (B, T, U, D_joint) target: Target label ID sequences. (B, L) t_len: Encoder output sequences lengths. (B,) u_len: Target label ID sequences lengths. (B,) Return: loss_transducer: Transducer loss value. cer_transducer: Character error rate for Transducer. wer_transducer: Word Error Rate for Transducer. """ if self.criterion_transducer is None: try: from warp_rnnt import rnnt_loss as RNNTLoss self.criterion_transducer = RNNTLoss except ImportError: logging.error( "warp-rnnt was not installed." "Please consult the installation documentation." ) exit(1) log_probs = torch.log_softmax(joint_out, dim=-1) loss_transducer = self.criterion_transducer( log_probs, target, t_len, u_len, reduction="mean", blank=self.blank_id, fastemit_lambda=self.fastemit_lambda, gather=True, ) if not self.training and (self.report_cer or self.report_wer): if self.error_calculator is None: from funasr.metrics import ErrorCalculatorTransducer as ErrorCalculator self.error_calculator = ErrorCalculator( self.decoder, self.joint_network, self.token_list, self.sym_space, self.sym_blank, report_cer=self.report_cer, report_wer=self.report_wer, ) cer_transducer, wer_transducer = self.error_calculator(encoder_out, target, t_len) return loss_transducer, cer_transducer, wer_transducer return loss_transducer, None, None def _calc_ctc_loss( self, encoder_out: torch.Tensor, target: torch.Tensor, t_len: torch.Tensor, u_len: torch.Tensor, ) -> torch.Tensor: """Compute CTC loss. Args: encoder_out: Encoder output sequences. (B, T, D_enc) target: Target label ID sequences. (B, L) t_len: Encoder output sequences lengths. (B,) u_len: Target label ID sequences lengths. (B,) Return: loss_ctc: CTC loss value. """ ctc_in = self.ctc_lin( torch.nn.functional.dropout(encoder_out, p=self.ctc_dropout_rate) ) ctc_in = torch.log_softmax(ctc_in.transpose(0, 1), dim=-1) target_mask = target != 0 ctc_target = target[target_mask].cpu() with torch.backends.cudnn.flags(deterministic=True): loss_ctc = torch.nn.functional.ctc_loss( ctc_in, ctc_target, t_len, u_len, zero_infinity=True, reduction="sum", ) loss_ctc /= target.size(0) return loss_ctc def _calc_lm_loss( self, decoder_out: torch.Tensor, target: torch.Tensor, ) -> torch.Tensor: """Compute LM loss. Args: decoder_out: Decoder output sequences. (B, U, D_dec) target: Target label ID sequences. (B, L) Return: loss_lm: LM loss value. """ lm_loss_in = self.lm_lin(decoder_out[:, :-1, :]).view(-1, self.vocab_size) lm_target = target.view(-1).type(torch.int64) with torch.no_grad(): true_dist = lm_loss_in.clone() true_dist.fill_(self.lm_loss_smoothing / (self.vocab_size - 1)) # Ignore blank ID (0) ignore = lm_target == 0 lm_target = lm_target.masked_fill(ignore, 0) true_dist.scatter_(1, lm_target.unsqueeze(1), (1 - self.lm_loss_smoothing)) loss_lm = torch.nn.functional.kl_div( torch.log_softmax(lm_loss_in, dim=1), true_dist, reduction="none", ) loss_lm = loss_lm.masked_fill(ignore.unsqueeze(1), 0).sum() / decoder_out.size( 0 ) return loss_lm def init_beam_search(self, **kwargs, ): # 1. Build ASR model scorers = {} if self.ctc != None: ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos) scorers.update( ctc=ctc ) token_list = kwargs.get("token_list") scorers.update( length_bonus=LengthBonus(len(token_list)), ) # 3. Build ngram model # ngram is not supported now ngram = None scorers["ngram"] = ngram beam_search = BeamSearchTransducer( self.decoder, self.joint_network, kwargs.get("beam_size", 2), nbest=1, ) # beam_search.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() # for scorer in scorers.values(): # if isinstance(scorer, torch.nn.Module): # scorer.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval() self.beam_search = beam_search def inference(self, data_in: list, data_lengths: list=None, key: list=None, tokenizer=None, **kwargs, ): if kwargs.get("batch_size", 1) > 1: raise NotImplementedError("batch decoding is not implemented") # init beamsearch is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None is_use_lm = kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None # if self.beam_search is None and (is_use_lm or is_use_ctc): logging.info("enable beam_search") self.init_beam_search(**kwargs) self.nbest = kwargs.get("nbest", 1) meta_data = {} # extract fbank feats time1 = time.perf_counter() audio_sample_list = load_audio_text_image_video(data_in, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000)) time2 = time.perf_counter() meta_data["load_data"] = f"{time2 - time1:0.3f}" speech, speech_lengths = extract_fbank(audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=self.frontend) time3 = time.perf_counter() meta_data["extract_feat"] = f"{time3 - time2:0.3f}" meta_data["batch_data_time"] = speech_lengths.sum().item() * self.frontend.frame_shift * self.frontend.lfr_n / 1000 speech = speech.to(device=kwargs["device"]) speech_lengths = speech_lengths.to(device=kwargs["device"]) # Encoder encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) if isinstance(encoder_out, tuple): encoder_out = encoder_out[0] # c. Passed the encoder result and the beam search nbest_hyps = self.beam_search(encoder_out[0], is_final=True) nbest_hyps = nbest_hyps[: self.nbest] results = [] b, n, d = encoder_out.size() for i in range(b): for nbest_idx, hyp in enumerate(nbest_hyps): ibest_writer = None if kwargs.get("output_dir") is not None: if not hasattr(self, "writer"): self.writer = DatadirWriter(kwargs.get("output_dir")) ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] # remove sos/eos and get results last_pos = -1 if isinstance(hyp.yseq, list): token_int = hyp.yseq#[1:last_pos] else: token_int = hyp.yseq#[1:last_pos].tolist() # remove blank symbol id, which is assumed to be 0 token_int = list(filter(lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int)) # Change integer-ids to tokens token = tokenizer.ids2tokens(token_int) text = tokenizer.tokens2text(token) text_postprocessed, _ = postprocess_utils.sentence_postprocess(token) result_i = {"key": key[i], "token": token, "text": text, "text_postprocessed": text_postprocessed} results.append(result_i) if ibest_writer is not None: ibest_writer["token"][key[i]] = " ".join(token) ibest_writer["text"][key[i]] = text ibest_writer["text_postprocessed"][key[i]] = text_postprocessed return results, meta_data class BAT(Transducer): pass funasr/train_utils/load_pretrained_model.py
@@ -96,8 +96,7 @@ obj = model dst_state = obj.state_dict() # import pdb; # pdb.set_trace() print(f"ckpt: {path}") if oss_bucket is None: src_state = torch.load(path, map_location=map_location) @@ -106,7 +105,9 @@ src_state = torch.load(buffer, map_location=map_location) if "state_dict" in src_state: src_state = src_state["state_dict"] src_state = src_state["model"] if "model" in src_state else src_state for k in dst_state.keys(): if not k.startswith("module.") and "module." + k in src_state.keys(): k_ddp = "module." + k @@ -115,7 +116,7 @@ if k_ddp in src_state: dst_state[k] = src_state[k_ddp] else: print(f"Miss key in ckpt: model: {k}, ckpt: {k_ddp}") print(f"Warning, miss key in ckpt: {k}, mapped: {k_ddp}") flag = obj.load_state_dict(dst_state, strict=True) # print(flag) funasr/train_utils/trainer.py
@@ -181,7 +181,7 @@ time2 = time.perf_counter() time_escaped = (time2 - time1)/3600.0 print(f"\nrank: {self.local_rank}, time_escaped_epoch: {time_escaped:.3f} hours, estimated to finish {self.max_epoch} epoch: {(self.max_epoch-epoch)*time_escaped:.3f}\n") print(f"\nrank: {self.local_rank}, time_escaped_epoch: {time_escaped:.3f} hours, estimated to finish {self.max_epoch} epoch: {(self.max_epoch-epoch)*time_escaped:.3f} hours\n") if self.rank == 0: average_checkpoints(self.output_dir, self.avg_nbest_model) @@ -293,7 +293,7 @@ f"{time_now}, " f"rank: {self.local_rank}, " f"epoch: {epoch}/{self.max_epoch}, " f"step: {batch_idx+1}/{len(self.dataloader_train)}, total: {self.batch_total}, " f"step: {batch_idx+1}/{len(self.dataloader_train)}, total step: {self.batch_total}, " f"(loss: {loss.detach().cpu().item():.3f}), " f"(lr: {lr:.3e}), " f"{[(k, round(v.cpu().item(), 3)) for k, v in stats.items()]}, " funasr/version.txt
@@ -1 +1 @@ 1.0.8 1.0.10
