examples/deepspeed_conf/ds_stage1.json
examples/deepspeed_conf/ds_stage2.json
examples/deepspeed_conf/ds_stage3.json
examples/deepspeed_conf/ds_z0_config.json
examples/deepspeed_conf/ds_z2_config.json
examples/deepspeed_conf/ds_z2_offload_config.json
examples/deepspeed_conf/ds_z3_config.json
examples/deepspeed_conf/ds_z3_offload_config.json
examples/industrial_data_pretraining/bicif_paraformer/finetune.sh
@@ -1,6 +1,8 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # method1, finetune from model hub # which gpu to train or finetune @@ -45,21 +47,38 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ deepspeed_config=${workspace}../../ds_stage1.json DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=20000 \ ++dataset="AudioDataset" \ ++dataset_conf.index_ds="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/contextual_paraformer/finetune.sh
@@ -1,6 +1,8 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # method1, finetune from model hub # which gpu to train or finetune @@ -46,22 +48,38 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ deepspeed_config=${workspace}../../ds_stage1.json DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++model_revision="${model_revision}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=20000 \ ++dataset="AudioDatasetHotword" \ ++dataset_conf.index_ds="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/llm_asr/demo_train_or_finetune.sh
@@ -30,10 +30,20 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ deepspeed_config=${workspace}../../ds_stage1.json DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ --config-path "${workspace}/conf" \ --config-name "${config}" \ ++train_data_set_list="${train_data}" \ @@ -41,6 +51,9 @@ ++dataset_conf.batch_size=4 \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=15 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0001 \ ++init_param="${init_param}" \ ++output_dir="${output_dir}" &> ${log_file} & examples/industrial_data_pretraining/llm_asr/demo_train_or_finetune2.sh
@@ -30,18 +30,39 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ deepspeed_config=${workspace}../../ds_stage1.json DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ --config-path "${workspace}/conf" \ --config-name "${config}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=1 \ ++dataset_conf.num_workers=0 \ ++train_conf.max_epoch=15 \ ++train_conf.save_checkpoint_interval=1000 \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.avg_nbest_model=10 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0001 \ ++init_param="${init_param}" \ ++output_dir="${output_dir}" &> ${log_file} & examples/industrial_data_pretraining/paraformer/finetune.sh
@@ -1,6 +1,7 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" @@ -40,27 +41,42 @@ output_dir="./outputs" log_file="${output_dir}/log.txt" deepspeed_config=${workspace}../../ds_stage1.json mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --node_rank 0 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=20000 \ ++dataset="AudioDataset" \ ++dataset_conf.index_ds="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.avg_nbest_model=10 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/paraformer/train_from_local.sh
File was deleted examples/industrial_data_pretraining/paraformer_streaming/finetune.sh
@@ -1,6 +1,7 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" @@ -41,25 +42,42 @@ output_dir="./outputs" log_file="${output_dir}/log.txt" deepspeed_config=${workspace}../../ds_stage1.json mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=20000 \ ++dataset="AudioDataset" \ ++dataset_conf.index_ds="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.avg_nbest_model=10 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/seaco_paraformer/finetune.sh
@@ -1,6 +1,7 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0,1" @@ -44,22 +45,38 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" torchrun \ --nnodes 1 \ --nproc_per_node ${gpu_num} \ ../../../funasr/bin/train.py \ DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=20000 \ ++dataset="AudioDatasetHotword" \ ++dataset_conf.index_ds="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.avg_keep_nbest_models_type='loss' \ ++train_conf.keep_nbest_models=20 \ ++train_conf.avg_nbest_model=10 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++output_dir="${output_dir}" &> ${log_file} examples/industrial_data_pretraining/sense_voice/finetune.sh
@@ -1,6 +1,7 @@ # Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. # MIT License (https://opensource.org/licenses/MIT) workspace=`pwd` # which gpu to train or finetune export CUDA_VISIBLE_DEVICES="0" @@ -44,26 +45,39 @@ mkdir -p ${output_dir} echo "log_file: ${log_file}" #torchrun \ #--nnodes 1 \ #--node_rank 0 \ #--nproc_per_node ${gpu_num} \ python \ ../../../funasr/bin/train.py \ deepspeed_config=${workspace}../../ds_stage1.json DISTRIBUTED_ARGS=" --nnodes ${WORLD_SIZE:-1} \ --nproc_per_node $gpu_num \ --node_rank ${RANK:-0} \ --master_addr ${MASTER_ADDR:-127.0.0.1} \ --master_port ${MASTER_PORT: 26669} " echo $DISTRIBUTED_ARGS torchrun $DISTRIBUTED_ARGS \ ../../../funasr/bin/train_ds.py \ ++model="${model_name_or_model_dir}" \ ++train_data_set_list="${train_data}" \ ++valid_data_set_list="${val_data}" \ ++dataset_conf.batch_size=500 \ ++dataset="SenseVoiceDataset" \ ++dataset_conf.IndexDSJsonl="IndexDSJsonl" \ ++dataset_conf.data_split_num=1 \ ++dataset_conf.batch_sampler="BatchSampler" \ ++dataset_conf.batch_size=6000 \ ++dataset_conf.sort_size=1024 \ ++dataset_conf.batch_type="token" \ ++dataset_conf.num_workers=0 \ ++dataset_conf.num_workers=4 \ ++train_conf.max_epoch=50 \ ++train_conf.log_interval=1 \ ++train_conf.resume=false \ ++train_conf.resume=true \ ++train_conf.validate_interval=2000 \ ++train_conf.save_checkpoint_interval=2000 \ ++train_conf.keep_nbest_models=20 \ ++train_conf.avg_nbest_model=10 \ ++train_conf.use_deepspeed=false \ ++train_conf.deepspeed_config=${deepspeed_config} \ ++optim_conf.lr=0.0002 \ ++debug=true \ ++device="cpu" \ ++output_dir="${output_dir}" #&> ${log_file}
