| | |
|---|
| | | mkdir -p ${output_dir} |
|---|
| | | echo "log_file: ${log_file}" |
|---|
| | | |
|---|
| | | torchrun \ |
|---|
| | | --nnodes 1 \ |
|---|
| | | --nproc_per_node ${gpu_num} \ |
|---|
| | | ../../../funasr/bin/train.py \ |
|---|
| | | deepspeed_config=${workspace}../../ds_stage1.json |
|---|
| | | |
|---|
| | | DISTRIBUTED_ARGS=" |
|---|
| | | --nnodes ${WORLD_SIZE:-1} \ |
|---|
| | | --nproc_per_node $gpu_num \ |
|---|
| | | --node_rank ${RANK:-0} \ |
|---|
| | | --master_addr ${MASTER_ADDR:-127.0.0.1} \ |
|---|
| | | --master_port ${MASTER_PORT: 26669} |
|---|
| | | " |
|---|
| | | |
|---|
| | | echo $DISTRIBUTED_ARGS |
|---|
| | | |
|---|
| | | torchrun $DISTRIBUTED_ARGS \ |
|---|
| | | ../../../funasr/bin/train_ds.py \ |
|---|
| | | --config-path "${workspace}/conf" \ |
|---|
| | | --config-name "${config}" \ |
|---|
| | | ++train_data_set_list="${train_data}" \ |
|---|
| | | ++valid_data_set_list="${val_data}" \ |
|---|
| | | ++dataset_conf.batch_size=1 \ |
|---|
| | | ++dataset_conf.num_workers=0 \ |
|---|
| | | ++train_conf.max_epoch=15 \ |
|---|
| | | ++train_conf.save_checkpoint_interval=1000 \ |
|---|
| | | ++dataset_conf.data_split_num=1 \ |
|---|
| | | ++dataset_conf.batch_sampler="BatchSampler" \ |
|---|
| | | ++dataset_conf.batch_size=6000 \ |
|---|
| | | ++dataset_conf.sort_size=1024 \ |
|---|
| | | ++dataset_conf.batch_type="token" \ |
|---|
| | | ++dataset_conf.num_workers=4 \ |
|---|
| | | ++train_conf.max_epoch=50 \ |
|---|
| | | ++train_conf.log_interval=1 \ |
|---|
| | | ++train_conf.resume=true \ |
|---|
| | | ++train_conf.validate_interval=2000 \ |
|---|
| | | ++train_conf.save_checkpoint_interval=2000 \ |
|---|
| | | ++train_conf.keep_nbest_models=20 \ |
|---|
| | | ++train_conf.avg_nbest_model=10 \ |
|---|
| | | ++train_conf.use_deepspeed=false \ |
|---|
| | | ++train_conf.deepspeed_config=${deepspeed_config} \ |
|---|
| | | ++optim_conf.lr=0.0001 \ |
|---|
| | | ++init_param="${init_param}" \ |
|---|
| | | ++output_dir="${output_dir}" &> ${log_file} & |
|---|
