| | |
|---|
| | | stage=1 |
|---|
| | | stop_stage=3 |
|---|
| | | |
|---|
| | | bert_model_root="../../huggingface_models" |
|---|
| | | bert_model_name="bert-base-chinese" |
|---|
| | | raw_dataset_path="../DATA" |
|---|
| | | model_path=${bert_model_root}/${bert_model_name} |
|---|
| | | nj=64 |
|---|
| | | model_path=${bert_model_name} |
|---|
| | | |
|---|
| | | . utils/parse_options.sh || exit 1; |
|---|
| | | |
|---|
| | | nj=32 |
|---|
| | | |
|---|
| | | for data_set in train dev test;do |
|---|
| | | scp=$raw_dataset_path/dump/fbank/${data_set}/text |
|---|
| | | scp=$raw_dataset_path/data/${data_set}/text |
|---|
| | | local_scp_dir_raw=${raw_dataset_path}/data/embeds/${data_set} |
|---|
| | | local_scp_dir=$local_scp_dir_raw/split$nj |
|---|
| | | local_records_dir=$local_scp_dir_raw/ark |
|---|
| | |
|---|
| | | cat ${local_records_dir}/embeds.${JOB}.shape || exit 1; |
|---|
| | | done > ${local_scp_dir_raw}/embeds.shape |
|---|
| | | fi |
|---|
| | | |
|---|
| | | cp ${local_scp_dir_raw}/embeds.scp ${raw_dataset_path}/data/${data_set}/embeds.scp |
|---|
| | | done |
|---|
| | | |
|---|
| | | echo "embeds is in: ${local_scp_dir_raw}" |
|---|
