TOLD/SOND: modify data preprocessing

志浩

2023-08-02 67190a7af089eea63f824fb730645213b97d4b54

 egs/callhome/diarization/sond/finetune.sh

@@ -30,6 +30,13 @@
  ln -s ${kaldi_root}/egs/callhome_diarization/v2/utils ./utils
fi

# callhome data root like path/to/NIST/LDC2001S97
callhome_root=
if [ -z "${kaldi_root}" ]; then
  echo "We need callhome corpus to prepare data."
  exit;
fi

# machines configuration
gpu_devices="0,1,2,3"  # for V100-16G, need 4 gpus.
gpu_num=4
@@ -41,9 +48,6 @@
# number of jobs for data process
nj=16
sr=8000

# dataset related
callhome_root=path/to/NIST/LDC2001S97

# experiment configuration
lang=en
@@ -92,8 +96,24 @@

# Prepare datasets
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
  echp "Stage 0: Prepare callhome data."
  echo "Stage 0: Prepare callhome data."
  local/make_callhome.sh ${callhome_root} ${datadir}/

  # split ref.rttm
  for dset in callhome1 callhome2; do
    rm -rf ${datadir}/${dset}/ref.rttm
    for name in `awk '{print $1}' ${datadir}/${dset}/wav.scp`; do
      grep ${name} ${datadir}/callhome/fullref.rttm >> ${datadir}/${dset}/ref.rttm;
    done

    # filter out records which don't have rttm labels.
    awk '{print $2}' ${datadir}/${dset}/ref.rttm | sort | uniq > ${datadir}/${dset}/uttid
    mv ${datadir}/${dset}/wav.scp ${datadir}/${dset}/wav.scp.bak
    awk '{if (NR==FNR){a[$1]=1}else{if (a[$1]==1){print $0}}}' ${datadir}/${dset}/uttid ${datadir}/${dset}/wav.scp.bak > ${datadir}/${dset}/wav.scp
    mkdir ${datadir}/${dset}/raw
    mv ${datadir}/${dset}/{reco2num_spk,segments,spk2utt,utt2spk,uttid,wav.scp.bak} ${datadir}/${dset}/raw/
    awk '{print $1,$1}' wav.scp > ${datadir}/${dset}/utt2spk
  done
fi

if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
@@ -123,10 +143,10 @@
    mkdir -p ${dumpdir}/${dset}/nonoverlap_0s
    python -Wignore script/extract_nonoverlap_segments.py \
      ${datadir}/${dset}/wav.scp ${datadir}/${dset}/ref.rttm ${dumpdir}/${dset}/nonoverlap_0s \
      --min_dur 0 --max_spk_num 8 --sr ${sr} --no_pbar --nj ${nj}
      --min_dur 0.1 --max_spk_num 8 --sr ${sr} --no_pbar --nj ${nj}

    mkdir -p ${datadir}/${dset}/nonoverlap_0s
    find `pwd`/${dumpdir}/${dset}/nonoverlap_0s | sort | awk -F'[/.]' '{print $(NF-1),$0}' > ${datadir}/${dset}/nonoverlap_0s/wav.scp
    find ${dumpdir}/${dset}/nonoverlap_0s/ -iname "*.wav" | sort | awk -F'[/.]' '{print $(NF-1),$0}' > ${datadir}/${dset}/nonoverlap_0s/wav.scp
    awk -F'[/.]' '{print $(NF-1),$(NF-2)}' ${datadir}/${dset}/nonoverlap_0s/wav.scp > ${datadir}/${dset}/nonoverlap_0s/utt2spk
    echo "Done."
  done
@@ -134,13 +154,17 @@

if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
  echo "Stage 3: Generate fbank features"
  home_path=`pwd`
  home_path=$(pwd)
  cd ${kaldi_root}/egs/callhome_diarization/v2 || exit

  . ./cmd.sh
  export train_cmd="run.pl"
  export cmd="run.pl"
  . ./path.sh
  cd $home_path || exit

  ln -s ${kaldi_root}/egs/callhome_diarization/v2/steps ./
  for dset in callhome1 callhome2; do
    mv ${datadir}/${dset}/segments ${datadir}/${dset}/segs
    steps/make_fbank.sh --write-utt2num-frames true --fbank-config conf/fbank.conf --nj ${nj} --cmd "$train_cmd" \
        ${datadir}/${dset} ${expdir}/make_fbank/${dset} ${dumpdir}/${dset}/fbank
    utils/fix_data_dir.sh ${datadir}/${dset}
@@ -151,8 +175,8 @@
        ${datadir}/${dset} ${expdir}/make_fbank/${dset} ${dumpdir}/${dset}/fbank
    utils/fix_data_dir.sh ${datadir}/${dset}
  done
  rm -f steps

  cd ${home_path} || exit
fi

if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then