fix bug

游雁

2024-06-09 3eee773814c392e497557bbad501e0add4c8eca9

 funasr/datasets/audio_datasets/update_jsonl.py

@@ -31,11 +31,11 @@
        else:
            for i in range(num_total):
                update_data(lines, i)
        print("All audio durations have been processed.")
        logging.info("All audio durations have been processed.")

        for line in lines:

            jsonl_file_out_f.write(line)
            jsonl_file_out_f.write(line + "\n")
            jsonl_file_out_f.flush()

    jsonl_file_out_f.close()
@@ -46,15 +46,17 @@
    data = json.loads(line.strip())

    wav_path = data["source"].replace("/cpfs01", "/cpfs_speech/data")
    waveform, _ = librosa.load(wav_path, sr=16000)
    sample_num = len(waveform)
    source_len = int(sample_num / 16000 * 1000 / 10)
    source_len_old = data["source_len"]
    if source_len_old != source_len:
        print(f"wav: {wav_path}, old: {source_len_old}, new: {source_len}")
    data["source_len"] = source_len
    jsonl_line = json.dumps(data, ensure_ascii=False)
    lines[i] = jsonl_line
    if os.path.exists(wav_path):
        waveform, _ = librosa.load(wav_path, sr=16000)
        sample_num = len(waveform)
        source_len = int(sample_num / 16000 * 1000 / 10)
        source_len_old = data["source_len"]
        # if (source_len_old - source_len) > 100 or (source_len - source_len_old) > 100:
        #     logging.info(f"old: {source_len_old}, new: {source_len}, wav: {wav_path}")
        data["source_len"] = source_len
        data["source"] = wav_path
        jsonl_line = json.dumps(data, ensure_ascii=False)
        lines[i] = jsonl_line


def update_wav_len(jsonl_file_list_in, jsonl_file_out_dir, ncpu=1):
@@ -66,7 +68,7 @@
        for i, jsonl in enumerate(data_file_lists):
            filename_with_extension = os.path.basename(jsonl.strip())
            jsonl_file_out = os.path.join(jsonl_file_out_dir, filename_with_extension)
            print(f"{i}/{len(data_file_lists)}, jsonl: {jsonl}, {jsonl_file_out}")
            logging.info(f"{i}/{len(data_file_lists)}, jsonl: {jsonl}, {jsonl_file_out}")

            gen_scp_from_jsonl(jsonl.strip(), jsonl_file_out, ncpu)

@@ -75,7 +77,7 @@
def main_hydra(cfg: DictConfig):

    kwargs = OmegaConf.to_container(cfg, resolve=True)
    print(kwargs)
    logging.info(kwargs)

    jsonl_file_list_in = kwargs.get(
        "jsonl_file_list_in", "/Users/zhifu/funasr1.0/data/list/data_jsonl.list"