| | |
|---|
| | | ["{}/{}/text".format(args.data_dir, args.valid_set), data_names[1], data_types[1]] |
|---|
| | | ] |
|---|
| | | if args.embed_path is not None: |
|---|
| | | args.train_data_path_and_name_and_type[0].append( |
|---|
| | | "{}/embed/kaldi_ark".format(os.path.join(args.embed_path, "embeds", args.train_set, "embeds.scp"))) |
|---|
| | | args.valid_data_path_and_name_and_type[0].append( |
|---|
| | | "{}/embed/kaldi_ark".format(os.path.join(args.embed_path, "embeds", args.dev_set, "embeds.scp"))) |
|---|
| | | args.train_data_path_and_name_and_type.append( |
|---|
| | | [os.path.join(args.embed_path, "embeds", args.train_set, "embeds.scp"), "embed", "kaldi_ark"]) |
|---|
| | | args.valid_data_path_and_name_and_type.append( |
|---|
| | | [os.path.join(args.embed_path, "embeds", args.valid_set, "embeds.scp"), "embed", "kaldi_ark"]) |
|---|
| | | else: |
|---|
| | | args.train_data_file = os.path.join(args.data_dir, args.train_set, "data.list") |
|---|
| | | args.valid_data_file = os.path.join(args.data_dir, args.valid_set, "data.list") |
|---|
| | | if args.embed_path is not None: |
|---|
| | | if not distributed or distributed_option.dist_rank == 0: |
|---|
| | | for d in [args.train_set, args.valid_set]: |
|---|
| | | file = os.path.join(args.data_dir, d, "data.list") |
|---|
| | | with open(file) as f: |
|---|
| | | lines = f.readlines() |
|---|
| | | out_file = os.path.join(args.data_dir, d, "data_with_embed.list") |
|---|
| | | with open(out_file, "w") as out_f: |
|---|
| | | for line in lines: |
|---|
| | | parts = line.strip().split() |
|---|
| | | idx = parts[0].split("/")[-2] |
|---|
| | | embed_file = os.path.join(args.embed_path, "embeds", args.valid_set, "ark", |
|---|
| | | "embeds.{}.ark".format(idx)) |
|---|
| | | out_f.write(parts[0] + " " + parts[1] + " " + embed_file + "\n") |
|---|
| | | args.train_data_file = os.path.join(args.data_dir, args.train_set, "data_with_embed.list") |
|---|
| | | args.valid_data_file = os.path.join(args.data_dir, args.valid_set, "data_with_embed.list") |
|---|
| | | if distributed: |
|---|
| | | dist.barrier() |
|---|
