From 9a6de675dc0bf16a8c3440c7f5e42cfccd1433ac Mon Sep 17 00:00:00 2001
From: speech_asr <wangjiaming.wjm@alibaba-inc.com>
Date: 星期一, 17 四月 2023 19:22:57 +0800
Subject: [PATCH] update
---
funasr/datasets/small_datasets/build_loader.py | 33 ++++++++++++++++++++++++++++++---
1 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/funasr/datasets/small_datasets/build_loader.py b/funasr/datasets/small_datasets/build_loader.py
index 6727602..d5d6f76 100644
--- a/funasr/datasets/small_datasets/build_loader.py
+++ b/funasr/datasets/small_datasets/build_loader.py
@@ -1,15 +1,42 @@
+import os
+
import torch
from funasr.datasets.small_datasets.dataset import ESPnetDataset
from funasr.datasets.small_datasets.preprocessor import build_preprocess
+from funasr.samplers.build_batch_sampler import build_batch_sampler
-def build_dataloader(args, train=False):
- preprocess_fn = build_preprocess(args, train=train)
+def build_dataloader(args, mode="train"):
+ preprocess_fn = build_preprocess(args, train=mode=="train")
dest_sample_rate = args.frontend_conf["fs"] if (args.frontend_conf is not None and "fs" in args.frontend_conf) else 16000
+ if mode == "train":
+ data_path_and_name_and_type = args.train_data_path_and_name_and_type
+ shape_files = args.train_shape_file
+ elif mode == "valid":
+ data_path_and_name_and_type = args.valid_data_path_and_name_and_type
+ shape_files = args.valid_shape_file
+ else:
+ raise NotImplementedError(f"mode={mode}")
dataset = ESPnetDataset(
- iter_options.data_path_and_name_and_type,
+ data_path_and_name_and_type,
float_dtype=args.train_dtype,
preprocess=preprocess_fn,
max_cache_size=args.max_cache_size,
max_cache_fd=args.max_cache_fd,
dest_sample_rate=dest_sample_rate,
)
+ if os.path.exists(os.path.join(data_path_and_name_and_type[0][0].parent, "utt2category")):
+ utt2category_file = os.path.join(data_path_and_name_and_type[0][0].parent, "utt2category")
+ else:
+ utt2category_file = None
+ batch_sampler = build_batch_sampler(
+ type=args.batch_type,
+ shape_files=iter_options.shape_files,
+ fold_lengths=args.fold_length,
+ batch_size=iter_options.batch_size,
+ batch_bins=iter_options.batch_bins,
+ sort_in_batch=args.sort_in_batch,
+ sort_batch=args.sort_batch,
+ drop_last=False,
+ min_batch_size=torch.distributed.get_world_size() if args.distributed else 1,
+ utt2category_file=utt2category_file,
+ )
\ No newline at end of file
--
Gitblit v1.9.1