From 2ac38adbe5f4e1374a079e032ed4b504351a207c Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 23 四月 2024 18:08:57 +0800
Subject: [PATCH] Dev gzf exp (#1647)
---
funasr/datasets/dataloader_entry.py | 16 +++++++++++++++-
1 files changed, 15 insertions(+), 1 deletions(-)
diff --git a/funasr/datasets/dataloader_entry.py b/funasr/datasets/dataloader_entry.py
index abb2828..70da722 100644
--- a/funasr/datasets/dataloader_entry.py
+++ b/funasr/datasets/dataloader_entry.py
@@ -40,7 +40,21 @@
self.dataset_val = dataset_val
self.kwargs = kwargs
- def build_iter(self, epoch=0):
+ # split dataset
+ self.data_split_num = kwargs["dataset_conf"].get("data_split_num", 1)
+ self.dataset_class = dataset_class
+ self.frontend = frontend
+ self.tokenizer = tokenizer
+ self.kwargs = kwargs
+
+ def build_iter(self, epoch=0, data_split_i=0, **kwargs):
+
+ # reload dataset slice
+ if self.data_split_num > 1:
+ del self.dataset_tr
+ self.dataset_tr = self.dataset_class(self.kwargs.get("train_data_set_list"), frontend=self.frontend, tokenizer=self.tokenizer,
+ is_training=True, **self.kwargs.get("dataset_conf"), data_split_i=data_split_i)
+
# dataloader
batch_sampler = self.kwargs["dataset_conf"].get("batch_sampler", "BatchSampler")
batch_sampler_val = None
--
Gitblit v1.9.1