From c880db53646ab9fd26417f4baf004ab44cc24e1a Mon Sep 17 00:00:00 2001
From: lingji-yidong <75744976+lingji-yidong@users.noreply.github.com>
Date: 星期五, 28 六月 2024 01:28:24 +0800
Subject: [PATCH] Fix: Return tuple ('', []) when char_list is empty to prevent ValueError (#1857)
---
funasr/datasets/large_datasets/datapipes/filter.py | 23 +++++++++++++++++++++++
1 files changed, 23 insertions(+), 0 deletions(-)
diff --git a/funasr/datasets/large_datasets/datapipes/filter.py b/funasr/datasets/large_datasets/datapipes/filter.py
new file mode 100644
index 0000000..c4f045d
--- /dev/null
+++ b/funasr/datasets/large_datasets/datapipes/filter.py
@@ -0,0 +1,23 @@
+from torch.utils.data import IterableDataset
+
+
+def default_fn(data):
+ return data
+
+
+class FilterIterDataPipe(IterableDataset):
+
+ def __init__(self, datapipe, fn=default_fn):
+ self.datapipe = datapipe
+ self.fn = fn
+
+ def set_epoch(self, epoch):
+ self.datapipe.set_epoch(epoch)
+
+ def __iter__(self):
+ assert callable(self.fn)
+ for data in self.datapipe:
+ if self.fn(data):
+ yield data
+ else:
+ continue
--
Gitblit v1.9.1