funasr/datasets/large_datasets/datapipes/filter.py
@@ -1,19 +1,18 @@ from torch.utils.data import IterableDataset def default_fn(data): return data class FilterIterDataPipe(IterableDataset): def __init__(self, datapipe, fn=default_fn): def __init__(self, datapipe, fn=default_fn): self.datapipe = datapipe self.fn = fn def set_epoch(self, epoch): self.epoch = epoch self.datapipe.set_epoch(epoch) def __iter__(self): assert callable(self.fn) @@ -21,4 +20,4 @@ if self.fn(data): yield data else: continue continue
