fix large_dataset epoch shuffle bug

haoneng.lhn

2023-09-19 9a7ea0f10e936277674f7e4cd17d4e9616706bff

fix large_dataset epoch shuffle bug

 funasr/datasets/large_datasets/datapipes/batch.py

@@ -39,7 +39,7 @@
        self.batch_mode = batch_mode

    def set_epoch(self, epoch):
        self.epoch = epoch
        self.datapipe.set_epoch(epoch)

    def __iter__(self):
        buffer = []

 funasr/datasets/large_datasets/datapipes/filter.py

@@ -13,7 +13,7 @@
        self.fn = fn

    def set_epoch(self, epoch):
        self.epoch = epoch
        self.datapipe.set_epoch(epoch)

    def __iter__(self):
        assert callable(self.fn)
@@ -21,4 +21,4 @@
            if self.fn(data):
                yield data
            else:
                continue
                continue

 funasr/datasets/large_datasets/datapipes/map.py

@@ -14,7 +14,7 @@
        self.fn = fn

    def set_epoch(self, epoch):
        self.epoch = epoch
        self.datapipe.set_epoch(epoch)

    def __iter__(self):
        assert callable(self.fn)