Dev gzf exp (#1665)

zhifu gao

2024-04-26 1cdb3cc28d4d89a576cc06e5cd8eb80da1f3a3aa

 funasr/datasets/large_datasets/datapipes/batch.py

@@ -19,13 +19,13 @@
class MaxTokenBucketizerIterDataPipe(IterableDataset):

    def __init__(
            self,
            datapipe,
            batch_size=8000,
            len_fn=_default_len_fn,
            buffer_size=10240,
            sort_size=500,
            batch_mode="padding",
        self,
        datapipe,
        batch_size=8000,
        len_fn=_default_len_fn,
        buffer_size=10240,
        sort_size=500,
        batch_mode="padding",
    ):
        assert batch_size > 0, "Batch size is required to be larger than 0!"
        assert buffer_size >= -1, "Buffer size is required to be larger than -1!"