| | |
|---|
| | | self.dataset_val = dataset_val |
|---|
| | | self.kwargs = kwargs |
|---|
| | | |
|---|
| | | def build_iter(self, epoch=0): |
|---|
| | | # split dataset |
|---|
| | | self.data_split_num = kwargs["dataset_conf"].get("data_split_num", 1) |
|---|
| | | self.dataset_class = dataset_class |
|---|
| | | self.frontend = frontend |
|---|
| | | self.tokenizer = tokenizer |
|---|
| | | self.kwargs = kwargs |
|---|
| | | |
|---|
| | | def build_iter(self, epoch=0, data_split_i=0, **kwargs): |
|---|
| | | |
|---|
| | | # reload dataset slice |
|---|
| | | if self.data_split_num > 1: |
|---|
| | | del self.dataset_tr |
|---|
| | | self.dataset_tr = self.dataset_class(self.kwargs.get("train_data_set_list"), frontend=self.frontend, tokenizer=self.tokenizer, |
|---|
| | | is_training=True, **self.kwargs.get("dataset_conf"), data_split_i=data_split_i) |
|---|
| | | |
|---|
| | | # dataloader |
|---|
| | | batch_sampler = self.kwargs["dataset_conf"].get("batch_sampler", "BatchSampler") |
|---|
| | | batch_sampler_val = None |
|---|
| | |
|---|
| | | batch_sampler_val = batch_sampler_class(self.dataset_val, is_training=False, **self.kwargs.get("dataset_conf")) |
|---|
| | | |
|---|
| | | batch_sampler["batch_sampler"].set_epoch(epoch) |
|---|
| | | batch_sampler_val.set_epoch(epohc) |
|---|
| | | batch_sampler_val["batch_sampler"].set_epoch(epoch) |
|---|
| | | dataloader_tr = torch.utils.data.DataLoader(self.dataset_tr, collate_fn=self.dataset_tr.collator, **batch_sampler) |
|---|
| | | dataloader_val = torch.utils.data.DataLoader(self.dataset_val, collate_fn=self.dataset_val.collator, **batch_sampler_val) |
|---|
| | | |
|---|
