| | |
|---|
| | | self.max_token_length = kwargs.get("max_token_length", 1024) |
|---|
| | | self.batch_size_scale_ratio_max = kwargs.get("batch_size_scale_ratio_max", 1.5) |
|---|
| | | self.batch_size_token_max = kwargs.get("batch_size_token_max", 2500) |
|---|
| | | self.audio_adaptor_downsample_rate = kwargs.get("audio_adaptor_downsample_rate", 2) |
|---|
| | | self.audio_encoder_downsample_rate = kwargs.get("audio_encoder_downsample_rate", 4) |
|---|
| | | |
|---|
| | | def get_source_len(self, index): |
|---|
| | | item = self.index_ds[index] |
|---|
| | |
|---|
| | | speech = speech.permute(0, 2, 1) |
|---|
| | | # if speech_lengths > self.batch_size: |
|---|
| | | # continue |
|---|
| | | if self.audio_encoder_downsample_rate == 4: |
|---|
| | | olens = 1 + (speech_lengths[0].item() - 3 + 2 * 1) // 2 |
|---|
| | | olens = 1 + (olens - 3 + 2 * 1) // 2 |
|---|
| | | elif self.audio_encoder_downsample_rate == 1: |
|---|
| | | olens = speech_lengths[0].item() |
|---|
| | | |
|---|
| | | olens = 1 + (speech_lengths[0].item() - 3 + 2 * 1) // 2 |
|---|
| | | olens = 1 + (olens - 3 + 2 * 1) // 2 |
|---|
| | | sub_token_len = (olens - 1) // 2 + 1 |
|---|
| | | sub_token_len = (olens - 1) // self.audio_adaptor_downsample_rate + 1 |
|---|
| | | sub_token = [0] * sub_token_len |
|---|
| | | fbank_beg_i = [len(source_ids)] |
|---|
| | | source_ids += sub_token |
|---|
| | |
|---|
| | | |
|---|
| | | splits = self.pattern.split(source_input) |
|---|
| | | source_ids = [] |
|---|
| | | fbank_i = [] |
|---|
| | | fbank_mask_i = [] |
|---|
| | | fbank_beg_i = [] |
|---|
| | | fbank_lens_i = [] |
|---|
| | |
|---|
| | | target_ids = self.tokenizer.encode(target_out) |
|---|
| | | input_ids += source_ids + target_ids |
|---|
| | | labels += source_mask + target_ids |
|---|
| | | fbank.append(speech[0, :, :]) |
|---|
| | | fbank_mask += fbank_mask_i |
|---|
| | | fbank_beg.append(fbank_beg_i) |
|---|
| | | if len(fbank_beg_i) < 1: |
|---|
| | | fbank_beg_i = [-1] |
|---|
| | | fbank_beg += fbank_beg_i |
|---|
| | | |
|---|
| | | if len(input_ids) > self.max_token_length: |
|---|
| | | logging.info( |
|---|
| | |
|---|
| | | attention_mask = torch.tensor([1] * len(input_ids), dtype=torch.int32) |
|---|
| | | labels = torch.tensor(labels, dtype=torch.int64) # [: self.max_token_length] |
|---|
| | | |
|---|
| | | fbank = speech[0, :, :] |
|---|
| | | # fbank = speech[0, :, :] |
|---|
| | | fbank_lens = speech_lengths |
|---|
| | | fbank_mask = torch.tensor(fbank_mask, dtype=torch.float32) |
|---|
| | | fbank_beg = torch.tensor(fbank_beg, dtype=torch.int32) |
|---|
| | |
|---|
| | | for key in sample.keys(): |
|---|
| | | if key not in outputs: |
|---|
| | | outputs[key] = [] |
|---|
| | | outputs[key].append(sample[key]) |
|---|
| | | if isinstance(sample[key], (list, tuple)): |
|---|
| | | outputs[key].extend(sample[key]) |
|---|
| | | else: |
|---|
| | | outputs[key].append(sample[key]) |
|---|
| | | |
|---|
| | | for key, data_list in outputs.items(): |
|---|
| | | if isinstance(data_list[0], torch.Tensor): |
|---|
