From c0008fd46134d60a3a41b022bf9156cea5b145e5 Mon Sep 17 00:00:00 2001 From: zhifu gao <zhifu.gzf@alibaba-inc.com> Date: 星期一, 11 十二月 2023 10:10:40 +0800 Subject: [PATCH] Merge branch 'dev_gzf_funasr2' into main --- funasr/datasets/dataset_jsonl.py | 18 +++++++++++------- 1 files changed, 11 insertions(+), 7 deletions(-) diff --git a/funasr/datasets/dataset_jsonl.py b/funasr/datasets/dataset_jsonl.py index 543b60e..eef67c5 100644 --- a/funasr/datasets/dataset_jsonl.py +++ b/funasr/datasets/dataset_jsonl.py @@ -78,21 +78,26 @@ def __getitem__(self, index): return self.contents[index] + + def get_source_len(self, data_dict): + return data_dict["source_len"] + + def get_target_len(self, data_dict): + + return data_dict["target_len"] if "target_len" in data_dict else 0 class AudioDataset(torch.utils.data.Dataset): - def __init__(self, path, frontend=None, tokenizer=None, token_id_converter=None): - + def __init__(self, path, frontend=None, tokenizer=None, int_pad_value: int = -1, float_pad_value: float = 0.0, **kwargs): super().__init__() self.indexed_dataset = IndexedDatasetJsonl(path) self.frontend = frontend.forward self.fs = 16000 if frontend is None else frontend.fs self.data_type = "sound" self.tokenizer = tokenizer - self.token_id_converter = token_id_converter - self.int_pad_value = -1 - self.float_pad_value = 0.0 + self.int_pad_value = int_pad_value + self.float_pad_value = float_pad_value @@ -108,8 +113,7 @@ data_src = load_audio(source, fs=self.fs) speech, speech_lengths = extract_features(data_src, self.data_type, self.frontend) target = item["target"] - text = self.tokenizer.text2tokens(target) - ids = self.token_id_converter.tokens2ids(text) + ids = self.tokenizer.encode(target) ids_lengths = len(ids) text, text_lengths = torch.tensor(ids, dtype=torch.int64), torch.tensor([ids_lengths], dtype=torch.int32) -- Gitblit v1.9.1