From 172e7ac986f299ad545cbd91a8cecc3ef967af36 Mon Sep 17 00:00:00 2001 From: zhifu gao <zhifu.gzf@alibaba-inc.com> Date: 星期一, 11 十二月 2023 10:17:22 +0800 Subject: [PATCH] Revert "Dev gzf funasr2" (#1164) --- funasr/datasets/dataset_jsonl.py | 18 +++++++----------- 1 files changed, 7 insertions(+), 11 deletions(-) diff --git a/funasr/datasets/dataset_jsonl.py b/funasr/datasets/dataset_jsonl.py index eef67c5..543b60e 100644 --- a/funasr/datasets/dataset_jsonl.py +++ b/funasr/datasets/dataset_jsonl.py @@ -78,26 +78,21 @@ def __getitem__(self, index): return self.contents[index] - - def get_source_len(self, data_dict): - return data_dict["source_len"] - - def get_target_len(self, data_dict): - - return data_dict["target_len"] if "target_len" in data_dict else 0 class AudioDataset(torch.utils.data.Dataset): - def __init__(self, path, frontend=None, tokenizer=None, int_pad_value: int = -1, float_pad_value: float = 0.0, **kwargs): + def __init__(self, path, frontend=None, tokenizer=None, token_id_converter=None): + super().__init__() self.indexed_dataset = IndexedDatasetJsonl(path) self.frontend = frontend.forward self.fs = 16000 if frontend is None else frontend.fs self.data_type = "sound" self.tokenizer = tokenizer + self.token_id_converter = token_id_converter - self.int_pad_value = int_pad_value - self.float_pad_value = float_pad_value + self.int_pad_value = -1 + self.float_pad_value = 0.0 @@ -113,7 +108,8 @@ data_src = load_audio(source, fs=self.fs) speech, speech_lengths = extract_features(data_src, self.data_type, self.frontend) target = item["target"] - ids = self.tokenizer.encode(target) + text = self.tokenizer.text2tokens(target) + ids = self.token_id_converter.tokens2ids(text) ids_lengths = len(ids) text, text_lengths = torch.tensor(ids, dtype=torch.int64), torch.tensor([ids_lengths], dtype=torch.int32) -- Gitblit v1.9.1