From 58b6154a73331a8807127d4579ed473432ce88de Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 20 二月 2024 17:02:44 +0800
Subject: [PATCH] update
---
funasr/datasets/audio_datasets/preprocessor.py | 45 ++++++---------------------------------------
1 files changed, 6 insertions(+), 39 deletions(-)
diff --git a/funasr/datasets/audio_datasets/preprocessor.py b/funasr/datasets/audio_datasets/preprocessor.py
index 6c21fbf..a3ba3a5 100644
--- a/funasr/datasets/audio_datasets/preprocessor.py
+++ b/funasr/datasets/audio_datasets/preprocessor.py
@@ -26,9 +26,10 @@
return waveform
speed = random.choice(self.speed_perturb)
if speed != 1.0:
- waveform, _ = torchaudio.sox_effects.apply_effects_tensor(
- torch.tensor(waveform).view(1, -1), fs, [['speed', str(speed)], ['rate', str(fs)]])
- waveform = waveform.view(-1)
+ with torch.no_grad():
+ waveform, _ = torchaudio.sox_effects.apply_effects_tensor(
+ torch.tensor(waveform).view(1, -1), fs, [['speed', str(speed)], ['rate', str(fs)]])
+ waveform = waveform.view(-1)
return waveform
@@ -41,43 +42,9 @@
**kwargs):
super().__init__()
- self.seg_dict = None
- if seg_dict is not None:
- self.seg_dict = {}
- with open(seg_dict, "r", encoding="utf8") as f:
- lines = f.readlines()
- for line in lines:
- s = line.strip().split()
- key = s[0]
- value = s[1:]
- self.seg_dict[key] = " ".join(value)
self.text_cleaner = TextCleaner(text_cleaner)
- self.split_with_space = split_with_space
def forward(self, text, **kwargs):
- if self.seg_dict is not None:
- text = self.text_cleaner(text)
- if self.split_with_space:
- tokens = text.strip().split(" ")
- if self.seg_dict is not None:
- text = seg_tokenize(tokens, self.seg_dict)
-
+ text = self.text_cleaner(text)
+
return text
-
-def seg_tokenize(txt, seg_dict):
- pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
- out_txt = ""
- for word in txt:
- word = word.lower()
- if word in seg_dict:
- out_txt += seg_dict[word] + " "
- else:
- if pattern.match(word):
- for char in word:
- if char in seg_dict:
- out_txt += seg_dict[char] + " "
- else:
- out_txt += "<unk>" + " "
- else:
- out_txt += "<unk>" + " "
- return out_txt.strip().split()
\ No newline at end of file
--
Gitblit v1.9.1