From 5c9bdfa238560b9c4a8d8a7d4b56c28496a3e094 Mon Sep 17 00:00:00 2001
From: aky15 <ankeyuthu@gmail.com>
Date: 星期四, 14 九月 2023 16:12:53 +0800
Subject: [PATCH] Merge pull request #952 from alibaba-damo-academy/aky15-patch-1
---
funasr/datasets/preprocessor.py | 5 ++---
1 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/funasr/datasets/preprocessor.py b/funasr/datasets/preprocessor.py
index bd2c972..9b5c4e7 100644
--- a/funasr/datasets/preprocessor.py
+++ b/funasr/datasets/preprocessor.py
@@ -11,7 +11,7 @@
import numpy as np
import scipy.signal
import soundfile
-
+import jieba
from funasr.text.build_tokenizer import build_tokenizer
from funasr.text.cleaner import TextCleaner
@@ -201,7 +201,7 @@
self.seg_dict = None
if seg_dict_file is not None:
self.seg_dict = {}
- with open(seg_dict_file) as f:
+ with open(seg_dict_file, "r", encoding="utf8") as f:
lines = f.readlines()
for line in lines:
s = line.strip().split()
@@ -659,7 +659,6 @@
self.split_text_name = split_text_name
self.seg_jieba = seg_jieba
if self.seg_jieba:
- import jieba
jieba.load_userdict(seg_dict_file)
@classmethod
--
Gitblit v1.9.1