From 2a66366be4c2715870e4859fd5a5db6e8a9dc00a Mon Sep 17 00:00:00 2001
From: chenmengzheAAA <123789350+chenmengzheAAA@users.noreply.github.com>
Date: 星期四, 14 九月 2023 19:00:17 +0800
Subject: [PATCH] Merge pull request #956 from alibaba-damo-academy/chenmengzheAAA-patch-4
---
funasr/datasets/preprocessor.py | 5 ++---
1 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/funasr/datasets/preprocessor.py b/funasr/datasets/preprocessor.py
index bd2c972..9b5c4e7 100644
--- a/funasr/datasets/preprocessor.py
+++ b/funasr/datasets/preprocessor.py
@@ -11,7 +11,7 @@
import numpy as np
import scipy.signal
import soundfile
-
+import jieba
from funasr.text.build_tokenizer import build_tokenizer
from funasr.text.cleaner import TextCleaner
@@ -201,7 +201,7 @@
self.seg_dict = None
if seg_dict_file is not None:
self.seg_dict = {}
- with open(seg_dict_file) as f:
+ with open(seg_dict_file, "r", encoding="utf8") as f:
lines = f.readlines()
for line in lines:
s = line.strip().split()
@@ -659,7 +659,6 @@
self.split_text_name = split_text_name
self.seg_jieba = seg_jieba
if self.seg_jieba:
- import jieba
jieba.load_userdict(seg_dict_file)
@classmethod
--
Gitblit v1.9.1