From bc723ea200144bd6fa8a5dff4b9a780feda144fc Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期四, 29 六月 2023 18:55:01 +0800
Subject: [PATCH] dcos
---
funasr/runtime/onnxruntime/src/tokenizer.cpp | 37 +++++++++++++++++++++++++++++++------
1 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/tokenizer.cpp b/funasr/runtime/onnxruntime/src/tokenizer.cpp
index ef0c533..cd3f027 100644
--- a/funasr/runtime/onnxruntime/src/tokenizer.cpp
+++ b/funasr/runtime/onnxruntime/src/tokenizer.cpp
@@ -1,11 +1,21 @@
- #include "precomp.h"
+ /**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License (https://opensource.org/licenses/MIT)
+*/
+#include "precomp.h"
+
+namespace funasr {
CTokenizer::CTokenizer(const char* sz_yamlfile):m_ready(false)
{
OpenYaml(sz_yamlfile);
}
CTokenizer::CTokenizer():m_ready(false)
+{
+}
+
+CTokenizer::~CTokenizer()
{
}
@@ -24,15 +34,20 @@
}
}
if (node.IsScalar()) {//锟角憋拷锟斤拷锟斤拷
- cout << node.as<string>() << endl;
+ LOG(INFO) << node.as<string>();
}
}
bool CTokenizer::OpenYaml(const char* sz_yamlfile)
{
- YAML::Node m_Config = YAML::LoadFile(sz_yamlfile);
- if (m_Config.IsNull())
- return false;
+ YAML::Node m_Config;
+ try{
+ m_Config = YAML::LoadFile(sz_yamlfile);
+ }catch(exception const &e){
+ LOG(INFO) << "Error loading file, yaml file error or not exist.";
+ exit(-1);
+ }
+
try
{
auto Tokens = m_Config["token_list"];
@@ -61,7 +76,7 @@
}
}
catch (YAML::BadFile& e) {
- std::cout << "read error!" << std::endl;
+ LOG(ERROR) << "Read error!";
return false;
}
m_ready = true;
@@ -125,6 +140,14 @@
result.push_back(m_punc2id[item]);
}
return result;
+}
+
+bool CTokenizer::IsPunc(string& Punc)
+{
+ if (m_punc2id.find(Punc) != m_punc2id.end())
+ return true;
+ else
+ return false;
}
vector<string> CTokenizer::SplitChineseString(const string & str_info)
@@ -206,3 +229,5 @@
}
id_out= String2Ids(str_out);
}
+
+} // namespace funasr
\ No newline at end of file
--
Gitblit v1.9.1