From 2a66366be4c2715870e4859fd5a5db6e8a9dc00a Mon Sep 17 00:00:00 2001
From: chenmengzheAAA <123789350+chenmengzheAAA@users.noreply.github.com>
Date: 星期四, 14 九月 2023 19:00:17 +0800
Subject: [PATCH] Merge pull request #956 from alibaba-damo-academy/chenmengzheAAA-patch-4
---
funasr/runtime/onnxruntime/src/tokenizer.cpp | 11 +++++++++++
1 files changed, 11 insertions(+), 0 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/tokenizer.cpp b/funasr/runtime/onnxruntime/src/tokenizer.cpp
index 5aff058..cd3f027 100644
--- a/funasr/runtime/onnxruntime/src/tokenizer.cpp
+++ b/funasr/runtime/onnxruntime/src/tokenizer.cpp
@@ -5,6 +5,7 @@
#include "precomp.h"
+namespace funasr {
CTokenizer::CTokenizer(const char* sz_yamlfile):m_ready(false)
{
OpenYaml(sz_yamlfile);
@@ -141,6 +142,14 @@
return result;
}
+bool CTokenizer::IsPunc(string& Punc)
+{
+ if (m_punc2id.find(Punc) != m_punc2id.end())
+ return true;
+ else
+ return false;
+}
+
vector<string> CTokenizer::SplitChineseString(const string & str_info)
{
vector<string> list;
@@ -220,3 +229,5 @@
}
id_out= String2Ids(str_out);
}
+
+} // namespace funasr
\ No newline at end of file
--
Gitblit v1.9.1