From 33d3d2084403fd34b79c835d2f2fe04f6cd8f738 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 13 九月 2023 09:33:54 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/runtime/onnxruntime/src/tokenizer.h | 15 ++++++++++++---
1 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/tokenizer.h b/funasr/runtime/onnxruntime/src/tokenizer.h
index 319975a..3b1d1c5 100644
--- a/funasr/runtime/onnxruntime/src/tokenizer.h
+++ b/funasr/runtime/onnxruntime/src/tokenizer.h
@@ -1,6 +1,12 @@
-#pragma once
-#include "yaml-cpp/yaml.h"
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License (https://opensource.org/licenses/MIT)
+*/
+#pragma once
+#include <yaml-cpp/yaml.h>
+
+namespace funasr {
class CTokenizer {
private:
@@ -12,6 +18,7 @@
CTokenizer(const char* sz_yamlfile);
CTokenizer();
+ ~CTokenizer();
bool OpenYaml(const char* sz_yamlfile);
void ReadYaml(const YAML::Node& node);
vector<string> Id2String(vector<int> input);
@@ -23,5 +30,7 @@
vector<string> SplitChineseString(const string& str_info);
void StrSplit(const string& str, const char split, vector<string>& res);
void Tokenize(const char* str_info, vector<string>& str_out, vector<int>& id_out);
-
+ bool IsPunc(string& Punc);
};
+
+} // namespace funasr
--
Gitblit v1.9.1