From d674c29323c930842727d0689100f827798d6ba2 Mon Sep 17 00:00:00 2001
From: 雾聪 <wucong.lyb@alibaba-inc.com>
Date: 星期一, 11 十二月 2023 15:51:38 +0800
Subject: [PATCH] add timestamp smooth
---
runtime/onnxruntime/src/vocab.cpp | 21 ++++++++++++++++-----
1 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/runtime/onnxruntime/src/vocab.cpp b/runtime/onnxruntime/src/vocab.cpp
index d29281c..20571c9 100644
--- a/runtime/onnxruntime/src/vocab.cpp
+++ b/runtime/onnxruntime/src/vocab.cpp
@@ -120,8 +120,8 @@
std::string combine = "";
std::string unicodeChar = "鈻�";
- for (auto it = in.begin(); it != in.end(); it++) {
- string word = vocab[*it];
+ for (i=0; i<in.size(); i++){
+ string word = vocab[in[i]];
// step1 space character skips
if (word == "<s>" || word == "</s>" || word == "<unk>")
continue;
@@ -146,9 +146,20 @@
int sub_word = !(word.find("@@") == string::npos);
// process word start and middle part
if (sub_word) {
- combine += word.erase(word.length() - 2);
- is_combining = true;
- continue;
+ // if badcase: lo@@ chinese
+ if (i == in.size()-1 || i<in.size()-1 && IsChinese(vocab[in[i+1]])){
+ word = word.erase(word.length() - 2) + " ";
+ if (is_combining) {
+ combine += word;
+ is_combining = false;
+ word = combine;
+ combine = "";
+ }
+ }else{
+ combine += word.erase(word.length() - 2);
+ is_combining = true;
+ continue;
+ }
}
// process word end part
else if (is_combining) {
--
Gitblit v1.9.1