From 0ef6f0813c4fbce20c83a1da829ff2aa08df2127 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 15 五月 2024 19:02:00 +0800
Subject: [PATCH] update with main (#1731)
---
runtime/onnxruntime/src/vocab.cpp | 21 ++++++++++++++++++++-
1 files changed, 20 insertions(+), 1 deletions(-)
diff --git a/runtime/onnxruntime/src/vocab.cpp b/runtime/onnxruntime/src/vocab.cpp
index 6991376..1416dd3 100644
--- a/runtime/onnxruntime/src/vocab.cpp
+++ b/runtime/onnxruntime/src/vocab.cpp
@@ -14,7 +14,7 @@
Vocab::Vocab(const char *filename)
{
ifstream in(filename);
- LoadVocabFromYaml(filename);
+ LoadVocabFromJson(filename);
}
Vocab::Vocab(const char *filename, const char *lex_file)
{
@@ -43,6 +43,25 @@
}
}
+void Vocab::LoadVocabFromJson(const char* filename){
+ nlohmann::json json_array;
+ std::ifstream file(filename);
+ if (file.is_open()) {
+ file >> json_array;
+ file.close();
+ } else {
+ LOG(INFO) << "Error loading token file, token file error or not exist.";
+ exit(-1);
+ }
+
+ int i = 0;
+ for (const auto& element : json_array) {
+ vocab.push_back(element);
+ token_id[element] = i;
+ i++;
+ }
+}
+
void Vocab::LoadLex(const char* filename){
std::ifstream file(filename);
std::string line;
--
Gitblit v1.9.1