support en-bpe model

雾聪

2023-10-10 4984724f6a580fdefc4dcec430e69fd75635d25b

 funasr/runtime/onnxruntime/src/ct-transformer.cpp

@@ -46,7 +46,7 @@
{
}

string CTTransformer::AddPunc(const char* sz_input)
string CTTransformer::AddPunc(const char* sz_input, std::string language)
{
    string strResult;
    vector<string> strOut;
@@ -139,8 +139,28 @@
            }
        }
    }
    for (auto& item : NewSentenceOut)

    for (auto& item : NewSentenceOut){
        strResult += item;
    }
    
    if(language == "en-bpe"){
        std::vector<std::string> chineseSymbols;
        chineseSymbols.push_back("，");
        chineseSymbols.push_back("。");
        chineseSymbols.push_back("、");
        chineseSymbols.push_back("？");

        std::string englishSymbols = ",.,?";
        for (size_t i = 0; i < chineseSymbols.size(); i++) {
            size_t pos = 0;
            while ((pos = strResult.find(chineseSymbols[i], pos)) != std::string::npos) {
                strResult.replace(pos, 3, 1, englishSymbols[i]);
                pos++;
            }
        }
    }

    return strResult;
}