1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
| #ifndef VOCAB_H
| #define VOCAB_H
|
| #include <stdint.h>
| #include <string>
| #include <vector>
| #include <map>
| using namespace std;
|
| namespace funasr {
| class Vocab {
| private:
| vector<string> vocab;
| std::map<string, int> token_id;
| bool IsEnglish(string ch);
| void LoadVocabFromYaml(const char* filename);
|
| public:
| Vocab(const char *filename);
| ~Vocab();
| int Size();
| bool IsChinese(string ch);
| void Vector2String(vector<int> in, std::vector<std::string> &preds);
| string Vector2StringV2(vector<int> in, std::string language="");
| int GetIdByToken(const std::string &token);
| };
|
| } // namespace funasr
| #endif
|
|
