DEV/k2-fsa_sherpa-onnx
1
0
Fork 0
mirror of https://github.com/k2-fsa/sherpa-onnx.git synced 2026-01-09 07:41:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support BPE models with byte fallback. (#2531)

Browse Source
This commit is contained in:
Fangjun Kuang 2025-08-26 12:03:02 +08:00 committed by GitHub
parent f45cd87a24
commit 9d0adcd3f5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
sherpa-onnx/csrc/symbol-table.cc
View File

@ -171,6 +171,12 @@ SymbolTable::SymbolTable(Manager *mgr, const std::string &filename) {
void SymbolTable::Init(std::istream &is) {
sym2id_ = ReadTokens(is, &id2sym_);
is_bbpe_ = IsByteBPE(sym2id_);
if (sym2id_.count("<0x00>") && sym2id_.count("<0xFF>") &&
((sym2id_.at("<0xFF>") - sym2id_.at("<0x00>")) == 255)) {
is_bpe_with_byte_fallback_ = true;
id_for_0x00_ = sym2id_.at("<0x00>");
}
}
std::string SymbolTable::ToString() const {
@ -197,13 +203,13 @@ const std::string SymbolTable::operator[](int32_t id) const {
// id 0 is blank, id 1 is sos/eos, id 2 is unk
//
// Note: For moonshine models, 0 is <unk>, 1, is <s>, 2 is</s>
if (id >= 3 && id <= 258 && sym.size() == 6 && sym[0] == '<' &&
if (is_bpe_with_byte_fallback_ && sym.size() == 6 && sym[0] == '<' &&
sym[1] == '0' && sym[2] == 'x' && sym[5] == '>') {
std::ostringstream os;
os << std::hex << std::uppercase << (id - 3);
os << std::hex << std::uppercase << (id - id_for_0x00_);
if (std::string(sym.data() + 3, sym.data() + 5) == os.str()) {
uint8_t i = id - 3;
uint8_t i = id - id_for_0x00_;
sym = std::string(&i, &i + 1);
}
}

9
sherpa-onnx/csrc/symbol-table.h
View File

@ -66,6 +66,15 @@ class SymbolTable {
private:
std::unordered_map<std::string, int32_t> sym2id_;
std::unordered_map<int32_t, std::string> id2sym_;
// see https://github.com/k2-fsa/sherpa-onnx/issues/2524
bool is_bpe_with_byte_fallback_ = false;
// used only when is_bpe_with_byte_fallback_ is true. It is the ID
// of <0x00> in tokens.txt
int32_t id_for_0x00_ = 0;
// true for byte BPE. false for non byte BPE.
bool is_bbpe_ = false;
};