mtc_incremental_bpe/
lib.rs

1mod aho_corasick;
2mod centroid;
3mod dict;
4mod eager;
5mod inc_bpe;
6mod normalize;
7mod successor;
8mod suf_suc;
9mod typed_vec;
10mod vocab;
11
12pub use crate::{
13    dict::{DictBuildError, Dictionary, Rule, RuleId, UnknownToken},
14    eager::EagerBpeTokenization,
15    inc_bpe::{IncBpeToken, IncBpeTokenChainIter, IncBpeTokenization, IncBpeTokenizer},
16    normalize::{NormalizedDict, NormalizedDictBuildError},
17    successor::SkipLen,
18    vocab::{Token, TokenId, Vocab, VocabBuildError},
19};
20
21#[cfg(test)]
22mod test_utils;