mtc-inc-bpe 0.9.0

Incremental BPE tokenization for all prefixes
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
mod automaton;
mod heavy_light;
mod index;
mod relabeling;
mod suf_link_tree;
mod trans;
mod trie;

pub(crate) use self::{
    automaton::ACAutomaton,
    index::{AC_NODE_ROOT, ACNodeId, ACNodeIdInlineVec},
    suf_link_tree::ACSuffixLinkTree,
    trans::ACTransTable,
    trie::ACTrie,
};