mtc-inc-bpe 0.9.0

Incremental BPE tokenization for all prefixes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13

use tinyvec::TinyVec;

use crate::typed_vec::typed_vec_index;

typed_vec_index!(pub(crate) ACNodeId, u32);

pub(crate) const AC_NODE_ROOT: ACNodeId = ACNodeId::ZERO;

pub(crate) type ACNodeIdInlineVec = TinyVec<[ACNodeId; 6]>;

const _: () = {
    assert!(std::mem::size_of::<ACNodeIdInlineVec>() == 32);
};