bytepiece 0.1.0

Rust version of bytepiece tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11

pub const PAD: &str = "<pad>";
pub const BOS: &str = "<bos>";
pub const EOS: &str = "<eos>";

#[derive(Debug)]
#[repr(usize)]
pub enum SpatialToken {
    Pad = 0,
    Bos,
    Eos,
}