tokeneer 0.0.0

Another tokenizer crate
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#![deny(warnings)]

mod bpe;
mod functions;
mod lpe;
mod tokeneer;

pub use bpe::Bpe;
pub use lpe::Lpe;
pub use tokeneer::Tokeneer;

/// `utok` for token id.
#[allow(non_camel_case_types)]
pub type utok = u32;

pub trait Method {
    fn unk_token(&self) -> utok;
    fn vocab_size(&self) -> usize;
    fn internal_special(&self) -> impl IntoIterator<Item = (&str, utok)>;
    fn encode(&self, text: &str) -> impl IntoIterator<Item = utok> + '_;
    fn decode(&self, token: utok) -> &[u8];
}