splintr 0.9.1

Fast Rust tokenizer (BPE + SentencePiece + WordPiece) with Python bindings
Documentation 
1
2
3
4
5
6

mod bindings;

pub use bindings::{
    register_agent_tokens, PyByteLevelStreamingDecoder, PySentencePieceTokenizer,
    PyStreamingDecoder, PyTokenizer,
};