splintr 0.10.0-beta.1

Fast Rust tokenizer (BPE + SentencePiece + WordPiece) with Python bindings
Documentation 
1
2
3
4
5
6

mod bindings;

pub use bindings::{
    from_json, from_json_bytes, register_agent_tokens, PyByteLevelStreamingDecoder,
    PySentencePieceTokenizer, PyStreamingDecoder, PyTokenizer, PyWordPieceTokenizer,
};