lattice-inference 0.6.1

Pure Rust transformer inference engine — safetensors loading, SIMD matmul, BGE/Qwen3 embeddings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12

//! Tokenizer module index and re-exports for WordPiece, BPE, common tokenizer types/loaders, and SentencePiece.
pub mod bpe;
pub mod common;
pub mod sentencepiece;
pub mod wordpiece;

// Re-export everything from wordpiece (was top-level `tokenizer` module)
pub use self::wordpiece::*;
// Re-export key types from other tokenizer modules
pub use self::bpe::BpeTokenizer;
pub use self::common::{TokenizedInput, Tokenizer, load_tokenizer, tokenizer_from_json_str};
pub use self::sentencepiece::SentencePieceTokenizer;