lattice-inference 0.9.0

Pure Rust transformer inference engine — safetensors loading, SIMD matmul, BGE/Qwen3 embeddings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

//! Tokenizer module index and re-exports for WordPiece, BPE, common tokenizer types/loaders, and SentencePiece.
pub mod bpe;
pub mod common;
pub(crate) mod detokenize;
pub mod gemma_bpe;
pub mod sentencepiece;
pub mod wordpiece;

// Re-export everything from wordpiece (was top-level `tokenizer` module)
pub use self::wordpiece::*;
// Re-export key types from other tokenizer modules
pub use self::bpe::BpeTokenizer;
pub use self::common::{TokenizedInput, Tokenizer, load_tokenizer, tokenizer_from_json_str};
pub use self::gemma_bpe::{
    GEMMA4_AUDIO_FRAME_LENGTH_SAMPLES, GEMMA4_AUDIO_HOP_LENGTH_SAMPLES,
    GEMMA4_AUDIO_MAX_SOFT_TOKENS, GEMMA4_AUDIO_MS_PER_SOFT_TOKEN, GEMMA4_AUDIO_SAMPLING_RATE_HZ,
    GEMMA4_IMAGE_SOFT_TOKENS_PER_IMAGE, GemmaBpeTokenizer, audio_marker_expansion_tokens,
    audio_marker_expansion_tokens_from_samples, image_marker_expansion_tokens,
    total_audio_marker_expansion_tokens,
};
pub use self::sentencepiece::SentencePieceTokenizer;