Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

High-level tokenizer combining pre-tokenization, encoding, and decoding.

Example

use tokie::Tokenizer;

let tokenizer = Tokenizer::from_json("tokenizer.json")?;
let enc = tokenizer.encode("Hello, world!", false);
let text = tokenizer.decode(&enc.ids);

Implementations

impl Tokenizer

pub fn to_file(&self, path: impl AsRef<Path>) -> Result<(), SerdeError>

Save the tokenizer to a file.

This saves the pre-built DAAC state, enabling fast loading without rebuilding the automaton.

pub fn save<W: Write>(&self, writer: &mut W) -> Result<(), SerdeError>

Save the tokenizer to a writer.

pub fn from_file(path: impl AsRef<Path>) -> Result<Self, SerdeError>

Load a tokenizer from a file.

This loads pre-built DAAC state for instant use without rebuilding.

pub fn load<R: Read>(reader: &mut R) -> Result<Self, SerdeError>

Load a tokenizer from a reader.

impl Tokenizer

pub fn new( encoder: Encoder, decoder: Decoder, pretokenizer_type: PretokType, normalizer: Normalizer, post_processor: PostProcessor, ) -> Self

pub fn set_added_tokens(&mut self, tokens: &[AddedTokenSpec])

Set added tokens. Non-normalized tokens are matched on the raw input before pretokenization; normalized: true tokens are matched on each normalized segment against their normalizer-transformed pattern, both like HuggingFace. Call this after the normalizer is in place — the normalized patterns are computed with self.normalizer.

pub fn added_tokens_raw(&self) -> &[AddedTokenSpec]

The added-token list backing the matchers.

pub fn added_tokens_serialized(&self) -> bool

Whether this tokenizer came from a .tkz that stores added tokens (v13+).

pub fn set_special_tokens(&mut self, tokens: Vec<(String, TokenId)>)

Set special token metadata (token string -> ID mapping).

pub fn special_tokens(&self) -> &[(String, TokenId)]

Get special token metadata as (token_string, token_id) pairs.

pub fn pretokenizer_type(&self) -> PretokType

pub fn normalizer(&self) -> &Normalizer

pub fn post_processor(&self) -> &PostProcessor

pub fn encoder_type(&self) -> EncoderType

pub fn decoder_type(&self) -> DecoderType

pub fn encoder(&self) -> &Encoder

pub fn decoder(&self) -> &Decoder

pub fn pretokenizer(&self) -> Option<&Pretokenizer>

pub fn set_pretokenizer(&mut self, pretok: Option<Pretokenizer>)

pub fn vocab_size(&self) -> usize

pub fn pad_token_id(&self) -> Option<TokenId>

pub fn padding(&self) -> Option<&PaddingParams>

pub fn truncation(&self) -> Option<&TruncationParams>

pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize

Number of special tokens added for a single sequence.

pub fn from_json(path: impl AsRef<Path>) -> Result<Self, JsonLoadError>

Load from a HuggingFace tokenizer.json file.

pub fn from_json_with_encoder( path: impl AsRef<Path>, encoder_type: EncoderType, ) -> Result<Self, JsonLoadError>

Load from a HuggingFace tokenizer.json with a specific encoder type.

pub fn enable_padding(&mut self, params: PaddingParams) -> &mut Self

pub fn enable_truncation(&mut self, params: TruncationParams) -> &mut Self

pub fn no_padding(&mut self) -> &mut Self

pub fn no_truncation(&mut self) -> &mut Self

pub fn set_pad_token_id(&mut self, id: TokenId) -> &mut Self

pub fn id_to_token(&self, id: TokenId) -> Option<Cow<'_, str>>

Get the token string for a given token ID. Returns lossy UTF-8 for byte-level tokens that aren’t valid UTF-8.

pub fn token_to_id(&self, token: &str) -> Option<TokenId>

Look up a token string and return its token ID (O(1) after first call).

pub fn get_vocab(&self) -> HashMap<String, TokenId>

Get the full vocabulary as a map from token strings to token IDs.

pub fn token_to_bytes(&self, token: TokenId) -> &[u8]

Get the byte sequence for a token.

pub fn encode(&self, text: &str, add_special_tokens: bool) -> Encoding

Encode text into an Encoding with token IDs, attention mask, and type IDs.

Example

let enc = tokenizer.encode("Hello, world!", true);
println!("{:?}", enc.ids);

pub fn encode_ids(&self, text: &str, add_special_tokens: bool) -> Vec<TokenId>

Encode to bare token ids (truncation + special tokens applied, no Encoding struct, no attention/type-id buffers). The low-latency path for callers that only consume ids.

pub fn encode_with_offsets( &self, text: &str, add_special_tokens: bool, ) -> Encoding

Encode text with byte offsets for each token.

Returns an Encoding with offsets populated — each entry is a (start, end) byte range in the (normalized) input text corresponding to that token.

Special tokens (CLS, SEP, BOS) get offset (0, 0).

Example

let enc = tokenizer.encode_with_offsets("Hello, world!", true);
for (id, (start, end)) in enc.ids.iter().zip(&enc.offsets) {
    println!("token {} -> bytes {}..{}", id, start, end);
}

pub fn encode_pair( &self, text_a: &str, text_b: &str, add_special_tokens: bool, ) -> Encoding

Encode a pair of texts (e.g. for cross-encoder models).

Example

let enc = tokenizer.encode_pair("What is Berlin?", "Berlin is the capital.", true);

pub fn encode_bytes(&self, bytes: &[u8]) -> Vec<TokenId>

Encode raw bytes directly (bypasses pretokenizer and normalizer).

pub fn encode_iter<'a>(&'a self, text: &'a str) -> TokenizeIter<'a>

Streaming iterator over encoded tokens.

pub fn encode_bytes_iter<'a>(&'a self, bytes: &'a [u8]) -> EncoderIter<'a>

Streaming iterator over encoded tokens from bytes (bypasses pretokenizer).

pub fn decode(&self, tokens: &[TokenId]) -> Option<String>

Decode token IDs back to a string, applying text-level post-processing.

Behavior depends on the DecoderType:

• WordPiece: Strips ## continuation prefixes, joins tokens with spaces, and skips special tokens (CLS, SEP, etc.)
• Metaspace (SentencePiece/Unigram): Replaces ▁ with spaces, strips leading space
• ByteLevel (BPE): Direct byte concatenation (already correct)

Returns None if the result is not valid UTF-8.

pub fn decode_bytes(&self, tokens: &[TokenId]) -> Vec<u8>

Raw byte-level decode without text post-processing.

pub fn decode_batch(&self, sequences: &[&[TokenId]]) -> Vec<Option<String>>

Decode multiple token sequences in parallel.

pub fn encode_batch( &self, texts: &[&str], add_special_tokens: bool, ) -> Vec<Encoding>

Encode multiple texts in parallel, with optional padding.

Example

let encodings = tokenizer.encode_batch(&["Hello!", "World"], true);

pub fn encode_batch_flat( &self, texts: &[&str], add_special_tokens: bool, ) -> (Vec<TokenId>, Vec<u64>)

Encode multiple texts in parallel into one contiguous id buffer.

Returns (ids, lens): every document’s token ids concatenated in order, and per-document id counts. This is the zero-materialization bulk contract — no per-document Encoding objects or vectors reach the caller, so bindings can hand the buffers over as flat arrays. Truncation and special tokens apply as in Self::encode_ids; padding does not (bulk consumers reconstruct boundaries from lens).

pub fn encode_files_flat<P: AsRef<Path>>( &self, paths: &[P], separator: &[u8], add_special_tokens: bool, ) -> Result<(Vec<TokenId>, Vec<u64>)>

Encode corpus files in bulk into one contiguous id buffer.

Reads each file’s bytes in Rust, splits every file on the separator byte sequence (documents never span files; an empty separator treats each file as a single document), drops empty documents — matching the usual Python [d for d in text.split(sep) if d] pre-split — and encodes all documents with the parallel bulk pipeline. No text ever crosses a binding boundary, so this is the fastest way to tokenize corpora from disk.

Each document is UTF-8-validated once; documents containing invalid UTF-8 fall back to lossy conversion (invalid sequences become U+FFFD) instead of failing, so arbitrary bytes are safe. Valid documents are borrowed straight from the read buffer — no copies.

Returns (ids, offsets): every document’s token ids concatenated in order, plus document boundaries with offsets.len() == ndocs + 1 — document i is ids[offsets[i] as usize..offsets[i + 1] as usize]. Truncation and special tokens apply as in Self::encode_batch_flat; padding does not.

pub fn count_tokens_files<P: AsRef<Path>>( &self, paths: &[P], separator: &[u8], ) -> Result<usize>

Count tokens across corpus files without materializing ids.

Same file reading, separator splitting, empty-document filtering, and lossy UTF-8 handling as Self::encode_files_flat; returns the total token count over all documents (no special tokens, as in Self::count_tokens).

pub fn count_tokens_batch(&self, texts: &[&str]) -> Vec<usize>

Count tokens for multiple texts in parallel.

pub fn count_tokens(&self, text: &str) -> usize

Count tokens without storing them (no special tokens).

pub fn token_count<'a>(&'a self, text: &'a str) -> TokenCount<'a>

Lazy token count with early termination for comparisons.

Example

if tokenizer.token_count(text) > 8192 {
    println!("text exceeds context window");
}