Crate fastokens 

Re-exports

pub use self::added_tokens::AddedTokenInfo;

pub use self::added_tokens::AddedTokens;

pub use self::json_structs::AddedTokenConfig;

pub use self::json_structs::DecoderConfig;

pub use self::json_structs::DecoderKind;

pub use self::json_structs::ModelConfig;

pub use self::json_structs::ModelKind;

pub use self::json_structs::NormalizerConfig;

pub use self::json_structs::NormalizerKind;

pub use self::json_structs::PostProcessorConfig;

pub use self::json_structs::PostProcessorKind;

pub use self::json_structs::PreTokenizerConfig;

pub use self::json_structs::PreTokenizerKind;

pub use self::json_structs::TokenizerConfig;

pub use self::json_structs::TokenizerJson;

pub use self::models::Model;

pub use self::normalizers::Nfc;

pub use self::normalizers::Normalizer;

pub use self::normalizers::Replace;

pub use self::post_processors::PostProcessor;

pub use self::pre_tokenizers::ByteLevel;

pub use self::pre_tokenizers::Pcre2Limits;

pub use self::pre_tokenizers::PreTokenizer;

pub use self::pre_tokenizers::Split;

pub use self::pre_tokenizers::SplitBehavior;

pub use self::tiktoken::CL100K_BASE_PATTERN;

pub use self::tiktoken::KIMI_PATTERN;

pub use self::tiktoken::KIMI_RESERVED_SPECIAL_TOKENS;

pub use self::tiktoken::O200K_BASE_PATTERN;

pub use self::tiktoken::TiktokenConfig;

pub use self::tiktoken::TiktokenFamily;

Modules

added_tokens

decoders

json_structs

models

normalizers

post_processors

pre_tokenized

pre_tokenizers

tiktoken

Support for the tiktoken model format (OpenAI’s .tiktoken / tiktoken.model files, e.g. cl100k_base, o200k_base, or the tiktoken.model shipped by models such as Kimi-K2).

Structs

DecodeStream

Stateful incremental decoder.

EncodeSegment

One piece of input for Tokenizer::encode_segments.

Tokenizer

An LLM tokenizer backed by tokenizer.json.

TokenizerOptions

Options applied while constructing a Tokenizer.

Enums

Error

Errors that can occur when constructing a Tokenizer.

Functions

decode_stream_step

Advance an incremental decode stream by one or more token IDs.