Crate fastokens 

Re-exports

pub use self::added_tokens::AddedTokenInfo;

pub use self::added_tokens::AddedTokens;

pub use self::json_structs::AddedTokenConfig;

pub use self::json_structs::DecoderConfig;

pub use self::json_structs::DecoderKind;

pub use self::json_structs::ModelConfig;

pub use self::json_structs::ModelKind;

pub use self::json_structs::NormalizerConfig;

pub use self::json_structs::NormalizerKind;

pub use self::json_structs::PostProcessorConfig;

pub use self::json_structs::PostProcessorKind;

pub use self::json_structs::PreTokenizerConfig;

pub use self::json_structs::PreTokenizerKind;

pub use self::json_structs::TokenizerJson;

pub use self::models::Model;

pub use self::normalizers::Nfc;

pub use self::normalizers::Normalizer;

pub use self::post_processors::PostProcessor;

pub use self::pre_tokenizers::ByteLevel;

pub use self::pre_tokenizers::PreTokenizer;

pub use self::pre_tokenizers::Split;

pub use self::pre_tokenizers::SplitBehavior;

Modules

added_tokens

decoders

json_structs

models

normalizers

post_processors

pre_tokenized

pre_tokenizers

Structs

DecodeStream

Stateful incremental decoder.

Tokenizer

An LLM tokenizer backed by tokenizer.json.

Enums

Error

Errors that can occur when constructing a Tokenizer.

Functions

decode_stream_step

Advance an incremental decode stream by one or more token IDs.