Struct BasetenTokenizer 

pub struct BasetenTokenizer { /* private fields */ }

Tokenizer backed by the basetenkenizer crate.

Implementations

impl BasetenTokenizer

pub fn from_file(path: &str) -> Result<Self>

Load a tokenizer from a Hugging Face tokenizer.json file.

Trait Implementations

impl Decoder for BasetenTokenizer

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<DecodeResult>

impl Encoder for BasetenTokenizer

fn encode(&self, input: &str) -> Result<Encoding>

fn encode_batch(&self, inputs: &[&str]) -> Result<Vec<Encoding>>

fn encode_segments(&self, segments: &[EncodeSegment<'_>]) -> Result<Encoding>

Encode Kimi K3-style renderer segments while preserving trusted control-token and untrusted content boundaries.

impl Tokenizer for BasetenTokenizer

fn validate_prefix_cache(&self) -> Result<()>

Validate that this tokenizer can be safely wrapped in the prefix cache.

fn with_options(self, options: TokenizerOptions) -> Self

Apply construction-time TokenizerOptions.