Struct KimiTokenizer 

pub struct KimiTokenizer { /* private fields */ }

Implementations

impl KimiTokenizer

pub fn new( encoder: HashMap<Vec<u8>, u32>, special_tokens: HashMap<String, u32>, ) -> Result<Self, KimiTokenizerError>

pub fn vocab_size(&self) -> usize

pub fn special_token_id(&self, name: &str) -> Option<u32>

pub fn encode(&self, text: &str) -> Vec<u32>

Encodes ordinary text (no inline special-token recognition – matches the real disallowed_special=() path used for untrusted/user text in tokenization_kimi.py, so a literal <|...|> substring in the input is BPE-encoded like any other text, never misread as a control token).

pub fn decode(&self, ids: &[u32]) -> String

Trait Implementations

impl TextTokenizer for KimiTokenizer

fn encode(&self, text: &str) -> Vec<usize>

fn decode(&self, ids: &[usize]) -> String