Struct SapientTokenizer 

pub struct SapientTokenizer {
    pub bos_id: Option<u32>,
    pub eos_id: Option<u32>,
    pub eos_ids: Vec<u32>,
    pub pad_id: Option<u32>,
    /* private fields */
}

Fields

bos_id: Option<u32>

eos_id: Option<u32>

eos_ids: Vec<u32>

Every EOS/turn-end token id present in this tokenizer’s vocab.

pad_id: Option<u32>

Implementations

impl SapientTokenizer

pub fn from_file(path: &Path, opts: TokenizerOptions) -> Result<Self>

Load from a tokenizer.json file.

pub fn from_pretrained(model_id: &str) -> Result<Self>

Load from a HuggingFace model ID string (uses the HF Hub cache).

pub fn encode(&self, text: &str) -> Result<Vec<u32>>

Encode a text string to token IDs.

pub fn decode(&self, ids: &[u32], skip_special: bool) -> Result<String>

Decode token IDs back to a string.

pub fn decode_token(&self, id: u32) -> Result<String>

Decode a single token ID to a string (for streaming).

pub fn vocab_size(&self) -> usize

Vocabulary size.

pub fn is_eos(&self, id: u32) -> bool

True if id is any of this tokenizer’s end-of-sequence markers.