Struct Tokenizer 

pub struct Tokenizer {
    pub bos_token_id: Option<u32>,
    pub eos_token_id: Option<u32>,
    pub pad_token_id: Option<u32>,
    pub im_start_id: Option<u32>,
    pub im_end_id: Option<u32>,
    pub chat_template: Option<String>,
    pub extra_eos: HashSet<u32>,
    pub add_bos: bool,
    /* private fields */
}

A loaded BPE tokenizer.

Fields

bos_token_id: Option<u32>

Special tokens

eos_token_id: Option<u32>

pad_token_id: Option<u32>

im_start_id: Option<u32>

Chat template special tokens

im_end_id: Option<u32>

chat_template: Option<String>

Jinja chat template carried by the container (spec §6.1); None → hardcoded ChatML fallback.

extra_eos: HashSet<u32>

Extra stop ids from the container’s generation config.

add_bos: bool

Generation prepends BOS (llama post_processor semantics).

Implementations

impl Tokenizer

pub fn from_file(path: impl AsRef<Path>) -> Result<Self, TokenizerError>

Load tokenizer from HuggingFace tokenizer.json file.

pub fn from_bytes(bytes: &[u8]) -> Result<Self, TokenizerError>

Load tokenizer from raw tokenizer.json bytes (CMF VOCAB section).

pub fn from_json(json: &str) -> Result<Self, TokenizerError>

Load tokenizer from JSON string.

pub fn byte_level() -> Self

Create a minimal tokenizer for testing (byte tokens, no merges).

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to token IDs.

Examples found in repository

examples/topk_probe.rs (line 27)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("usage: topk_probe <model.cmf> <raw text> [k]");
        std::process::exit(2);
    }
    let k: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10);

    let model = Arc::new(CmfModel::open_sharded(&args[1]).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    let ids = pipeline.tokenizer.encode(&args[2]);
    eprintln!("prompt tokens: {}", ids.len());
    eprintln!("last 8 ids: {:?}", &ids[ids.len().saturating_sub(8)..]);

    let logits = pipeline.prefill_next_logits(&ids, None);
    let max = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
    let sum: f64 = logits.iter().map(|&l| ((l - max) as f64).exp()).sum();

    let mut order: Vec<usize> = (0..logits.len()).collect();
    order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]));
    for &i in order.iter().take(k) {
        let p = ((logits[i] - max) as f64).exp() / sum;
        println!(
            "{:>8}  logit {:>9.4}  p {:.6}  {:?}",
            i,
            logits[i],
            p,
            pipeline.tokenizer.decode_token(i as u32)
        );
    }
}

examples/decode_flat.rs (line 38)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 4 {
        eprintln!("usage: decode_flat <model.cmf> <ctx> <tokens> [o1spec|off]");
        std::process::exit(2);
    }
    let path = &args[1];
    let ctx: usize = args[2].parse().expect("ctx");
    let tokens: usize = args[3].parse().expect("tokens");
    let o1spec = args.get(4).cloned().unwrap_or_else(|| "off".to_string());

    let model = Arc::new(CmfModel::open_sharded(path).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    pipeline.set_o1(cortiq_engine::nystrom::O1Cfg::from_spec(
        &o1spec, None, None, None, None,
    ));
    eprintln!("o1_active = {}", pipeline.o1_active());

    // Same synthetic prompt shape as `cortiq bench --ctx`.
    let prompt = "The quick brown fox jumps over the lazy dog. ".repeat(ctx / 8 + 2);
    let mut ids = pipeline.tokenizer.encode(&prompt);
    ids.truncate(ctx);
    assert_eq!(ids.len(), ctx, "prompt too short for ctx");

    // Warm the mmap so the first token is not billed for page faults.
    let _ = pipeline.forward_ids(&ids[..2], None).expect("warm");

    let stamps: Arc<std::sync::Mutex<Vec<Instant>>> = Arc::default();
    let st = stamps.clone();
    let cb: cortiq_engine::TokenCallback = Box::new(move |_t| {
        st.lock().unwrap().push(Instant::now());
        true
    });
    let t0 = Instant::now();
    let _ = pipeline
        .generate_from_ids(&ids, tokens, None, Some(cb))
        .expect("generate");

    let stamps = stamps.lock().unwrap();
    // stamps[0] fires after generation's prefill + the one-off o1 seal:
    // that gap is TTFT, not a decode step.
    println!(
        "ctx={ctx} o1={o1spec} ttft_s={:.3}",
        stamps[0].duration_since(t0).as_secs_f64()
    );
    println!("# idx  ms");
    for i in 1..stamps.len() {
        println!(
            "{:4}  {:7.2}",
            i,
            (stamps[i] - stamps[i - 1]).as_secs_f64() * 1e3
        );
    }
    // Mean of the last half — past any startup transient.
    let half = stamps.len() / 2;
    if stamps.len() > 2 && half >= 1 {
        let dt = (stamps[stamps.len() - 1] - stamps[half]).as_secs_f64();
        let n = (stamps.len() - 1 - half) as f64;
        println!(
            "last-half: {:.2} ms/tok = {:.2} tok/s",
            dt / n * 1e3,
            n / dt
        );
    }
}

pub fn decode(&self, ids: &[u32]) -> String

Decode token IDs back to text. Special tokens are skipped; added tokens are raw text; everything else reverses the byte-level map.

pub fn decode_token(&self, id: u32) -> String

Streaming decode of ONE token: no sequence-level Strip — a per-token strip would eat the ▁-spaces of every SP word.

Examples found in repository

examples/topk_probe.rs (line 44)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("usage: topk_probe <model.cmf> <raw text> [k]");
        std::process::exit(2);
    }
    let k: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10);

    let model = Arc::new(CmfModel::open_sharded(&args[1]).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    let ids = pipeline.tokenizer.encode(&args[2]);
    eprintln!("prompt tokens: {}", ids.len());
    eprintln!("last 8 ids: {:?}", &ids[ids.len().saturating_sub(8)..]);

    let logits = pipeline.prefill_next_logits(&ids, None);
    let max = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
    let sum: f64 = logits.iter().map(|&l| ((l - max) as f64).exp()).sum();

    let mut order: Vec<usize> = (0..logits.len()).collect();
    order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]));
    for &i in order.iter().take(k) {
        let p = ((logits[i] - max) as f64).exp() / sum;
        println!(
            "{:>8}  logit {:>9.4}  p {:.6}  {:?}",
            i,
            logits[i],
            p,
            pipeline.tokenizer.decode_token(i as u32)
        );
    }
}

pub fn apply_chat_template(&self, messages: &[(String, String)]) -> Vec<u32>

Render the container’s Jinja chat template (HF semantics: trim_blocks + lstrip_blocks + loop controls) and encode it. Falls back to hardcoded ChatML when the file carries none.

pub fn apply_chat_template_opts( &self, messages: &[(String, String)], enable_thinking: Option<bool>, ) -> Vec<u32>

Like apply_chat_template, with an explicit enable_thinking value for reasoning-model templates (Qwen3/3.5 emit an empty block when it is false, so the model answers directly). None leaves the variable undefined — the template’s own default applies.

pub fn with_bos(&self, ids: Vec<u32>) -> Vec<u32>

Prepend BOS when the tokenizer declares it (llama family).

pub fn render_chat(&self, messages: &[(String, String)]) -> Option<String>

Render the carried template to text (parity-testable surface).

pub fn render_chat_opts( &self, messages: &[(String, String)], enable_thinking: Option<bool>, ) -> Option<String>

Render the carried template to text with explicit thinking mode.

pub fn vocab_size(&self) -> usize

Vocabulary size.

pub fn is_eos(&self, id: u32) -> bool

Check if token ID is EOS.

Trait Implementations

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.