Struct Tokenizer 

pub struct Tokenizer {
    pub bos_token_id: Option<u32>,
    pub eos_token_id: Option<u32>,
    pub pad_token_id: Option<u32>,
    pub im_start_id: Option<u32>,
    pub im_end_id: Option<u32>,
    pub chat_template: Option<String>,
    pub extra_eos: HashSet<u32>,
    pub add_bos: bool,
    /* private fields */
}

A loaded BPE tokenizer.

Fields

bos_token_id: Option<u32>

Special tokens

eos_token_id: Option<u32>

pad_token_id: Option<u32>

im_start_id: Option<u32>

Chat template special tokens

im_end_id: Option<u32>

chat_template: Option<String>

Jinja chat template carried by the container (spec §6.1); None → hardcoded ChatML fallback.

extra_eos: HashSet<u32>

Extra stop ids from the container’s generation config.

add_bos: bool

Generation prepends BOS (llama post_processor semantics).

Implementations

impl Tokenizer

pub fn from_file(path: impl AsRef<Path>) -> Result<Self, TokenizerError>

Load tokenizer from HuggingFace tokenizer.json file.

pub fn from_bytes(bytes: &[u8]) -> Result<Self, TokenizerError>

Load tokenizer from raw tokenizer.json bytes (CMF VOCAB section).

Examples found in repository

examples/zimage_stepcheck.rs (line 37)

fn main() {
    let a: Vec<String> = std::env::args().collect();
    let model = Arc::new(cortiq_core::CmfModel::open(&a[1]).unwrap());
    let (prompt, hh, ww) = (&a[2], a[3].parse::<usize>().unwrap(), a[4].parse::<usize>().unwrap());
    let (steps, shift, i) = (a[5].parse::<usize>().unwrap(), a[6].parse::<f32>().unwrap(), a[7].parse::<usize>().unwrap());
    let tok = Tokenizer::from_bytes(model.vocab.as_deref().unwrap()).unwrap();
    let ids = cortiq_engine::zimagegen::prompt_ids(&tok, prompt, 512);
    let cap = {
        let _p = cortiq_engine::gpu::pause_gpu();
        cortiq_engine::qwen3te::Qwen3Encoder::from_cmf(&model).unwrap().encode(&ids)
    };
    let dit = ZImageDit::from_cmf(&model).unwrap();
    let sig = zimage::sigmas_torch_f32(steps, shift);
    let t = zimage::t_model(sig[i]);
    let mods = dit.mods_for_steps(&[t]);
    let fs = dit.final_scale_for_steps(&[t]);
    let shape = ZShape::new(hh, ww, ids.len());
    let prep = dit.prepare(&cap, shape, 1, None).unwrap();
    println!("device prepared: {}", prep.device);
    let (c, lh, lw) = (dit.cfg.in_channels, hh / 8, ww / 8);
    let lat = |dir: &str| -> Vec<f32> {
        if i == 0 {
            read_f32(&std::env::var("CMF_INIT_LATENT").unwrap())
        } else {
            read_f32(&format!("{dir}/lat_{i}.f32"))
        }
    };
    let xa = lat(&a[8]);
    // `ZC_NEG=<negative prompt>`: the CFG pair as one batch-2 device
    // forward against the two items stepped one by one.
    if let Ok(neg) = std::env::var("ZC_NEG") {
        let nids = cortiq_engine::zimagegen::prompt_ids(&tok, &neg, 512);
        let ncap = {
            let _p = cortiq_engine::gpu::pause_gpu();
            cortiq_engine::qwen3te::Qwen3Encoder::from_cmf(&model).unwrap().encode(&nids)
        };
        let mut np = dit.prepare(&ncap, ZShape::new(hh, ww, nids.len()), 2, None).unwrap();
        let tok_a = dit.tokens(&xa, &shape);
        let vp = dit.step(&prep, i, &tok_a, &mods, &fs);
        let vn = dit.step(&np, i, &tok_a, &mods, &fs);
        np.device = false;
        let vn_cpu = dit.step(&np, i, &tok_a, &mods, &fs);
        let ok = dit.attach_device_pair(&prep, &np, 3, None);
        println!("pair prepared: {ok}  (neg L = {})", nids.len());
        if let Some((pp, pn)) = dit.step_pair_device(3, shape.n_img, i, &tok_a, &mods, &fs) {
            let nan = pp.iter().chain(&pn).filter(|v| !v.is_finite()).count();
            println!("pair vs singles: pos {:.3e}  neg {:.3e}  non-finite {nan}   single neg dev vs cpu {:.3e}",
                rel(&pp, &vp), rel(&pn, &vn), rel(&vn, &vn_cpu));
        }
        return;
    }
    let tok_a = dit.tokens(&xa, &shape);
    let va_dev = zimage::unpatchify(&dit.step(&prep, i, &tok_a, &mods, &fs), c, lh, lw);
    let mut hp = zimage::ZPrepared { device: false, ..prep };
    let va_cpu = zimage::unpatchify(&dit.step(&hp, i, &tok_a, &mods, &fs), c, lh, lw);
    let va_tr = read_f32(&format!("{}/v_{i}.f32", a[8]));
    println!("step {i} on A's lat: dev vs cpu {:.3e}   cpu vs A's v {:.3e}   dev vs A's v {:.3e}",
        rel(&va_dev, &va_cpu), rel(&va_cpu, &va_tr), rel(&va_dev, &va_tr));
    if let Some(b) = a.get(9) {
        hp.device = true;
        let xb = lat(b);
        let vb_dev = zimage::unpatchify(&dit.step(&hp, i, &dit.tokens(&xb, &shape), &mods, &fs), c, lh, lw);
        println!("inputs A vs B {:.3e}   device outputs {:.3e}   (B's own v {:.3e})",
            rel(&xb, &xa), rel(&vb_dev, &va_dev), rel(&read_f32(&format!("{b}/v_{i}.f32")), &va_dev));
    }
}

pub fn from_json(json: &str) -> Result<Self, TokenizerError>

Load tokenizer from JSON string.

pub fn byte_level() -> Self

Create a minimal tokenizer for testing (byte tokens, no merges).

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to token IDs.

Examples found in repository

examples/topk_probe.rs (line 27)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("usage: topk_probe <model.cmf> <raw text> [k]");
        std::process::exit(2);
    }
    let k: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10);

    let model = Arc::new(CmfModel::open_sharded(&args[1]).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    let ids = pipeline.tokenizer.encode(&args[2]);
    eprintln!("prompt tokens: {}", ids.len());
    eprintln!("last 8 ids: {:?}", &ids[ids.len().saturating_sub(8)..]);

    let logits = pipeline.prefill_next_logits(&ids, None);
    let max = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
    let sum: f64 = logits.iter().map(|&l| ((l - max) as f64).exp()).sum();

    let mut order: Vec<usize> = (0..logits.len()).collect();
    order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]));
    for &i in order.iter().take(k) {
        let p = ((logits[i] - max) as f64).exp() / sum;
        println!(
            "{:>8}  logit {:>9.4}  p {:.6}  {:?}",
            i,
            logits[i],
            p,
            pipeline.tokenizer.decode_token(i as u32)
        );
    }
}

examples/decode_flat.rs (line 38)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 4 {
        eprintln!("usage: decode_flat <model.cmf> <ctx> <tokens> [o1spec|off]");
        std::process::exit(2);
    }
    let path = &args[1];
    let ctx: usize = args[2].parse().expect("ctx");
    let tokens: usize = args[3].parse().expect("tokens");
    let o1spec = args.get(4).cloned().unwrap_or_else(|| "off".to_string());

    let model = Arc::new(CmfModel::open_sharded(path).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    pipeline.set_o1(cortiq_engine::nystrom::O1Cfg::from_spec(
        &o1spec, None, None, None, None,
    ));
    eprintln!("o1_active = {}", pipeline.o1_active());

    // Same synthetic prompt shape as `cortiq bench --ctx`.
    let prompt = "The quick brown fox jumps over the lazy dog. ".repeat(ctx / 8 + 2);
    let mut ids = pipeline.tokenizer.encode(&prompt);
    ids.truncate(ctx);
    assert_eq!(ids.len(), ctx, "prompt too short for ctx");

    // Warm the mmap so the first token is not billed for page faults.
    let _ = pipeline.forward_ids(&ids[..2], None).expect("warm");

    let stamps: Arc<std::sync::Mutex<Vec<Instant>>> = Arc::default();
    let st = stamps.clone();
    let cb: cortiq_engine::TokenCallback = Box::new(move |_t| {
        st.lock().unwrap().push(Instant::now());
        true
    });
    let t0 = Instant::now();
    let _ = pipeline
        .generate_from_ids(&ids, tokens, None, Some(cb))
        .expect("generate");

    let stamps = stamps.lock().unwrap();
    // stamps[0] fires after generation's prefill + the one-off o1 seal:
    // that gap is TTFT, not a decode step.
    println!(
        "ctx={ctx} o1={o1spec} ttft_s={:.3}",
        stamps[0].duration_since(t0).as_secs_f64()
    );
    println!("# idx  ms");
    for i in 1..stamps.len() {
        println!(
            "{:4}  {:7.2}",
            i,
            (stamps[i] - stamps[i - 1]).as_secs_f64() * 1e3
        );
    }
    // Mean of the last half — past any startup transient.
    let half = stamps.len() / 2;
    if stamps.len() > 2 && half >= 1 {
        let dt = (stamps[stamps.len() - 1] - stamps[half]).as_secs_f64();
        let n = (stamps.len() - 1 - half) as f64;
        println!(
            "last-half: {:.2} ms/tok = {:.2} tok/s",
            dt / n * 1e3,
            n / dt
        );
    }
}

pub fn decode(&self, ids: &[u32]) -> String

Decode token IDs back to text. Special tokens are skipped; added tokens are raw text; everything else reverses the byte-level map.

pub fn decode_token(&self, id: u32) -> String

Streaming decode of ONE token: no sequence-level Strip — a per-token strip would eat the ▁-spaces of every SP word.

Examples found in repository

examples/topk_probe.rs (line 44)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("usage: topk_probe <model.cmf> <raw text> [k]");
        std::process::exit(2);
    }
    let k: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10);

    let model = Arc::new(CmfModel::open_sharded(&args[1]).expect("open model"));
    let mut pipeline = Pipeline::from_model(&model, SamplerConfig::default()).expect("pipeline");
    let ids = pipeline.tokenizer.encode(&args[2]);
    eprintln!("prompt tokens: {}", ids.len());
    eprintln!("last 8 ids: {:?}", &ids[ids.len().saturating_sub(8)..]);

    let logits = pipeline.prefill_next_logits(&ids, None);
    let max = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
    let sum: f64 = logits.iter().map(|&l| ((l - max) as f64).exp()).sum();

    let mut order: Vec<usize> = (0..logits.len()).collect();
    order.sort_by(|&a, &b| logits[b].total_cmp(&logits[a]));
    for &i in order.iter().take(k) {
        let p = ((logits[i] - max) as f64).exp() / sum;
        println!(
            "{:>8}  logit {:>9.4}  p {:.6}  {:?}",
            i,
            logits[i],
            p,
            pipeline.tokenizer.decode_token(i as u32)
        );
    }
}

pub fn decode_token_for_hash(&self, id: u32) -> String

Decode one vocabulary entry for Engram’s compressed token map while retaining special tokens.

pub fn decode_for_protocol(&self, ids: &[u32]) -> String

Decode generated protocol text while retaining special markers. The V4.1 harmony parser needs <think>, EOS, and spaced DSML tags.

pub fn raw_token_for_hash(&self, id: u32) -> String

Return the backend vocabulary spelling for an Engram map entry.

pub fn apply_chat_template(&self, messages: &[(String, String)]) -> Vec<u32>

Render the container’s Jinja chat template (HF semantics: trim_blocks + lstrip_blocks + loop controls) and encode it. Falls back to hardcoded ChatML when the file carries none.

pub fn apply_chat_template_json( &self, messages: &[Value], tools: Option<&[Value]>, enable_thinking: Option<bool>, ) -> Vec<u32>

Like apply_chat_template, with an explicit enable_thinking value for reasoning-model templates (Qwen3/3.5 emit an empty block when it is false, so the model answers directly). None leaves the variable undefined — the template’s own default applies. Chat template with the FULL message shape and a tool list.

The pair-based API below flattens every message to (role, text), which silently drops exactly what agentic use needs: the tools array, role: "tool" results, and tool_calls on assistant turns. The templates this format embeds — Qwen-family, Nanbeige — have carried a {%- if tools %} branch all along; this is the call that finally feeds it. Messages arrive as JSON objects in the OpenAI shape and pass through to minijinja unflattened, so a template sees the same fields a Python apply_chat_template would.

pub fn render_chat_json( &self, messages: &[Value], tools: Option<&[Value]>, enable_thinking: Option<bool>, ) -> Option<String>

Render the template against JSON-shaped messages (parity surface).

pub fn apply_chat_template_opts( &self, messages: &[(String, String)], enable_thinking: Option<bool>, ) -> Vec<u32>

pub fn with_bos(&self, ids: Vec<u32>) -> Vec<u32>

Prepend BOS when the tokenizer declares it (llama family).

pub fn render_chat(&self, messages: &[(String, String)]) -> Option<String>

Render the carried template to text (parity-testable surface).

pub fn render_chat_opts( &self, messages: &[(String, String)], enable_thinking: Option<bool>, ) -> Option<String>

Render the carried template to text with explicit thinking mode.

pub fn vocab_size(&self) -> usize

Vocabulary size.

pub fn token_to_id(&self, token: &str) -> Option<u32>

Return the ID for an exact token spelling, including added/special tokens. Multimodal prompt preparation uses this to validate the image placeholder against the model configuration.

pub fn convert_tokens_to_ids(&self, token: &str) -> Option<u32>

Alias matching the HuggingFace tokenizer API used by the official DeepSeek image processor.

pub fn is_eos(&self, id: u32) -> bool

Check if token ID is EOS.

Trait Implementations

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.