toktier 0.3.3

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
use toktier::{DecodeOptions, Device, Runtime};

fn main() -> toktier::Result<()> {
    let runtime = Runtime::builder().device(Device::Cpu).build()?;
    let tokenizer = runtime.load("qwen3_8b")?;
    let encoded = tokenizer.encode("Hello from Rust. 你好!")?;
    println!("family={} ids={:?}", tokenizer.family(), encoded.ids());
    println!(
        "decoded={}",
        tokenizer.decode(encoded.token_buffer(), DecodeOptions::default())?
    );
    println!("route={:?}", tokenizer.plan().backends);
    Ok(())
}