toktier 0.3.3

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
use toktier::{Device, Runtime};

fn main() -> toktier::Result<()> {
    let tokenizer = Runtime::builder()
        .device(Device::Cpu)
        .build()?
        .load("qwen3_8b")?;
    let mut session = tokenizer.open_session("agent-42")?;
    let seed = session.seed("The agent considered an internation")?;
    let mut downstream = seed.ids().to_vec();
    let patch = session.append("alization proposal. 你好")?;
    downstream.truncate(patch.keep_tokens() as usize);
    downstream.extend_from_slice(patch.replacement_ids());
    assert_eq!(downstream, session.snapshot()?.ids());
    println!(
        "revision={} keep={} replacement={} total={}",
        patch.revision(),
        patch.keep_tokens(),
        patch.replacement_ids().len(),
        patch.token_count()
    );
    Ok(())
}