toktier 0.3.3

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
#[cfg(all(feature = "serving", feature = "prebuilt-gpu"))]
fn main() -> toktier::Result<()> {
    use std::collections::BTreeMap;
    use std::path::PathBuf;
    use std::time::{Duration, Instant};

    use serde::Serialize;
    use toktier::{Device, GpuDelivery, Policy, Runtime, ServingLimits, ServingPool};

    #[derive(Serialize)]
    struct Summary {
        schema: &'static str,
        documents: usize,
        bytes_per_document: usize,
        utf8_bytes: usize,
        devices: Vec<u32>,
        worker_threads: usize,
        queued_total_us: u64,
        docs_per_second: f64,
        gib_per_second: f64,
        response_total_p50_us: u64,
        response_total_p95_us: u64,
        response_total_p99_us: u64,
        queue_p50_us: u64,
        queue_p95_us: u64,
        queue_p99_us: u64,
        engine_p50_us: u64,
        engine_p95_us: u64,
        engine_p99_us: u64,
        maximum_observed_batch_rows: usize,
        device_rows: BTreeMap<usize, usize>,
        exact_matches: usize,
    }

    fn micros(value: Duration) -> u64 {
        value.as_micros().min(u128::from(u64::MAX)) as u64
    }

    fn percentile(values: &mut [u64], percentile: usize) -> u64 {
        values.sort_unstable();
        values[(values.len().saturating_sub(1) * percentile) / 100]
    }

    let artifact_cache = std::env::var_os("TOKTIER_ARTIFACT_CACHE")
        .map(PathBuf::from)
        .ok_or_else(|| std::io::Error::other("set TOKTIER_ARTIFACT_CACHE"))?;
    let devices = std::env::var("TOKTIER_SERVING_CUDA_DEVICES")
        .unwrap_or_else(|_| "0".to_owned())
        .split(',')
        .map(|value| {
            value
                .parse::<u32>()
                .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))
        })
        .collect::<std::io::Result<Vec<_>>>()?;
    if devices.is_empty() {
        return Err(std::io::Error::other("at least one CUDA device is required").into());
    }
    let documents = std::env::var("TOKTIER_SERVING_DOCUMENTS")
        .ok()
        .and_then(|value| value.parse::<usize>().ok())
        .unwrap_or(256);
    let bytes_per_document = std::env::var("TOKTIER_SERVING_BYTES")
        .ok()
        .and_then(|value| value.parse::<usize>().ok())
        .unwrap_or(64 * 1024);
    if documents == 0 || bytes_per_document == 0 {
        return Err(std::io::Error::other("document and byte counts must be non-zero").into());
    }

    let inputs = (0..documents)
        .map(|index| {
            let unit = format!("gpu-serving-row-{index:04}-exact-ids-0123456789 ");
            let mut text = unit.repeat(bytes_per_document.div_ceil(unit.len()));
            text.truncate(bytes_per_document);
            text
        })
        .collect::<Vec<_>>();
    let reference = Runtime::builder()
        .artifact_cache(&artifact_cache)
        .device(Device::Cpu)
        .gpu_delivery(GpuDelivery::Disabled)
        .policy(Policy::Reference)
        .build()?
        .load("qwen3_8b")?;
    let expected = inputs
        .iter()
        .map(|text| {
            reference
                .encode(text)
                .map(|encoding| encoding.ids().to_vec())
        })
        .collect::<toktier::Result<Vec<_>>>()?;

    let mut tokenizers = devices
        .iter()
        .map(|device| {
            Runtime::builder()
                .artifact_cache(&artifact_cache)
                .device(Device::Cuda(*device))
                .gpu_delivery(GpuDelivery::Prebuilt)
                .gpu_min_bytes(1)
                .policy(Policy::Certified)
                .build()?
                .load("qwen3_8b")
        })
        .collect::<toktier::Result<Vec<_>>>()?;
    let first = tokenizers.remove(0);
    let mut builder = ServingPool::builder(first);
    for tokenizer in tokenizers {
        builder = builder.device(tokenizer);
    }
    let utf8_bytes = documents.saturating_mul(bytes_per_document);
    let worker_threads = devices.len();
    let pool = builder
        .limits(ServingLimits {
            max_queued_requests: documents,
            max_queued_bytes: utf8_bytes,
            max_batch_rows: 64,
            max_batch_bytes: 4 * 1024 * 1024,
            max_session_requests: 16,
            batch_window: Duration::from_micros(200),
            worker_threads,
        })
        .build()?;

    let started = Instant::now();
    let pending = inputs
        .into_iter()
        .map(|text| pool.submit(text))
        .collect::<toktier::Result<Vec<_>>>()?;
    let mut responses = Vec::with_capacity(documents);
    for request in pending {
        responses.push(request.wait()?);
    }
    let elapsed = started.elapsed();
    pool.shutdown();
    let exact_matches = responses
        .iter()
        .zip(&expected)
        .filter(|(actual, expected)| actual.value.ids() == expected.as_slice())
        .count();
    if exact_matches != documents {
        return Err(std::io::Error::other("GPU serving output diverged").into());
    }

    let mut total = responses
        .iter()
        .map(|row| micros(row.timings.total))
        .collect::<Vec<_>>();
    let mut queue = responses
        .iter()
        .map(|row| micros(row.timings.queue))
        .collect::<Vec<_>>();
    let mut engine = responses
        .iter()
        .map(|row| micros(row.timings.engine))
        .collect::<Vec<_>>();
    let mut device_rows = BTreeMap::new();
    for row in &responses {
        *device_rows.entry(row.timings.device_index).or_insert(0) += 1;
    }
    let seconds = elapsed.as_secs_f64();
    let summary = Summary {
        schema: "toktier.rust_gpu_serving.performance.v1",
        documents,
        bytes_per_document,
        utf8_bytes,
        devices,
        worker_threads,
        queued_total_us: micros(elapsed),
        docs_per_second: documents as f64 / seconds,
        gib_per_second: utf8_bytes as f64 / (1024.0 * 1024.0 * 1024.0) / seconds,
        response_total_p50_us: percentile(&mut total, 50),
        response_total_p95_us: percentile(&mut total, 95),
        response_total_p99_us: percentile(&mut total, 99),
        queue_p50_us: percentile(&mut queue, 50),
        queue_p95_us: percentile(&mut queue, 95),
        queue_p99_us: percentile(&mut queue, 99),
        engine_p50_us: percentile(&mut engine, 50),
        engine_p95_us: percentile(&mut engine, 95),
        engine_p99_us: percentile(&mut engine, 99),
        maximum_observed_batch_rows: responses
            .iter()
            .map(|row| row.timings.batch_rows)
            .max()
            .unwrap_or(0),
        device_rows,
        exact_matches,
    };
    println!(
        "{}",
        serde_json::to_string_pretty(&summary)
            .map_err(|error| std::io::Error::other(error.to_string()))?
    );
    Ok(())
}

#[cfg(not(all(feature = "serving", feature = "prebuilt-gpu")))]
fn main() {
    eprintln!("build with the serving and prebuilt-gpu features");
}