oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Tokenizer quality metrics on held-out text.
//!
//! Reports compression, coverage, and encode/decode throughput without needing
//! model weights.

use crate::tokenizer::BpeTokenizer;
use std::collections::HashSet;
use std::fmt::Write as _;
use std::hint::black_box;
use std::time::Instant;

/// Quality metrics for a tokenizer evaluated on held-out bytes.
#[derive(Clone, Debug, PartialEq)]
pub struct TokenizerQualityReport {
    pub bytes: usize,
    pub characters: usize,
    pub tokens: usize,
    pub unique_tokens: usize,
    pub unknown_tokens: usize,
    pub vocab_size: usize,
    /// Input bytes per emitted token (higher means stronger compression).
    pub compression_ratio: f64,
    pub tokens_per_character: f64,
    /// Fraction of emitted tokens equal to the configured UNK id.
    pub unknown_token_rate: f64,
    /// Fraction of vocabulary ids observed at least once in the encoding.
    pub vocabulary_utilization: f64,
    pub encode_bytes_per_second: f64,
    pub encode_tokens_per_second: f64,
    pub decode_bytes_per_second: f64,
    pub decode_tokens_per_second: f64,
    pub throughput_repeats: usize,
}

/// Evaluate `tokenizer` on UTF-8 `text`.
pub fn evaluate(
    tokenizer: &BpeTokenizer,
    text: &str,
    throughput_repeats: usize,
) -> TokenizerQualityReport {
    evaluate_bytes(tokenizer, text.as_bytes(), throughput_repeats)
}

/// Evaluate `tokenizer` on arbitrary held-out `bytes`.
///
/// Throughput is measured over `throughput_repeats` timed passes after one
/// warm-up encode/decode. Metrics use the encoding from the timed encode loop.
pub fn evaluate_bytes(
    tokenizer: &BpeTokenizer,
    bytes: &[u8],
    throughput_repeats: usize,
) -> TokenizerQualityReport {
    let repeats = throughput_repeats.max(1);
    let characters = match std::str::from_utf8(bytes) {
        Ok(text) => text.chars().count(),
        Err(_) => bytes.len(),
    };

    // Warm up so the timed loop excludes first-touch allocator costs.
    let warm_tokens = tokenizer.encode_bytes(bytes);
    let _ = black_box(tokenizer.decode_bytes(&warm_tokens));

    let encode_started = Instant::now();
    let mut tokens = warm_tokens;
    for _ in 0..repeats {
        tokens = tokenizer.encode_bytes(bytes);
        black_box(&tokens);
    }
    let encode_seconds = encode_started.elapsed().as_secs_f64().max(f64::EPSILON);

    let decode_started = Instant::now();
    let mut decoded = Vec::new();
    for _ in 0..repeats {
        decoded = tokenizer.decode_bytes(&tokens);
        black_box(&decoded);
    }
    let decode_seconds = decode_started.elapsed().as_secs_f64().max(f64::EPSILON);

    let token_count = tokens.len();
    let unk = tokenizer.special_tokens.unk;
    let unknown_tokens = tokens.iter().filter(|&&token| token == unk).count();
    let unique_tokens = tokens.iter().copied().collect::<HashSet<_>>().len();
    let vocab_size = tokenizer.vocab_size();

    let compression_ratio = if token_count == 0 {
        0.0
    } else {
        bytes.len() as f64 / token_count as f64
    };
    let tokens_per_character = if characters == 0 {
        0.0
    } else {
        token_count as f64 / characters as f64
    };
    let unknown_token_rate = if token_count == 0 {
        0.0
    } else {
        unknown_tokens as f64 / token_count as f64
    };
    let vocabulary_utilization = if vocab_size == 0 {
        0.0
    } else {
        unique_tokens as f64 / vocab_size as f64
    };

    let total_encode_bytes = bytes.len().saturating_mul(repeats) as f64;
    let total_encode_tokens = token_count.saturating_mul(repeats) as f64;
    let total_decode_bytes = decoded.len().saturating_mul(repeats) as f64;
    let total_decode_tokens = token_count.saturating_mul(repeats) as f64;

    TokenizerQualityReport {
        bytes: bytes.len(),
        characters,
        tokens: token_count,
        unique_tokens,
        unknown_tokens,
        vocab_size,
        compression_ratio,
        tokens_per_character,
        unknown_token_rate,
        vocabulary_utilization,
        encode_bytes_per_second: total_encode_bytes / encode_seconds,
        encode_tokens_per_second: total_encode_tokens / encode_seconds,
        decode_bytes_per_second: total_decode_bytes / decode_seconds,
        decode_tokens_per_second: total_decode_tokens / decode_seconds,
        throughput_repeats: repeats,
    }
}

/// Format a human-readable quality report.
pub fn format_report(report: &TokenizerQualityReport) -> String {
    let mut output = String::new();
    let _ = writeln!(
        output,
        "tokenizer_quality bytes={} characters={} tokens={} unique_tokens={} unknown_tokens={} vocab_size={}",
        report.bytes,
        report.characters,
        report.tokens,
        report.unique_tokens,
        report.unknown_tokens,
        report.vocab_size
    );
    let _ = writeln!(
        output,
        "tokenizer_quality compression_ratio={:.4} tokens_per_character={:.4} unknown_token_rate={:.6} vocabulary_utilization={:.4}",
        report.compression_ratio,
        report.tokens_per_character,
        report.unknown_token_rate,
        report.vocabulary_utilization
    );
    let _ = writeln!(
        output,
        "tokenizer_quality encode_bytes_per_second={:.1} encode_tokens_per_second={:.1} decode_bytes_per_second={:.1} decode_tokens_per_second={:.1} repeats={}",
        report.encode_bytes_per_second,
        report.encode_tokens_per_second,
        report.decode_bytes_per_second,
        report.decode_tokens_per_second,
        report.throughput_repeats
    );
    output
}

#[cfg(test)]
mod tests {
    use super::{evaluate, evaluate_bytes, format_report};
    use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};

    #[test]
    fn quality_report_measures_compression_and_utilization() {
        let corpus = "red fish blue fish red fish blue fish";
        let tokenizer =
            BpeTokenizer::train_from_bytes(corpus.as_bytes(), FIRST_MERGE_ID + 16).unwrap();
        let held_out = "blue fish red";

        let report = evaluate(&tokenizer, held_out, 2);

        assert_eq!(report.bytes, held_out.len());
        assert_eq!(report.characters, held_out.chars().count());
        assert_eq!(report.tokens, tokenizer.encode(held_out).len());
        assert!(report.tokens > 0);
        assert!(report.tokens < held_out.len());
        assert!(report.compression_ratio > 1.0);
        assert!(report.tokens_per_character > 0.0);
        assert!(report.tokens_per_character < 1.0);
        assert_eq!(report.unknown_tokens, 0);
        assert_eq!(report.unknown_token_rate, 0.0);
        assert!(report.unique_tokens > 0);
        assert!(report.unique_tokens <= report.vocab_size);
        assert!(report.vocabulary_utilization > 0.0);
        assert!(report.vocabulary_utilization <= 1.0);
        assert!(report.encode_bytes_per_second > 0.0);
        assert!(report.encode_tokens_per_second > 0.0);
        assert!(report.decode_bytes_per_second > 0.0);
        assert!(report.decode_tokens_per_second > 0.0);
        assert_eq!(report.throughput_repeats, 2);

        let formatted = format_report(&report);
        assert!(formatted.contains("compression_ratio="));
        assert!(formatted.contains("tokens_per_character="));
        assert!(formatted.contains("unknown_token_rate="));
        assert!(formatted.contains("vocabulary_utilization="));
        assert!(formatted.contains("encode_bytes_per_second="));
        assert!(formatted.contains("decode_tokens_per_second="));
    }

    #[test]
    fn empty_input_reports_zeros() {
        let tokenizer = BpeTokenizer::train_from_bytes(b"abab", FIRST_MERGE_ID + 1).unwrap();
        let report = evaluate_bytes(&tokenizer, b"", 1);
        assert_eq!(report.bytes, 0);
        assert_eq!(report.tokens, 0);
        assert_eq!(report.compression_ratio, 0.0);
        assert_eq!(report.tokens_per_character, 0.0);
        assert_eq!(report.unknown_token_rate, 0.0);
        assert_eq!(report.vocabulary_utilization, 0.0);
    }

    #[test]
    fn byte_level_encode_has_zero_unknown_rate() {
        let tokenizer = BpeTokenizer::new();
        let report = evaluate(&tokenizer, "any utf-8 ☃", 1);
        assert_eq!(report.unknown_tokens, 0);
        assert_eq!(report.unknown_token_rate, 0.0);
        assert_eq!(report.tokens, report.bytes);
        assert_eq!(report.compression_ratio, 1.0);
    }
}