use crate::tokenizer::BpeTokenizer;
use std::collections::HashSet;
use std::fmt::Write as _;
use std::hint::black_box;
use std::time::Instant;
#[derive(Clone, Debug, PartialEq)]
pub struct TokenizerQualityReport {
pub bytes: usize,
pub characters: usize,
pub tokens: usize,
pub unique_tokens: usize,
pub unknown_tokens: usize,
pub vocab_size: usize,
pub compression_ratio: f64,
pub tokens_per_character: f64,
pub unknown_token_rate: f64,
pub vocabulary_utilization: f64,
pub encode_bytes_per_second: f64,
pub encode_tokens_per_second: f64,
pub decode_bytes_per_second: f64,
pub decode_tokens_per_second: f64,
pub throughput_repeats: usize,
}
pub fn evaluate(
tokenizer: &BpeTokenizer,
text: &str,
throughput_repeats: usize,
) -> TokenizerQualityReport {
evaluate_bytes(tokenizer, text.as_bytes(), throughput_repeats)
}
pub fn evaluate_bytes(
tokenizer: &BpeTokenizer,
bytes: &[u8],
throughput_repeats: usize,
) -> TokenizerQualityReport {
let repeats = throughput_repeats.max(1);
let characters = match std::str::from_utf8(bytes) {
Ok(text) => text.chars().count(),
Err(_) => bytes.len(),
};
let warm_tokens = tokenizer.encode_bytes(bytes);
let _ = black_box(tokenizer.decode_bytes(&warm_tokens));
let encode_started = Instant::now();
let mut tokens = warm_tokens;
for _ in 0..repeats {
tokens = tokenizer.encode_bytes(bytes);
black_box(&tokens);
}
let encode_seconds = encode_started.elapsed().as_secs_f64().max(f64::EPSILON);
let decode_started = Instant::now();
let mut decoded = Vec::new();
for _ in 0..repeats {
decoded = tokenizer.decode_bytes(&tokens);
black_box(&decoded);
}
let decode_seconds = decode_started.elapsed().as_secs_f64().max(f64::EPSILON);
let token_count = tokens.len();
let unk = tokenizer.special_tokens.unk;
let unknown_tokens = tokens.iter().filter(|&&token| token == unk).count();
let unique_tokens = tokens.iter().copied().collect::<HashSet<_>>().len();
let vocab_size = tokenizer.vocab_size();
let compression_ratio = if token_count == 0 {
0.0
} else {
bytes.len() as f64 / token_count as f64
};
let tokens_per_character = if characters == 0 {
0.0
} else {
token_count as f64 / characters as f64
};
let unknown_token_rate = if token_count == 0 {
0.0
} else {
unknown_tokens as f64 / token_count as f64
};
let vocabulary_utilization = if vocab_size == 0 {
0.0
} else {
unique_tokens as f64 / vocab_size as f64
};
let total_encode_bytes = bytes.len().saturating_mul(repeats) as f64;
let total_encode_tokens = token_count.saturating_mul(repeats) as f64;
let total_decode_bytes = decoded.len().saturating_mul(repeats) as f64;
let total_decode_tokens = token_count.saturating_mul(repeats) as f64;
TokenizerQualityReport {
bytes: bytes.len(),
characters,
tokens: token_count,
unique_tokens,
unknown_tokens,
vocab_size,
compression_ratio,
tokens_per_character,
unknown_token_rate,
vocabulary_utilization,
encode_bytes_per_second: total_encode_bytes / encode_seconds,
encode_tokens_per_second: total_encode_tokens / encode_seconds,
decode_bytes_per_second: total_decode_bytes / decode_seconds,
decode_tokens_per_second: total_decode_tokens / decode_seconds,
throughput_repeats: repeats,
}
}
pub fn format_report(report: &TokenizerQualityReport) -> String {
let mut output = String::new();
let _ = writeln!(
output,
"tokenizer_quality bytes={} characters={} tokens={} unique_tokens={} unknown_tokens={} vocab_size={}",
report.bytes,
report.characters,
report.tokens,
report.unique_tokens,
report.unknown_tokens,
report.vocab_size
);
let _ = writeln!(
output,
"tokenizer_quality compression_ratio={:.4} tokens_per_character={:.4} unknown_token_rate={:.6} vocabulary_utilization={:.4}",
report.compression_ratio,
report.tokens_per_character,
report.unknown_token_rate,
report.vocabulary_utilization
);
let _ = writeln!(
output,
"tokenizer_quality encode_bytes_per_second={:.1} encode_tokens_per_second={:.1} decode_bytes_per_second={:.1} decode_tokens_per_second={:.1} repeats={}",
report.encode_bytes_per_second,
report.encode_tokens_per_second,
report.decode_bytes_per_second,
report.decode_tokens_per_second,
report.throughput_repeats
);
output
}
#[cfg(test)]
mod tests {
use super::{evaluate, evaluate_bytes, format_report};
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
#[test]
fn quality_report_measures_compression_and_utilization() {
let corpus = "red fish blue fish red fish blue fish";
let tokenizer =
BpeTokenizer::train_from_bytes(corpus.as_bytes(), FIRST_MERGE_ID + 16).unwrap();
let held_out = "blue fish red";
let report = evaluate(&tokenizer, held_out, 2);
assert_eq!(report.bytes, held_out.len());
assert_eq!(report.characters, held_out.chars().count());
assert_eq!(report.tokens, tokenizer.encode(held_out).len());
assert!(report.tokens > 0);
assert!(report.tokens < held_out.len());
assert!(report.compression_ratio > 1.0);
assert!(report.tokens_per_character > 0.0);
assert!(report.tokens_per_character < 1.0);
assert_eq!(report.unknown_tokens, 0);
assert_eq!(report.unknown_token_rate, 0.0);
assert!(report.unique_tokens > 0);
assert!(report.unique_tokens <= report.vocab_size);
assert!(report.vocabulary_utilization > 0.0);
assert!(report.vocabulary_utilization <= 1.0);
assert!(report.encode_bytes_per_second > 0.0);
assert!(report.encode_tokens_per_second > 0.0);
assert!(report.decode_bytes_per_second > 0.0);
assert!(report.decode_tokens_per_second > 0.0);
assert_eq!(report.throughput_repeats, 2);
let formatted = format_report(&report);
assert!(formatted.contains("compression_ratio="));
assert!(formatted.contains("tokens_per_character="));
assert!(formatted.contains("unknown_token_rate="));
assert!(formatted.contains("vocabulary_utilization="));
assert!(formatted.contains("encode_bytes_per_second="));
assert!(formatted.contains("decode_tokens_per_second="));
}
#[test]
fn empty_input_reports_zeros() {
let tokenizer = BpeTokenizer::train_from_bytes(b"abab", FIRST_MERGE_ID + 1).unwrap();
let report = evaluate_bytes(&tokenizer, b"", 1);
assert_eq!(report.bytes, 0);
assert_eq!(report.tokens, 0);
assert_eq!(report.compression_ratio, 0.0);
assert_eq!(report.tokens_per_character, 0.0);
assert_eq!(report.unknown_token_rate, 0.0);
assert_eq!(report.vocabulary_utilization, 0.0);
}
#[test]
fn byte_level_encode_has_zero_unknown_rate() {
let tokenizer = BpeTokenizer::new();
let report = evaluate(&tokenizer, "any utf-8 ☃", 1);
assert_eq!(report.unknown_tokens, 0);
assert_eq!(report.unknown_token_rate, 0.0);
assert_eq!(report.tokens, report.bytes);
assert_eq!(report.compression_ratio, 1.0);
}
}