use crate::generate::SamplingConfig;
use crate::gpu::{GpuContext, GpuError};
use crate::model::{CausalLM, ModelError};
use crate::tokenizer::BpeTokenizer;
use crate::{Device, IndexTensor, Tensor, TensorError};
use rand::prelude::*;
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::time::Instant;
pub const FORWARD_VALIDATE_SEED: u64 = 42;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EvalTask {
pub id: String,
pub prompt: String,
pub expected: String,
}
#[derive(Clone, Debug, PartialEq)]
pub struct PerplexityReport {
pub loss: f32,
pub perplexity: f32,
pub tokens_processed: usize,
pub sequences: usize,
pub sequence_length: usize,
pub wall_time_seconds: f64,
}
#[derive(Clone, Debug, PartialEq)]
pub struct NextTokenAccuracyReport {
pub correct: usize,
pub total: usize,
pub accuracy: f64,
}
#[derive(Clone, Debug, PartialEq)]
pub struct TaskResult {
pub id: String,
pub prompt: String,
pub expected: String,
pub generated: String,
pub exact_match: bool,
pub prefix_match: bool,
pub bytes_correct: usize,
pub bytes_total: usize,
}
#[derive(Clone, Debug, PartialEq)]
pub struct TaskSuiteReport {
pub tasks: Vec<TaskResult>,
pub exact_match_rate: f64,
pub prefix_match_rate: f64,
pub byte_accuracy: f64,
}
#[derive(Clone, Debug, PartialEq)]
pub struct QualityReport {
pub perplexity: Option<PerplexityReport>,
pub next_token: Option<NextTokenAccuracyReport>,
pub tasks: Option<TaskSuiteReport>,
}
#[derive(Debug)]
pub enum QualityError {
InvalidData(String),
Model(ModelError),
Tensor(TensorError),
Gpu(GpuError),
Io(std::io::Error),
Json(serde_json::Error),
Generate(crate::generate::GeneratorError),
}
impl fmt::Display for QualityError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidData(message) => write!(formatter, "quality eval error: {message}"),
Self::Model(error) => error.fmt(formatter),
Self::Tensor(error) => error.fmt(formatter),
Self::Gpu(error) => error.fmt(formatter),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => error.fmt(formatter),
Self::Generate(error) => error.fmt(formatter),
}
}
}
impl Error for QualityError {}
impl From<ModelError> for QualityError {
fn from(error: ModelError) -> Self {
Self::Model(error)
}
}
impl From<TensorError> for QualityError {
fn from(error: TensorError) -> Self {
Self::Tensor(error)
}
}
impl From<GpuError> for QualityError {
fn from(error: GpuError) -> Self {
Self::Gpu(error)
}
}
impl From<std::io::Error> for QualityError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for QualityError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl From<crate::generate::GeneratorError> for QualityError {
fn from(error: crate::generate::GeneratorError) -> Self {
Self::Generate(error)
}
}
pub fn builtin_tiny_tasks() -> Vec<EvalTask> {
vec![
EvalTask {
id: "all_speak".into(),
prompt: "All:\n".into(),
expected: "Speak, speak.".into(),
},
EvalTask {
id: "resolved".into(),
prompt: "All:\nResolved. ".into(),
expected: "resolved.".into(),
},
EvalTask {
id: "we_know".into(),
prompt: "All:\nWe know't, ".into(),
expected: "we know't.".into(),
},
EvalTask {
id: "hear_me".into(),
prompt: "First Citizen:\nBefore we proceed any further, ".into(),
expected: "hear me speak.".into(),
},
EvalTask {
id: "patricians".into(),
prompt: "First Citizen:\nWe are accounted poor citizens, the ".into(),
expected: "patricians good.".into(),
},
EvalTask {
id: "corn_price".into(),
prompt: "First Citizen:\nLet us kill him, and we'll have corn at our own ".into(),
expected: "price.".into(),
},
EvalTask {
id: "away".into(),
prompt: "All:\nNo more talking on't; let it be done: ".into(),
expected: "away, away!".into(),
},
EvalTask {
id: "citizens".into(),
prompt: "Second Citizen:\nOne word, good ".into(),
expected: "citizens.".into(),
},
]
}
pub fn load_tasks(path: impl AsRef<Path>) -> Result<Vec<EvalTask>, QualityError> {
let text = fs::read_to_string(path)?;
let tasks: Vec<EvalTask> = serde_json::from_str(&text)?;
if tasks.is_empty() {
return Err(QualityError::InvalidData(
"task file must contain at least one task".into(),
));
}
for task in &tasks {
if task.id.is_empty() {
return Err(QualityError::InvalidData(
"task id must not be empty".into(),
));
}
if task.expected.is_empty() {
return Err(QualityError::InvalidData(format!(
"task '{}' expected continuation must not be empty",
task.id
)));
}
}
Ok(tasks)
}
pub fn evaluate_held_out(
model: &CausalLM,
tokens: &[usize],
sequence_length: usize,
batch_size: usize,
gpu: Option<&GpuContext>,
) -> Result<(PerplexityReport, NextTokenAccuracyReport), QualityError> {
let _no_grad = crate::no_grad();
if sequence_length == 0 {
return Err(QualityError::InvalidData(
"sequence length must be positive".into(),
));
}
if batch_size == 0 {
return Err(QualityError::InvalidData(
"batch size must be positive".into(),
));
}
if sequence_length > model.config.max_sequence_length {
return Err(QualityError::InvalidData(format!(
"sequence length {sequence_length} exceeds model max {}",
model.config.max_sequence_length
)));
}
if tokens.len() <= sequence_length {
return Err(QualityError::InvalidData(
"held-out corpus is too short for one evaluation sequence".into(),
));
}
if let Some(bad) = tokens
.iter()
.copied()
.find(|token| *token >= model.config.vocab_size)
{
return Err(QualityError::InvalidData(format!(
"held-out token id {bad} is outside vocabulary {}",
model.config.vocab_size
)));
}
let started = Instant::now();
let sequence_count = (tokens.len() - 1) / sequence_length;
let mut weighted_loss = 0.0f64;
let mut tokens_processed = 0usize;
let mut correct = 0usize;
for first_sequence in (0..sequence_count).step_by(batch_size) {
let batch = batch_size.min(sequence_count - first_sequence);
let batch_tokens = batch * sequence_length;
let mut inputs = Vec::with_capacity(batch_tokens);
let mut targets = Vec::with_capacity(batch_tokens);
for batch_index in 0..batch {
let start = (first_sequence + batch_index) * sequence_length;
let end = start + sequence_length;
inputs.extend_from_slice(&tokens[start..end]);
targets.extend_from_slice(&tokens[start + 1..=end]);
}
let shape = vec![batch, sequence_length];
let input_ids = IndexTensor::new(inputs, shape.clone());
let target_ids = IndexTensor::new(targets.clone(), shape);
let logits = if let Some(gpu) = gpu {
model.forward_gpu(&input_ids, gpu)?
} else {
model.forward(&input_ids)?
};
let batch_loss = if let Some(gpu) = gpu {
model
.compute_loss_gpu(&logits, &target_ids, gpu)?
.to_cpu()?
.item()
} else {
model.compute_loss(&logits, &target_ids)?.item()
};
let cpu_logits = if logits.device() == Device::Cpu {
logits
} else {
logits.to_cpu()?
};
correct += count_argmax_matches(&cpu_logits, &targets, model.config.vocab_size);
weighted_loss += batch_loss as f64 * batch_tokens as f64;
tokens_processed += batch_tokens;
}
let loss = (weighted_loss / tokens_processed as f64) as f32;
Ok((
PerplexityReport {
loss,
perplexity: loss.exp(),
tokens_processed,
sequences: sequence_count,
sequence_length,
wall_time_seconds: started.elapsed().as_secs_f64(),
},
NextTokenAccuracyReport {
correct,
total: tokens_processed,
accuracy: correct as f64 / tokens_processed as f64,
},
))
}
pub fn evaluate_held_out_text(
model: &CausalLM,
tokenizer: &BpeTokenizer,
text: &str,
sequence_length: usize,
batch_size: usize,
gpu: Option<&GpuContext>,
) -> Result<(PerplexityReport, NextTokenAccuracyReport), QualityError> {
if tokenizer.vocab_size() != model.config.vocab_size {
return Err(QualityError::InvalidData(
"tokenizer and model vocabulary sizes differ".into(),
));
}
let tokens = tokenizer.encode(text);
evaluate_held_out(model, &tokens, sequence_length, batch_size, gpu)
}
pub fn evaluate_tasks(
model: &CausalLM,
tokenizer: &BpeTokenizer,
tasks: &[EvalTask],
gpu: Option<&GpuContext>,
) -> Result<TaskSuiteReport, QualityError> {
let _no_grad = crate::no_grad();
if tasks.is_empty() {
return Err(QualityError::InvalidData(
"task suite must contain at least one task".into(),
));
}
if tokenizer.vocab_size() != model.config.vocab_size {
return Err(QualityError::InvalidData(
"tokenizer and model vocabulary sizes differ".into(),
));
}
let generator = match gpu {
Some(gpu) => crate::generate::Generator::with_gpu(model, tokenizer, gpu),
None => crate::generate::Generator::new(model, tokenizer),
};
let mut results = Vec::with_capacity(tasks.len());
let mut exact_matches = 0usize;
let mut prefix_matches = 0usize;
let mut bytes_correct = 0usize;
let mut bytes_total = 0usize;
for task in tasks {
let expected_tokens = tokenizer.encode(&task.expected);
if expected_tokens.is_empty() {
return Err(QualityError::InvalidData(format!(
"task '{}' expected continuation encodes to zero tokens",
task.id
)));
}
let sampling = SamplingConfig {
max_new_tokens: expected_tokens.len(),
temperature: 0.0,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed: 0,
return_logprobs: false,
top_logprobs: 0,
..Default::default()
};
let generated_ids = generator.generate_ids(&task.prompt, &sampling)?;
let generated = tokenizer.decode(&generated_ids);
let exact_match = generated == task.expected;
let prefix_match = generated.starts_with(&task.expected)
|| task.expected.starts_with(&generated) && !generated.is_empty();
let (task_bytes_correct, task_bytes_total) =
byte_overlap(generated.as_bytes(), task.expected.as_bytes());
if exact_match {
exact_matches += 1;
}
if prefix_match {
prefix_matches += 1;
}
bytes_correct += task_bytes_correct;
bytes_total += task_bytes_total;
results.push(TaskResult {
id: task.id.clone(),
prompt: task.prompt.clone(),
expected: task.expected.clone(),
generated,
exact_match,
prefix_match,
bytes_correct: task_bytes_correct,
bytes_total: task_bytes_total,
});
}
let task_count = results.len() as f64;
Ok(TaskSuiteReport {
tasks: results,
exact_match_rate: exact_matches as f64 / task_count,
prefix_match_rate: prefix_matches as f64 / task_count,
byte_accuracy: if bytes_total == 0 {
0.0
} else {
bytes_correct as f64 / bytes_total as f64
},
})
}
pub fn evaluate_quality(
model: &CausalLM,
tokenizer: &BpeTokenizer,
held_out_text: Option<&str>,
sequence_length: usize,
batch_size: usize,
tasks: Option<&[EvalTask]>,
gpu: Option<&GpuContext>,
) -> Result<QualityReport, QualityError> {
if held_out_text.is_none() && tasks.is_none() {
return Err(QualityError::InvalidData(
"provide held-out text and/or a task suite".into(),
));
}
let mut report = QualityReport {
perplexity: None,
next_token: None,
tasks: None,
};
if let Some(text) = held_out_text {
let (perplexity, next_token) =
evaluate_held_out_text(model, tokenizer, text, sequence_length, batch_size, gpu)?;
report.perplexity = Some(perplexity);
report.next_token = Some(next_token);
}
if let Some(tasks) = tasks {
report.tasks = Some(evaluate_tasks(model, tokenizer, tasks, gpu)?);
}
Ok(report)
}
pub fn validate_forward(
model: &CausalLM,
seed: u64,
batch_size: usize,
sequence_length: usize,
) -> Result<ForwardValidationResult, QualityError> {
let _no_grad = crate::no_grad();
if batch_size == 0 || sequence_length < 2 {
return Err(QualityError::InvalidData(
"forward validation requires batch_size >= 1 and sequence_length >= 2".into(),
));
}
if sequence_length > model.config.max_sequence_length {
return Err(QualityError::InvalidData(format!(
"sequence length {sequence_length} exceeds model max {}",
model.config.max_sequence_length
)));
}
let vocab = model.config.vocab_size.max(1);
let token_count = batch_size.saturating_mul(sequence_length);
let mut rng = StdRng::seed_from_u64(seed);
let mut inputs = Vec::with_capacity(token_count);
let mut targets = Vec::with_capacity(token_count);
for _ in 0..batch_size {
let mut seq = Vec::with_capacity(sequence_length + 1);
for _ in 0..=sequence_length {
seq.push(rng.random_range(0..vocab));
}
inputs.extend_from_slice(&seq[..sequence_length]);
targets.extend_from_slice(&seq[1..=sequence_length]);
}
let shape = vec![batch_size, sequence_length];
let input_ids = IndexTensor::new(inputs, shape.clone());
let target_ids = IndexTensor::new(targets, shape);
let logits = model.forward(&input_ids)?;
let cpu_logits = if logits.device() == Device::Cpu {
logits
} else {
logits.to_cpu()?
};
let data = cpu_logits.data();
let mut logit_nan_count = 0usize;
let mut logit_inf_count = 0usize;
for value in &data {
if value.is_nan() {
logit_nan_count += 1;
} else if value.is_infinite() {
logit_inf_count += 1;
}
}
let loss = model.compute_loss(&cpu_logits, &target_ids)?.item();
let loss_finite = loss.is_finite();
let passed = logit_nan_count == 0 && logit_inf_count == 0 && loss_finite;
Ok(ForwardValidationResult {
seed,
batch_size,
sequence_length,
logit_nan_count,
logit_inf_count,
loss,
loss_finite,
passed,
})
}
#[derive(Clone, Debug, PartialEq)]
pub struct ForwardValidationResult {
pub seed: u64,
pub batch_size: usize,
pub sequence_length: usize,
pub logit_nan_count: usize,
pub logit_inf_count: usize,
pub loss: f32,
pub loss_finite: bool,
pub passed: bool,
}
pub fn evaluate_kl_held_out(
model: &CausalLM,
reference: &CausalLM,
tokens: &[usize],
sequence_length: usize,
batch_size: usize,
) -> Result<KlDivergenceReport, QualityError> {
let _no_grad = crate::no_grad();
if model.config.vocab_size != reference.config.vocab_size {
return Err(QualityError::InvalidData(
"model and reference vocabulary sizes differ".into(),
));
}
if sequence_length == 0 || batch_size == 0 {
return Err(QualityError::InvalidData(
"sequence length and batch size must be positive".into(),
));
}
if sequence_length > model.config.max_sequence_length
|| sequence_length > reference.config.max_sequence_length
{
return Err(QualityError::InvalidData(
"sequence length exceeds model or reference max_sequence_length".into(),
));
}
if tokens.len() <= sequence_length {
return Err(QualityError::InvalidData(
"held-out corpus is too short for one evaluation sequence".into(),
));
}
let vocab = model.config.vocab_size;
if let Some(bad) = tokens.iter().copied().find(|token| *token >= vocab) {
return Err(QualityError::InvalidData(format!(
"held-out token id {bad} is outside vocabulary {vocab}"
)));
}
let sequence_count = (tokens.len() - 1) / sequence_length;
let mut weighted_kl = 0.0f64;
let mut tokens_compared = 0usize;
for first_sequence in (0..sequence_count).step_by(batch_size) {
let batch = batch_size.min(sequence_count - first_sequence);
let batch_tokens = batch * sequence_length;
let mut inputs = Vec::with_capacity(batch_tokens);
for batch_index in 0..batch {
let start = (first_sequence + batch_index) * sequence_length;
let end = start + sequence_length;
inputs.extend_from_slice(&tokens[start..end]);
}
let shape = vec![batch, sequence_length];
let input_ids = IndexTensor::new(inputs, shape);
let model_logits = model.forward(&input_ids)?.to_cpu()?;
let ref_logits = reference.forward(&input_ids)?.to_cpu()?;
let batch_kl = mean_kl_logits(&ref_logits, &model_logits, vocab)?;
weighted_kl += batch_kl * batch_tokens as f64;
tokens_compared += batch_tokens;
}
Ok(KlDivergenceReport {
mean_kl: weighted_kl / tokens_compared as f64,
tokens_compared,
sequences: sequence_count,
sequence_length,
})
}
pub fn evaluate_kl_held_out_text(
model: &CausalLM,
reference: &CausalLM,
tokenizer: &BpeTokenizer,
text: &str,
sequence_length: usize,
batch_size: usize,
) -> Result<KlDivergenceReport, QualityError> {
if tokenizer.vocab_size() != model.config.vocab_size {
return Err(QualityError::InvalidData(
"tokenizer and model vocabulary sizes differ".into(),
));
}
let tokens = tokenizer.encode(text);
evaluate_kl_held_out(model, reference, &tokens, sequence_length, batch_size)
}
pub fn mean_kl_logits(
reference_logits: &Tensor,
model_logits: &Tensor,
vocab_size: usize,
) -> Result<f64, QualityError> {
if reference_logits.shape() != model_logits.shape() {
return Err(QualityError::InvalidData(
"reference and model logit shapes differ".into(),
));
}
let shape = reference_logits.shape();
if shape.len() != 3 || shape[2] != vocab_size {
return Err(QualityError::InvalidData(
"logits must have shape [batch, seq, vocab]".into(),
));
}
let rows = shape[0].saturating_mul(shape[1]);
if rows == 0 {
return Err(QualityError::InvalidData("empty logits".into()));
}
let ref_data = reference_logits.data();
let model_data = model_logits.data();
let mut total = 0.0f64;
for row in 0..rows {
let start = row * vocab_size;
let end = start + vocab_size;
total += kl_row(&ref_data[start..end], &model_data[start..end]);
}
Ok(total / rows as f64)
}
#[derive(Clone, Debug, PartialEq)]
pub struct KlDivergenceReport {
pub mean_kl: f64,
pub tokens_compared: usize,
pub sequences: usize,
pub sequence_length: usize,
}
fn kl_row(reference_logits: &[f32], model_logits: &[f32]) -> f64 {
let ref_log = log_softmax(reference_logits);
let model_log = log_softmax(model_logits);
let mut kl = 0.0f64;
for index in 0..reference_logits.len() {
let log_p = ref_log[index];
let p = log_p.exp();
if p > 0.0 {
kl += p * (log_p - model_log[index]);
}
}
kl
}
fn log_softmax(logits: &[f32]) -> Vec<f64> {
let max = logits
.iter()
.copied()
.fold(f32::NEG_INFINITY, f32::max) as f64;
let mut sum = 0.0f64;
for value in logits {
sum += ((*value as f64) - max).exp();
}
let ln_sum = sum.ln();
logits
.iter()
.map(|value| (*value as f64) - max - ln_sum)
.collect()
}
pub fn format_report(report: &QualityReport) -> String {
let mut output = String::new();
if let Some(ppl) = &report.perplexity {
let _ = writeln!(
output,
"quality_perplexity loss={:.5} perplexity={:.3} tokens={} sequences={} sequence_length={} wall_s={:.3}",
ppl.loss,
ppl.perplexity,
ppl.tokens_processed,
ppl.sequences,
ppl.sequence_length,
ppl.wall_time_seconds
);
}
if let Some(next) = &report.next_token {
let _ = writeln!(
output,
"quality_next_token correct={} total={} accuracy={:.4}",
next.correct, next.total, next.accuracy
);
}
if let Some(tasks) = &report.tasks {
let _ = writeln!(
output,
"quality_tasks count={} exact_match_rate={:.4} prefix_match_rate={:.4} byte_accuracy={:.4}",
tasks.tasks.len(),
tasks.exact_match_rate,
tasks.prefix_match_rate,
tasks.byte_accuracy
);
for task in &tasks.tasks {
let _ = writeln!(
output,
"quality_task id={} exact={} prefix={} byte_accuracy={:.4} generated={:?}",
task.id,
task.exact_match,
task.prefix_match,
if task.bytes_total == 0 {
0.0
} else {
task.bytes_correct as f64 / task.bytes_total as f64
},
task.generated
);
}
}
output
}
fn count_argmax_matches(logits: &Tensor, targets: &[usize], vocab_size: usize) -> usize {
let data = logits.data();
let shape = logits.shape();
let batch = shape[0];
let sequence = shape[1];
let mut correct = 0usize;
for batch_index in 0..batch {
for position in 0..sequence {
let row_start = (batch_index * sequence + position) * vocab_size;
let row = &data[row_start..row_start + vocab_size];
let predicted = row
.iter()
.enumerate()
.max_by(|(_, left), (_, right)| left.total_cmp(right))
.map(|(index, _)| index)
.unwrap_or(0);
if predicted == targets[batch_index * sequence + position] {
correct += 1;
}
}
}
correct
}
fn byte_overlap(generated: &[u8], expected: &[u8]) -> (usize, usize) {
let total = expected.len();
let correct = generated
.iter()
.zip(expected.iter())
.take_while(|(left, right)| left == right)
.count();
(correct, total)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::CausalLmConfig;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
fn tiny_pair() -> (CausalLM, BpeTokenizer) {
let tokenizer = BpeTokenizer::train_from_bytes(
b"All:\nSpeak, speak.\nResolved. resolved.\nWe know't, we know't.\nFirst Citizen:\nBefore we proceed any further, hear me speak.\nWe are accounted poor citizens, the patricians good.\nLet us kill him, and we'll have corn at our own price.\nNo more talking on't; let it be done: away, away!\nSecond Citizen:\nOne word, good citizens.",
FIRST_MERGE_ID + 8,
)
.unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 16,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 7,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
(model, tokenizer)
}
#[test]
fn held_out_perplexity_is_finite_and_counts_tokens() {
let (model, tokenizer) = tiny_pair();
let text = "All:\nSpeak, speak.\nFirst Citizen:\nhear me speak. All:\nSpeak, speak.";
let (ppl, next) = evaluate_held_out_text(&model, &tokenizer, text, 4, 2, None).unwrap();
assert!(ppl.loss.is_finite());
assert!(ppl.perplexity.is_finite());
assert!(ppl.perplexity > 0.0);
assert_eq!(ppl.tokens_processed, ppl.sequences * ppl.sequence_length);
assert!(ppl.sequences >= 1);
assert_eq!(next.total, ppl.tokens_processed);
assert!(next.accuracy >= 0.0 && next.accuracy <= 1.0);
assert!((next.accuracy - next.correct as f64 / next.total as f64).abs() < 1e-12);
}
#[test]
fn held_out_rejects_short_corpus() {
let (model, _) = tiny_pair();
let err = evaluate_held_out(&model, &[1, 2, 3], 8, 1, None).unwrap_err();
assert!(matches!(err, QualityError::InvalidData(_)));
}
#[test]
fn builtin_tasks_run_greedily_and_format() {
let (model, tokenizer) = tiny_pair();
let tasks = vec![EvalTask {
id: "speak".into(),
prompt: "All:\n".into(),
expected: "Speak".into(),
}];
let suite = evaluate_tasks(&model, &tokenizer, &tasks, None).unwrap();
assert_eq!(suite.tasks.len(), 1);
assert_eq!(suite.tasks[0].id, "speak");
assert_eq!(suite.tasks[0].bytes_total, "Speak".len());
assert!(suite.byte_accuracy >= 0.0 && suite.byte_accuracy <= 1.0);
let report = QualityReport {
perplexity: None,
next_token: None,
tasks: Some(suite),
};
let formatted = format_report(&report);
assert!(formatted.contains("quality_tasks"));
assert!(formatted.contains("quality_task id=speak"));
}
#[test]
fn load_tasks_round_trips_json() {
let path = std::env::temp_dir().join("oxidelm_quality_tasks.json");
fs::write(
&path,
r#"[{"id":"t1","prompt":"Hello ","expected":"world"}]"#,
)
.unwrap();
let tasks = load_tasks(&path).unwrap();
assert_eq!(tasks.len(), 1);
assert_eq!(tasks[0].id, "t1");
assert_eq!(tasks[0].expected, "world");
let _ = fs::remove_file(&path);
}
#[test]
fn builtin_tiny_tasks_are_non_empty() {
let tasks = builtin_tiny_tasks();
assert_eq!(tasks.len(), 8);
assert!(tasks.iter().all(|task| !task.expected.is_empty()));
}
#[test]
fn forward_validation_passes_on_fresh_model() {
let (model, _) = tiny_pair();
let result = validate_forward(&model, FORWARD_VALIDATE_SEED, 1, 4).unwrap();
assert!(result.passed);
assert_eq!(result.logit_nan_count, 0);
assert_eq!(result.logit_inf_count, 0);
assert!(result.loss_finite);
assert!(result.loss.is_finite());
}
#[test]
fn kl_against_self_is_near_zero() {
let (model, tokenizer) = tiny_pair();
let text = "All:\nSpeak, speak.\nFirst Citizen:\nhear me speak. All:\nSpeak, speak.";
let report =
evaluate_kl_held_out_text(&model, &model, &tokenizer, text, 4, 1).unwrap();
assert!(report.tokens_compared > 0);
assert!(report.mean_kl.abs() < 1e-5);
}
#[test]
fn byte_overlap_counts_shared_prefix() {
assert_eq!(byte_overlap(b"abc", b"abd"), (2, 3));
assert_eq!(byte_overlap(b"", b"ab"), (0, 2));
assert_eq!(byte_overlap(b"abcd", b"ab"), (2, 2));
}
}