use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::Validity;
use crate::error::VeridictError;
use crate::metrics::{FailureBreakdown, tally_status};
pub const MANIFEST_SCHEMA_VERSION: u32 = 1;
const MAX_REPORTED_VIOLATIONS: usize = 500;
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub manifest_schema_version: u32,
pub experiment_id: Option<String>,
pub candidate_id: Option<String>,
pub baseline_id: Option<String>,
pub lineage_id: Option<String>,
pub dataset_sha256: Option<String>,
pub split_sha256: Option<String>,
pub teacher_manifest_sha256: Option<String>,
pub binary_sha256: Option<String>,
pub weight_sha256: Option<String>,
pub init_seed: Option<i64>,
pub split_seed: Option<i64>,
pub shuffle_seed: Option<i64>,
pub validity: Option<Validity>,
pub schema_version: Option<String>,
pub config_sha256: Option<String>,
pub schedule: Option<Vec<String>>,
}
impl Manifest {
fn has_anything_to_verify(&self) -> bool {
self.schedule.is_some()
|| self.experiment_id.is_some()
|| self.dataset_sha256.is_some()
|| self.binary_sha256.is_some()
|| self.weight_sha256.is_some()
|| self.config_sha256.is_some()
}
}
pub fn parse_manifest(path: &str, text: &str) -> Result<Manifest, VeridictError> {
let manifest: Manifest = toml::from_str(text).map_err(|source| VeridictError::InvalidToml {
path: path.to_string(),
source,
})?;
if manifest.manifest_schema_version != MANIFEST_SCHEMA_VERSION {
return Err(VeridictError::UnsupportedManifestSchemaVersion {
found: manifest.manifest_schema_version,
supported: MANIFEST_SCHEMA_VERSION,
});
}
Ok(manifest)
}
#[derive(Debug, Clone, Deserialize)]
pub struct VerifyRunRecord {
pub id: Option<String>,
pub global_index: Option<u64>,
pub role: Option<String>,
pub include_in_gate: Option<bool>,
pub experiment_id: Option<String>,
pub dataset_sha256: Option<String>,
pub binary_sha256: Option<String>,
pub weight_sha256: Option<String>,
pub config_sha256: Option<String>,
pub baseline_status: Option<String>,
pub candidate_status: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct Violation {
pub check: &'static str,
pub id: Option<String>,
pub lines: Vec<usize>,
pub field: Option<&'static str>,
pub detail: String,
}
#[derive(Debug, Serialize)]
pub struct SkippedCheck {
pub check: &'static str,
pub field: Option<&'static str>,
pub reason: String,
}
#[derive(Debug, Serialize)]
pub struct VerifyRunReport {
pub schema_version: u32,
pub validity: Validity,
pub reason: String,
pub violations: Vec<Violation>,
pub violation_count: u64,
pub violations_truncated: bool,
pub checks_skipped: Vec<SkippedCheck>,
pub warnings: Vec<String>,
pub record_count: u64,
pub paired_count: u64,
pub timeouts: u64,
pub crashes: u64,
pub invalid: u64,
pub failure_breakdown: FailureBreakdown,
pub manifest: Manifest,
}
fn group_by_id(
records: &[(usize, VerifyRunRecord)],
) -> BTreeMap<String, Vec<(usize, &VerifyRunRecord)>> {
let mut groups: BTreeMap<String, Vec<(usize, &VerifyRunRecord)>> = BTreeMap::new();
for (line, record) in records {
if let Some(id) = &record.id {
groups.entry(id.clone()).or_default().push((*line, record));
}
}
groups
}
fn check_pair_completeness(
groups: &BTreeMap<String, Vec<(usize, &VerifyRunRecord)>>,
) -> Vec<Violation> {
groups
.iter()
.filter(|(_, records)| records.len() != 2)
.map(|(id, records)| Violation {
check: "pair_completeness",
id: Some(id.clone()),
lines: records.iter().map(|(line, _)| *line).collect(),
field: Some("id"),
detail: format!(
"id '{id}' appears {} time(s); pairing requires exactly 2 records per id",
records.len()
),
})
.collect()
}
fn check_global_index_uniqueness(
records: &[(usize, VerifyRunRecord)],
) -> (Vec<Violation>, Option<SkippedCheck>) {
let mut by_index: BTreeMap<u64, Vec<usize>> = BTreeMap::new();
for (line, record) in records {
if let Some(index) = record.global_index {
by_index.entry(index).or_default().push(*line);
}
}
if by_index.is_empty() {
return (
Vec::new(),
Some(SkippedCheck {
check: "global_index_uniqueness",
field: Some("global_index"),
reason: "no record declared global_index".to_string(),
}),
);
}
let violations = by_index
.into_iter()
.filter(|(_, lines)| lines.len() > 1)
.map(|(index, lines)| Violation {
check: "global_index_uniqueness",
id: None,
lines,
field: Some("global_index"),
detail: format!("global_index {index} appears on more than one record"),
})
.collect();
(violations, None)
}
fn check_role_consistency(
groups: &BTreeMap<String, Vec<(usize, &VerifyRunRecord)>>,
) -> (Vec<Violation>, Option<SkippedCheck>) {
let mut violations = Vec::new();
let mut role_pairings: BTreeMap<(String, String), Vec<usize>> = BTreeMap::new();
let mut any_role_seen = false;
for (id, records) in groups {
if records.len() != 2 {
continue;
}
let (line_a, a) = records[0];
let (line_b, b) = records[1];
let (Some(role_a), Some(role_b)) = (&a.role, &b.role) else {
continue;
};
any_role_seen = true;
if role_a == role_b {
violations.push(Violation {
check: "role_consistency",
id: Some(id.clone()),
lines: vec![line_a, line_b],
field: Some("role"),
detail: format!(
"pair '{id}' has the same role ('{role_a}') on both records; a valid pair \
requires two different roles"
),
});
continue;
}
if a.include_in_gate == Some(false) || b.include_in_gate == Some(false) {
continue;
}
let mut combo = [role_a.clone(), role_b.clone()];
combo.sort();
let [first, second] = combo;
role_pairings
.entry((first, second))
.or_default()
.push(line_a);
}
if !any_role_seen {
return (
violations,
Some(SkippedCheck {
check: "role_consistency",
field: Some("role"),
reason: "no record declared role".to_string(),
}),
);
}
if role_pairings.len() > 1 {
let combos: Vec<String> = role_pairings
.keys()
.map(|(a, b)| format!("{{{a}, {b}}}"))
.collect();
violations.push(Violation {
check: "role_consistency",
id: None,
lines: role_pairings.values().flatten().copied().collect(),
field: Some("role"),
detail: format!(
"more than one distinct role pairing is used across the run: {}",
combos.join(", ")
),
});
}
(violations, None)
}
fn check_schedule_order(
manifest: &Manifest,
records: &[(usize, VerifyRunRecord)],
) -> (Vec<Violation>, Option<SkippedCheck>) {
let Some(schedule) = &manifest.schedule else {
return (
Vec::new(),
Some(SkippedCheck {
check: "schedule_order",
field: Some("schedule"),
reason: "manifest.schedule not declared".to_string(),
}),
);
};
let mut seen = std::collections::HashSet::new();
let mut actual = Vec::new();
for (_, record) in records {
if record.include_in_gate == Some(false) {
continue;
}
if let Some(id) = &record.id
&& seen.insert(id.clone())
{
actual.push(id.clone());
}
}
if &actual == schedule {
return (Vec::new(), None);
}
let mismatch_at = actual
.iter()
.zip(schedule.iter())
.position(|(a, e)| a != e)
.unwrap_or_else(|| actual.len().min(schedule.len()));
let describe = |ids: &[String], at: usize| {
ids.get(at)
.map(|id| format!("'{id}'"))
.unwrap_or_else(|| "<end of list>".to_string())
};
(
vec![Violation {
check: "schedule_order",
id: None,
lines: Vec::new(),
field: Some("schedule"),
detail: format!(
"actual gated pair order diverges from manifest.schedule at position \
{mismatch_at} (expected {}, got {}); full expected order: {schedule:?}, full \
actual order: {actual:?}",
describe(schedule, mismatch_at),
describe(&actual, mismatch_at),
),
}],
None,
)
}
fn check_experiment_contamination(
manifest: &Manifest,
records: &[(usize, VerifyRunRecord)],
) -> (Vec<Violation>, Option<SkippedCheck>) {
let Some(expected) = &manifest.experiment_id else {
return (
Vec::new(),
Some(SkippedCheck {
check: "experiment_contamination",
field: Some("experiment_id"),
reason: "manifest.experiment_id not declared".to_string(),
}),
);
};
if !records
.iter()
.any(|(_, record)| record.experiment_id.is_some())
{
return (
Vec::new(),
Some(SkippedCheck {
check: "experiment_contamination",
field: Some("experiment_id"),
reason: "no record declared experiment_id".to_string(),
}),
);
}
let violations = records
.iter()
.filter_map(|(line, record)| {
let actual = record.experiment_id.as_ref()?;
(actual != expected).then(|| Violation {
check: "experiment_contamination",
id: record.id.clone(),
lines: vec![*line],
field: Some("experiment_id"),
detail: format!(
"record's experiment_id '{actual}' does not match manifest's '{expected}'"
),
})
})
.collect();
(violations, None)
}
fn check_one_hash_field(
field: &'static str,
expected: &Option<String>,
records: &[(usize, VerifyRunRecord)],
get: impl Fn(&VerifyRunRecord) -> &Option<String>,
) -> (Vec<Violation>, Option<SkippedCheck>) {
let Some(expected) = expected else {
return (
Vec::new(),
Some(SkippedCheck {
check: "environment_consistency",
field: Some(field),
reason: format!("manifest.{field} not declared"),
}),
);
};
if !records.iter().any(|(_, record)| get(record).is_some()) {
return (
Vec::new(),
Some(SkippedCheck {
check: "environment_consistency",
field: Some(field),
reason: format!("no record declared {field}"),
}),
);
}
let violations = records
.iter()
.filter_map(|(line, record)| {
let actual = get(record).as_ref()?;
(actual != expected).then(|| Violation {
check: "environment_consistency",
id: record.id.clone(),
lines: vec![*line],
field: Some(field),
detail: format!(
"record's {field} '{actual}' does not match manifest's declared '{expected}'"
),
})
})
.collect();
(violations, None)
}
type HashFieldGetter = fn(&VerifyRunRecord) -> &Option<String>;
fn check_environment_consistency(
manifest: &Manifest,
records: &[(usize, VerifyRunRecord)],
) -> (Vec<Violation>, Vec<SkippedCheck>) {
let fields: [(&'static str, &Option<String>, HashFieldGetter); 4] = [
("dataset_sha256", &manifest.dataset_sha256, |r| {
&r.dataset_sha256
}),
("binary_sha256", &manifest.binary_sha256, |r| {
&r.binary_sha256
}),
("weight_sha256", &manifest.weight_sha256, |r| {
&r.weight_sha256
}),
("config_sha256", &manifest.config_sha256, |r| {
&r.config_sha256
}),
];
let mut violations = Vec::new();
let mut skipped = Vec::new();
for (field, expected, get) in fields {
let (v, s) = check_one_hash_field(field, expected, records, get);
violations.extend(v);
skipped.extend(s);
}
(violations, skipped)
}
fn tally_failures(records: &[(usize, VerifyRunRecord)]) -> Result<FailureBreakdown, VeridictError> {
let mut failures = FailureBreakdown::default();
for (line, record) in records {
if let Some(status) = record.baseline_status.as_deref() {
tally_status(status, *line, "baseline_status", &mut failures.baseline)?;
}
if let Some(status) = record.candidate_status.as_deref() {
tally_status(status, *line, "candidate_status", &mut failures.candidate)?;
}
}
Ok(failures)
}
pub fn run(
manifest: &Manifest,
records: Vec<(usize, VerifyRunRecord)>,
) -> Result<VerifyRunReport, VeridictError> {
if records.is_empty() {
return Err(VeridictError::EmptyInput);
}
if !manifest.has_anything_to_verify() {
return Err(VeridictError::ManifestDeclaresNothingToVerify);
}
let groups = group_by_id(&records);
let mut violations = check_pair_completeness(&groups);
let mut checks_skipped = Vec::new();
let (v, s) = check_global_index_uniqueness(&records);
violations.extend(v);
checks_skipped.extend(s);
let (v, s) = check_role_consistency(&groups);
violations.extend(v);
checks_skipped.extend(s);
let (v, s) = check_schedule_order(manifest, &records);
violations.extend(v);
checks_skipped.extend(s);
let (v, s) = check_experiment_contamination(manifest, &records);
violations.extend(v);
checks_skipped.extend(s);
let (v, s) = check_environment_consistency(manifest, &records);
violations.extend(v);
checks_skipped.extend(s);
violations.sort_by(|a, b| {
let key = |v: &Violation| {
(
v.lines.first().copied().unwrap_or(usize::MAX),
v.check,
v.field.unwrap_or(""),
v.id.clone().unwrap_or_default(),
)
};
key(a).cmp(&key(b))
});
let failure_breakdown = tally_failures(&records)?;
let paired_count = groups
.values()
.filter(|g| {
g.len() == 2
&& g[0].1.include_in_gate != Some(false)
&& g[1].1.include_in_gate != Some(false)
})
.count() as u64;
let warnings = checks_skipped
.iter()
.map(|s| format!("{}: {}", s.check, s.reason))
.collect();
let validity = if violations.is_empty() {
Validity::Valid
} else {
Validity::Invalid
};
let violation_count = violations.len() as u64;
let violations_truncated = violations.len() > MAX_REPORTED_VIOLATIONS;
let reason = if violations.is_empty() {
"No structural violations found.".to_string()
} else {
format!(
"{violation_count} structural violation(s) found; see `violations`.{}",
if violations_truncated {
format!(" (showing first {MAX_REPORTED_VIOLATIONS})")
} else {
String::new()
}
)
};
violations.truncate(MAX_REPORTED_VIOLATIONS);
Ok(VerifyRunReport {
schema_version: crate::report::REPORT_SCHEMA_VERSION,
validity,
reason,
violations,
violation_count,
violations_truncated,
checks_skipped,
warnings,
record_count: records.len() as u64,
paired_count,
timeouts: failure_breakdown.baseline.timeout + failure_breakdown.candidate.timeout,
crashes: failure_breakdown.baseline.crash + failure_breakdown.candidate.crash,
invalid: failure_breakdown.baseline.invalid + failure_breakdown.candidate.invalid,
failure_breakdown,
manifest: manifest.clone(),
})
}
impl VerifyRunReport {
pub fn to_json_pretty(&self) -> String {
serde_json::to_string_pretty(self).expect(
"VerifyRunReport contains only finite fields and strings; serialization cannot fail",
)
}
pub fn to_markdown(&self) -> String {
let b = &self.failure_breakdown.baseline;
let c = &self.failure_breakdown.candidate;
let mut violations = String::new();
for v in &self.violations {
violations.push_str(&format!(
"- [{}] {} (lines: {:?})\n",
v.check, v.detail, v.lines
));
}
let mut skipped = String::new();
for s in &self.checks_skipped {
skipped.push_str(&format!("- {}: {}\n", s.check, s.reason));
}
format!(
"# Veridict Verify-Run Report\n\n\
Validity: {validity}\n\n\
{reason}\n\n\
Violations ({violation_count}{truncated_note}):\n{violations}\n\
Checks skipped ({skipped_count}):\n{skipped}\n\
Records: {record_count}, paired: {paired_count}\n\
Status counts:\n\
- timeout: {timeouts} (baseline={b_timeout}, candidate={c_timeout})\n\
- crash: {crashes} (baseline={b_crash}, candidate={c_crash})\n\
- invalid: {invalid} (baseline={b_invalid}, candidate={c_invalid})\n",
validity = crate::report::serde_str(&self.validity),
reason = self.reason,
violation_count = self.violation_count,
truncated_note = if self.violations_truncated {
format!(", showing first {}", self.violations.len())
} else {
String::new()
},
skipped_count = self.checks_skipped.len(),
record_count = self.record_count,
paired_count = self.paired_count,
timeouts = self.timeouts,
crashes = self.crashes,
invalid = self.invalid,
b_timeout = b.timeout,
c_timeout = c.timeout,
b_crash = b.crash,
c_crash = c.crash,
b_invalid = b.invalid,
c_invalid = c.invalid,
)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn empty_manifest() -> Manifest {
Manifest {
manifest_schema_version: MANIFEST_SCHEMA_VERSION,
experiment_id: None,
candidate_id: None,
baseline_id: None,
lineage_id: None,
dataset_sha256: None,
split_sha256: None,
teacher_manifest_sha256: None,
binary_sha256: None,
weight_sha256: None,
init_seed: None,
split_seed: None,
shuffle_seed: None,
validity: None,
schema_version: None,
config_sha256: None,
schedule: None,
}
}
fn record(id: &str, role: &str) -> VerifyRunRecord {
VerifyRunRecord {
id: Some(id.to_string()),
global_index: None,
role: Some(role.to_string()),
include_in_gate: None,
experiment_id: None,
dataset_sha256: None,
binary_sha256: None,
weight_sha256: None,
config_sha256: None,
baseline_status: None,
candidate_status: None,
}
}
fn clean_pairs(n: usize) -> Vec<(usize, VerifyRunRecord)> {
(0..n)
.flat_map(|i| {
let id = format!("op{i}");
[(i * 2 + 1, record(&id, "a")), (i * 2 + 2, record(&id, "b"))]
})
.collect()
}
#[test]
fn parse_manifest_reads_a_minimal_valid_manifest() {
let manifest = parse_manifest("m.toml", "manifest_schema_version = 1\n").unwrap();
assert_eq!(manifest.manifest_schema_version, 1);
assert_eq!(manifest.experiment_id, None);
}
#[test]
fn parse_manifest_rejects_malformed_toml() {
assert!(matches!(
parse_manifest("m.toml", "not = [valid\n"),
Err(VeridictError::InvalidToml { .. })
));
}
#[test]
fn parse_manifest_rejects_unsupported_schema_version() {
assert!(matches!(
parse_manifest("m.toml", "manifest_schema_version = 999\n"),
Err(VeridictError::UnsupportedManifestSchemaVersion {
found: 999,
supported: 1
})
));
}
#[test]
fn parse_manifest_rejects_unknown_fields() {
assert!(matches!(
parse_manifest(
"m.toml",
"manifest_schema_version = 1\ntypo_field = \"oops\"\n"
),
Err(VeridictError::InvalidToml { .. })
));
}
#[test]
fn clean_run_with_matching_schedule_is_valid() {
let mut manifest = empty_manifest();
manifest.schedule = Some(vec!["op0".to_string(), "op1".to_string()]);
let records = clean_pairs(2);
let report = run(&manifest, records).unwrap();
assert_eq!(report.validity, Validity::Valid);
assert!(report.violations.is_empty());
assert_eq!(report.paired_count, 2);
assert_eq!(report.record_count, 4);
}
#[test]
fn empty_records_is_an_error() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
assert!(matches!(
run(&manifest, Vec::new()),
Err(VeridictError::EmptyInput)
));
}
#[test]
fn manifest_declaring_nothing_is_an_error() {
let manifest = empty_manifest();
let records = clean_pairs(1);
assert!(matches!(
run(&manifest, records),
Err(VeridictError::ManifestDeclaresNothingToVerify)
));
}
#[test]
fn incomplete_pair_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = clean_pairs(2);
records.push((100, record("lonely", "a")));
let report = run(&manifest, records).unwrap();
assert_eq!(report.validity, Validity::Invalid);
assert!(
report
.violations
.iter()
.any(|v| v.check == "pair_completeness" && v.id.as_deref() == Some("lonely"))
);
}
#[test]
fn tripled_id_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = clean_pairs(1);
records.push((100, record("op0", "a")));
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "pair_completeness" && v.lines.len() == 3)
);
}
#[test]
fn same_role_on_both_pair_members_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let records = vec![(1, record("op0", "a")), (2, record("op0", "a"))];
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "role_consistency" && v.id.as_deref() == Some("op0"))
);
}
#[test]
fn inconsistent_role_pairing_across_run_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = clean_pairs(3); records.push((100, record("odd", "x")));
records.push((101, record("odd", "y"))); let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "role_consistency"
&& v.detail.contains("more than one distinct role pairing"))
);
}
#[test]
fn burn_in_pair_with_a_different_role_vocabulary_is_not_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = clean_pairs(3); records.push((
100,
VerifyRunRecord {
include_in_gate: Some(false),
..record("burn_in", "x")
},
));
records.push((
101,
VerifyRunRecord {
include_in_gate: Some(false),
..record("burn_in", "y")
},
));
let report = run(&manifest, records).unwrap();
assert!(
!report
.violations
.iter()
.any(|v| v.check == "role_consistency")
);
}
#[test]
fn role_consistency_is_skipped_when_no_record_declares_role() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let records: Vec<_> = (0..2)
.flat_map(|i| {
let id = format!("op{i}");
[
(
i * 2 + 1,
VerifyRunRecord {
role: None,
..record(&id, "unused")
},
),
(
i * 2 + 2,
VerifyRunRecord {
role: None,
..record(&id, "unused")
},
),
]
})
.collect();
let report = run(&manifest, records).unwrap();
assert!(
report
.checks_skipped
.iter()
.any(|s| s.check == "role_consistency")
);
assert!(
!report
.violations
.iter()
.any(|v| v.check == "role_consistency")
);
}
#[test]
fn duplicate_global_index_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let records = vec![
(
1,
VerifyRunRecord {
global_index: Some(7),
..record("op0", "a")
},
),
(
2,
VerifyRunRecord {
global_index: Some(7),
..record("op0", "b")
},
),
];
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "global_index_uniqueness")
);
}
#[test]
fn violations_beyond_the_cap_are_counted_but_not_all_emitted() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let records: Vec<_> = (0..600u64)
.flat_map(|i| {
let id = format!("op{i}");
[
(
(i * 2 + 1) as usize,
VerifyRunRecord {
global_index: Some(i),
..record(&id, "a")
},
),
(
(i * 2 + 2) as usize,
VerifyRunRecord {
global_index: Some(i),
..record(&id, "b")
},
),
]
})
.collect();
let report = run(&manifest, records).unwrap();
assert_eq!(report.validity, Validity::Invalid);
assert_eq!(report.violation_count, 600);
assert!(report.violations_truncated);
assert_eq!(report.violations.len(), 500);
assert!(report.reason.contains("600"));
}
#[test]
fn schedule_mismatch_is_a_violation() {
let mut manifest = empty_manifest();
manifest.schedule = Some(vec!["op1".to_string(), "op0".to_string()]); let records = clean_pairs(2);
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "schedule_order")
);
}
#[test]
fn burn_in_records_are_excluded_from_schedule_order() {
let mut manifest = empty_manifest();
manifest.schedule = Some(vec!["op0".to_string()]);
let mut records = clean_pairs(1);
records.push((
100,
VerifyRunRecord {
include_in_gate: Some(false),
..record("burn_in", "a")
},
));
records.push((
101,
VerifyRunRecord {
include_in_gate: Some(false),
..record("burn_in", "b")
},
));
let report = run(&manifest, records).unwrap();
assert!(
!report
.violations
.iter()
.any(|v| v.check == "schedule_order")
);
assert_eq!(report.paired_count, 1);
assert_eq!(report.record_count, 4);
}
#[test]
fn mismatched_experiment_id_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = clean_pairs(1);
records.push((
100,
VerifyRunRecord {
experiment_id: Some("exp2".to_string()),
..record("contaminant", "a")
},
));
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "experiment_contamination")
);
}
#[test]
fn declared_experiment_id_without_record_values_is_reported_as_skipped() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let report = run(&manifest, clean_pairs(1)).unwrap();
assert!(report.checks_skipped.iter().any(|s| {
s.check == "experiment_contamination"
&& s.field == Some("experiment_id")
&& s.reason.contains("no record declared")
}));
assert!(
report
.warnings
.iter()
.any(|warning| warning.contains("experiment_contamination"))
);
}
#[test]
fn hash_drift_is_a_violation() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
manifest.binary_sha256 = Some("abc123".to_string());
let mut records = clean_pairs(1);
records.push((
100,
VerifyRunRecord {
binary_sha256: Some("different".to_string()),
..record("op1", "a")
},
));
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "environment_consistency" && v.field == Some("binary_sha256"))
);
}
#[test]
fn config_drift_is_a_violation() {
let mut manifest = empty_manifest();
manifest.config_sha256 = Some("cfg1".to_string());
let mut records = clean_pairs(1);
records.push((
100,
VerifyRunRecord {
config_sha256: Some("cfg2".to_string()),
..record("op1", "a")
},
));
let report = run(&manifest, records).unwrap();
assert!(
report
.violations
.iter()
.any(|v| v.check == "environment_consistency" && v.field == Some("config_sha256"))
);
}
#[test]
fn environment_consistency_reports_a_skip_per_undeclared_field() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let report = run(&manifest, clean_pairs(1)).unwrap();
let skipped_fields: Vec<_> = report
.checks_skipped
.iter()
.filter(|s| s.check == "environment_consistency")
.filter_map(|s| s.field)
.collect();
assert_eq!(skipped_fields.len(), 4); }
#[test]
fn declared_hash_without_record_values_is_reported_as_skipped() {
let mut manifest = empty_manifest();
manifest.binary_sha256 = Some("expected".to_string());
let report = run(&manifest, clean_pairs(1)).unwrap();
assert!(report.checks_skipped.iter().any(|s| {
s.check == "environment_consistency"
&& s.field == Some("binary_sha256")
&& s.reason.contains("no record declared")
}));
assert!(
report
.warnings
.iter()
.any(|warning| warning.contains("binary_sha256"))
);
}
#[test]
fn violations_are_sorted_deterministically() {
let mut manifest = empty_manifest();
manifest.experiment_id = Some("exp1".to_string());
let mut records = vec![(1, record("z_lonely", "a")), (2, record("a_lonely", "a"))];
records.truncate(2);
let report1 = run(&manifest, records.clone()).unwrap();
let report2 = run(&manifest, records).unwrap();
let lines1: Vec<_> = report1.violations.iter().map(|v| v.lines.clone()).collect();
let lines2: Vec<_> = report2.violations.iter().map(|v| v.lines.clone()).collect();
assert_eq!(lines1, lines2);
assert_eq!(report1.violations[0].lines, vec![1]);
}
}