use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use serde::{Deserialize, Serialize};
use crate::capability::{CoverageLevel, GradientFamilyExpectation};
use crate::phase::{ExecutionPhase, ExecutionStep};
use super::{EdgeEvent, GradientState, MemoryAction, RunOutcome, TraceDocument};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum HealthSeverity {
Error,
Warning,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HealthIssue {
pub severity: HealthSeverity,
pub code: String,
pub message: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct EvidenceCoverage {
pub spans: usize,
pub closed_spans: usize,
pub root_spans: usize,
pub measured_spans: usize,
pub operations: usize,
pub tensors: usize,
pub memory_events: usize,
pub device_memory_samples: usize,
pub device_intervals: usize,
pub gradients: usize,
pub call_edges: usize,
pub data_edges: usize,
pub forward_spans: usize,
pub backward_spans: usize,
pub optimizer_spans: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TraceHealth {
pub structurally_valid: bool,
pub capture_complete: bool,
pub issues: Vec<HealthIssue>,
pub coverage: EvidenceCoverage,
}
impl TraceHealth {
pub fn gaps(&self) -> impl Iterator<Item = &HealthIssue> {
self.issues
.iter()
.filter(|issue| issue.severity == HealthSeverity::Warning)
}
}
pub fn analyze_health(doc: &TraceDocument) -> TraceHealth {
let failed = doc.terminal.outcome == RunOutcome::Failed;
let ids: HashSet<&str> = doc.spans.iter().map(|span| span.id.as_str()).collect();
let by_id: HashMap<&str, _> = doc
.spans
.iter()
.map(|span| (span.id.as_str(), span))
.collect();
let mut issues = Vec::new();
if let Err(provenance_error) = doc.run.validate() {
error(
&mut issues,
"run_provenance_invalid",
provenance_error.to_string(),
);
}
match doc.terminal.outcome {
RunOutcome::Complete if doc.terminal.reason.is_some() => error(
&mut issues,
"complete_with_failure_reason",
"complete terminal outcome cannot contain a failure reason",
),
RunOutcome::Failed
if doc
.terminal
.reason
.as_deref()
.is_none_or(|reason| reason.trim().is_empty()) =>
{
error(
&mut issues,
"failed_without_reason",
"failed terminal outcome requires a non-empty reason",
)
}
_ => {}
}
let latest_host_timestamp_ns = doc
.spans
.iter()
.map(|span| {
span.start_ns
.saturating_add(if span.closed { span.duration_ns } else { 0 })
})
.chain(
doc.ops
.iter()
.map(|op| op.timestamp_ns.saturating_add(op.duration_ns)),
)
.chain(doc.memory.iter().map(|event| event.timestamp_ns))
.chain(doc.device_memory.iter().map(|event| event.timestamp_ns))
.max()
.unwrap_or(0);
if doc.terminal.timestamp_ns < latest_host_timestamp_ns {
error(
&mut issues,
"terminal_precedes_evidence",
format!(
"terminal timestamp {} precedes host evidence ending at {latest_host_timestamp_ns}",
doc.terminal.timestamp_ns
),
);
}
if failed {
warning(
&mut issues,
"capture_failed",
doc.terminal
.reason
.as_deref()
.unwrap_or("capture ended with a failed outcome"),
);
}
if ids.len() != doc.spans.len() {
error(&mut issues, "duplicate_span_id", "span IDs must be unique");
}
let root_spans = doc
.spans
.iter()
.filter(|span| span.parent_id.is_none())
.count();
if root_spans != 1 {
error(
&mut issues,
"root_count",
format!("expected exactly one root span, found {root_spans}"),
);
}
let measured_spans = doc.spans.iter().filter(|span| span.measured).count();
if measured_spans != 1 && !failed {
error(
&mut issues,
"measurement_count",
format!("expected exactly one measured region, found {measured_spans}"),
);
}
for span in &doc.spans {
if !span.closed {
if failed {
warning(
&mut issues,
"open_span",
format!("span `{}` was interrupted", span.id),
);
} else {
error(
&mut issues,
"open_span",
format!("span `{}` was not closed", span.id),
);
}
}
if let Some(parent) = span.parent_id.as_deref() {
match by_id.get(parent) {
None => error(
&mut issues,
"unknown_parent",
format!("span `{}` refers to missing parent `{parent}`", span.id),
),
Some(parent_span) if span.closed && parent_span.closed => {
let child_end = span.start_ns.saturating_add(span.duration_ns);
let parent_end = parent_span.start_ns.saturating_add(parent_span.duration_ns);
if span.start_ns < parent_span.start_ns || child_end > parent_end {
error(
&mut issues,
"child_outside_parent",
format!("span `{}` lies outside parent `{parent}`", span.id),
);
}
}
Some(_) => {}
}
}
let mut current = Some(span.id.as_str());
let mut seen = HashSet::new();
while let Some(id) = current {
if !seen.insert(id) {
error(
&mut issues,
"span_cycle",
format!("span `{}` participates in a parent cycle", span.id),
);
break;
}
current = by_id.get(id).and_then(|item| item.parent_id.as_deref());
}
}
for (kind, span_id) in doc
.ops
.iter()
.map(|x| ("operation", x.span_id.as_str()))
.chain(doc.tensors.iter().map(|x| ("tensor", x.span_id.as_str())))
.chain(
doc.tensor_stats
.iter()
.map(|x| ("tensor stats", x.span_id.as_str())),
)
.chain(doc.memory.iter().map(|x| ("memory", x.span_id.as_str())))
.chain(
doc.device_intervals
.iter()
.map(|x| ("device interval", x.span_id.as_str())),
)
{
if !ids.contains(span_id) {
error(
&mut issues,
"unknown_span",
format!("{kind} evidence refers to missing span `{span_id}`"),
);
}
}
for interval in &doc.device_intervals {
if interval.duration_ns == 0 {
error(
&mut issues,
"empty_device_interval",
format!(
"device interval for `{}` has zero duration",
interval.span_id
),
);
}
}
for op in &doc.ops {
if let Some(span) = by_id.get(op.span_id.as_str()).filter(|span| span.closed) {
let span_end = span.start_ns.saturating_add(span.duration_ns);
let op_end = op.timestamp_ns.saturating_add(op.duration_ns);
if op.timestamp_ns < span.start_ns || op_end > span_end {
error(
&mut issues,
"operation_outside_span",
format!(
"operation `{}` lies outside span `{}`",
op.op_name, op.span_id
),
);
}
}
}
for event in &doc.memory {
if let Some(span) = by_id.get(event.span_id.as_str()).filter(|span| span.closed) {
let span_end = span.start_ns.saturating_add(span.duration_ns);
if event.timestamp_ns < span.start_ns || event.timestamp_ns > span_end {
error(
&mut issues,
"memory_outside_span",
format!(
"memory event for storage `{}` lies outside span `{}`",
event.storage_id, event.span_id
),
);
}
}
}
for sample in &doc.device_memory {
if sample.used_bytes.is_none()
&& sample.free_bytes.is_none()
&& sample.reserved_bytes.is_none()
&& sample.capacity_bytes.is_none()
{
error(
&mut issues,
"empty_device_memory_sample",
format!(
"device-memory sample for `{}` contains no measurements",
sample.device
),
);
}
}
let known_tensors = doc
.tensors
.iter()
.map(|tensor| tensor.tensor_id.as_str())
.chain(
doc.ops
.iter()
.flat_map(|op| op.inputs.iter().map(String::as_str)),
)
.chain(doc.ops.iter().filter_map(|op| op.output.as_deref()))
.collect::<HashSet<_>>();
for edge in &doc.edges {
match edge {
EdgeEvent::Call {
from_span,
to_span,
..
} if !ids.contains(from_span.as_str()) || !ids.contains(to_span.as_str()) => error(
&mut issues,
"unknown_call_edge_span",
format!("call edge `{from_span}` -> `{to_span}` refers to a missing span"),
),
EdgeEvent::Call {
from_span,
to_span,
host_duration_ns,
} => {
let target = by_id[to_span.as_str()];
if target.parent_id.as_deref() != Some(from_span.as_str()) {
error(
&mut issues,
"call_edge_hierarchy_mismatch",
format!(
"call edge `{from_span}` -> `{to_span}` does not match the span hierarchy"
),
);
}
if target.closed && *host_duration_ns != target.duration_ns {
error(
&mut issues,
"call_edge_duration_mismatch",
format!(
"call edge `{from_span}` -> `{to_span}` reports {host_duration_ns} ns but the span reports {} ns",
target.duration_ns
),
);
}
}
EdgeEvent::Data {
from_tensor,
to_tensor,
} if from_tensor.is_empty() || to_tensor.is_empty() => error(
&mut issues,
"empty_data_edge_endpoint",
"data-edge tensor IDs cannot be empty",
),
EdgeEvent::Data {
from_tensor,
to_tensor,
} if !known_tensors.contains(from_tensor.as_str())
|| !known_tensors.contains(to_tensor.as_str()) =>
{
error(
&mut issues,
"unknown_data_edge_tensor",
format!(
"data edge `{from_tensor}` -> `{to_tensor}` refers to unknown tensor evidence"
),
)
}
_ => {}
}
}
let mut live_memory: HashMap<(&str, &str), (u64, HashSet<&str>)> = HashMap::new();
let mut memory = doc.memory.iter().collect::<Vec<_>>();
memory.sort_by_key(|event| event.timestamp_ns);
for event in memory {
let key = (event.device.as_str(), event.storage_id.as_str());
match event.action {
MemoryAction::Alloc => match live_memory.get_mut(&key) {
Some((bytes, _)) if *bytes != event.bytes => error(
&mut issues,
"allocation_size_mismatch",
format!(
"storage `{}` on `{}` has conflicting allocation sizes",
event.storage_id, event.device
),
),
Some((_, tensor_ids)) => {
if !tensor_ids.insert(event.tensor_id.as_str()) {
error(
&mut issues,
"duplicate_allocation",
format!(
"tensor `{}` repeated an allocation for storage `{}` on `{}`",
event.tensor_id, event.storage_id, event.device
),
);
}
}
None => {
live_memory.insert(
key,
(event.bytes, HashSet::from([event.tensor_id.as_str()])),
);
}
},
MemoryAction::Free => match live_memory.remove(&key) {
None => error(
&mut issues,
"unpaired_free",
format!(
"storage `{}` on `{}` was freed while not live",
event.storage_id, event.device
),
),
Some((bytes, _)) if bytes != event.bytes => error(
&mut issues,
"allocation_size_mismatch",
format!(
"storage `{}` allocated {bytes} bytes but freed {}",
event.storage_id, event.bytes
),
),
Some(_) => {}
},
}
}
if !live_memory.is_empty() {
warning(
&mut issues,
"retained_allocations",
format!(
"{} storages remained live at capture end",
live_memory.len()
),
);
}
validate_gradient_manifest(doc, failed, &mut issues);
let coverage = EvidenceCoverage {
spans: doc.spans.len(),
closed_spans: doc.spans.iter().filter(|span| span.closed).count(),
root_spans,
measured_spans,
operations: doc.ops.len(),
tensors: doc.tensors.len(),
memory_events: doc.memory.len(),
device_memory_samples: doc.device_memory.len(),
device_intervals: doc.device_intervals.len(),
gradients: doc.gradients.len(),
call_edges: doc
.edges
.iter()
.filter(|edge| matches!(edge, EdgeEvent::Call { .. }))
.count(),
data_edges: doc
.edges
.iter()
.filter(|edge| matches!(edge, EdgeEvent::Data { .. }))
.count(),
forward_spans: step_count(doc, ExecutionStep::Forward),
backward_spans: step_count(doc, ExecutionStep::Backward),
optimizer_spans: step_count(doc, ExecutionStep::Optimizer),
};
for (empty, code, message) in [
(
coverage.operations == 0,
"operations_absent",
"no operation evidence was captured",
),
(
coverage.tensors == 0,
"tensors_absent",
"no tensor checkpoints were captured",
),
(
coverage.memory_events == 0,
"logical_memory_absent",
"no logical storage events were captured",
),
(
coverage.device_memory_samples == 0,
"physical_memory_absent",
"no physical device-memory samples were captured",
),
(
coverage.device_intervals == 0,
"device_timing_absent",
"no device timing intervals were captured",
),
(
coverage.gradients == 0 && doc.run.phase == ExecutionPhase::Train,
"gradients_absent",
"no gradient facts were captured for this training run",
),
(
coverage.forward_spans == 0 && doc.run.phase == ExecutionPhase::Train,
"forward_absent",
"no forward span was tagged",
),
(
coverage.backward_spans == 0 && doc.run.phase == ExecutionPhase::Train,
"backward_absent",
"no backward span was tagged",
),
(
coverage.optimizer_spans == 0 && doc.run.phase == ExecutionPhase::Train,
"optimizer_absent",
"no optimizer span was tagged",
),
] {
if empty {
warning(&mut issues, code, message);
}
}
let mut required_labels = HashSet::new();
for required in &doc.run.capture_contract.required_semantic_labels {
if required.trim().is_empty() {
required_semantic_label_issue(
&mut issues,
failed,
"empty_required_semantic_label",
"required semantic labels must not be empty",
);
}
if !required_labels.insert(required.as_str()) {
required_semantic_label_issue(
&mut issues,
failed,
"duplicate_required_semantic_label",
format!("required semantic label `{required}` is declared more than once"),
);
}
let count = doc
.spans
.iter()
.filter(|span| span.name == *required)
.count();
if count != 1 {
required_semantic_label_issue(
&mut issues,
failed,
"required_semantic_label_cardinality",
format!(
"required semantic label `{required}` must occur exactly once; observed {count}"
),
);
}
}
let contract = &doc.run.capture_contract;
let explicitly_classified = !contract.gpu_expected_semantic_labels.is_empty()
|| !contract.cpu_only_semantic_labels.is_empty();
if explicitly_classified {
let mut classified_labels = HashSet::new();
for (class, labels) in [
("GPU-expected", &contract.gpu_expected_semantic_labels),
("CPU-only", &contract.cpu_only_semantic_labels),
] {
let mut class_labels = HashSet::new();
for label in labels {
if label.trim().is_empty() {
required_semantic_label_issue(
&mut issues,
failed,
"empty_semantic_label_classification",
format!("{class} semantic labels must not be empty"),
);
}
if !class_labels.insert(label.as_str()) {
required_semantic_label_issue(
&mut issues,
failed,
"duplicate_semantic_label_classification",
format!("{class} semantic label `{label}` is declared more than once"),
);
}
if !required_labels.contains(label.as_str()) {
required_semantic_label_issue(
&mut issues,
failed,
"unrequired_semantic_label_classification",
format!(
"{class} semantic label `{label}` is not a required application label"
),
);
}
if !classified_labels.insert(label.as_str()) {
required_semantic_label_issue(
&mut issues,
failed,
"overlapping_semantic_label_classification",
format!(
"semantic label `{label}` is classified as both GPU-expected and CPU-only"
),
);
}
}
}
if classified_labels != required_labels {
required_semantic_label_issue(
&mut issues,
failed,
"incomplete_semantic_label_partition",
"GPU-expected and CPU-only semantic labels must partition all required application labels",
);
}
}
TraceHealth {
structurally_valid: !issues
.iter()
.any(|issue| issue.severity == HealthSeverity::Error),
capture_complete: !failed,
issues,
coverage,
}
}
fn validate_gradient_manifest(doc: &TraceDocument, failed: bool, issues: &mut Vec<HealthIssue>) {
let mut event_ids = HashSet::new();
let mut observed = BTreeMap::<(&str, &str), usize>::new();
let mut events_by_key = BTreeMap::new();
for gradient in &doc.gradients {
if gradient.event_id.trim().is_empty() {
error(
issues,
"empty_gradient_event_id",
"gradient event IDs must not be empty",
);
}
if gradient.root.trim().is_empty() || gradient.key.trim().is_empty() {
error(
issues,
"empty_gradient_parameter_key",
"gradient roots and parameter keys must not be empty",
);
}
if !event_ids.insert(gradient.event_id.as_str()) {
error(
issues,
"duplicate_gradient_event_id",
format!(
"gradient event ID {:?} occurs more than once",
gradient.event_id
),
);
}
*observed
.entry((gradient.root.as_str(), gradient.key.as_str()))
.or_default() += 1;
events_by_key
.entry((gradient.root.as_str(), gradient.key.as_str()))
.or_insert(gradient);
if !gradient.state.norm_is_valid(gradient.norm) {
error(
issues,
"gradient_state_norm_inconsistent",
format!(
"gradient ({:?}, {:?}) state `{}` is inconsistent with norm {:?}",
gradient.root, gradient.key, gradient.state, gradient.norm
),
);
}
}
let declared = doc.run.capture_contract.gradients;
let contract = doc.run.capture_contract.gradient_contract.as_ref();
match (declared, contract) {
(CoverageLevel::Complete, None) => {
error(
issues,
"gradient_contract_missing",
"complete gradient coverage requires an exact gradient contract",
);
return;
}
(CoverageLevel::Complete, Some(_)) | (_, None) => {}
(_, Some(_)) => {
error(
issues,
"gradient_contract_without_complete_coverage",
"an exact gradient contract requires complete declared gradient coverage",
);
return;
}
}
let Some(contract) = contract else {
return;
};
if let Err(contract_error) = contract.validate() {
error(
issues,
"gradient_contract_invalid",
contract_error.to_string(),
);
return;
}
let expected = contract
.expected
.iter()
.map(|gradient| (gradient.root.as_str(), gradient.key.as_str()))
.collect::<BTreeSet<_>>();
for (&(root, key), &count) in &observed {
if count != 1 {
error(
issues,
"gradient_manifest_duplicate_key",
format!("gradient ({root:?}, {key:?}) occurs {count} times; expected exactly once"),
);
}
if !expected.contains(&(root, key)) {
error(
issues,
"gradient_manifest_undeclared_key",
format!("gradient ({root:?}, {key:?}) is absent from the manifest"),
);
}
}
for parameter in &contract.expected {
if !observed.contains_key(&(parameter.root.as_str(), parameter.key.as_str())) {
let root = ¶meter.root;
let key = ¶meter.key;
let message = format!("manifest gradient ({root:?}, {key:?}) was not captured");
if failed {
warning(issues, "gradient_manifest_missing_key", message);
} else {
error(issues, "gradient_manifest_missing_key", message);
}
}
}
for family in &contract.families {
let expected_members = contract
.expected
.iter()
.filter(|parameter| parameter.family == family.family)
.count();
let members = contract
.expected
.iter()
.filter(|parameter| parameter.family == family.family)
.filter_map(|parameter| {
events_by_key
.get(&(parameter.root.as_str(), parameter.key.as_str()))
.copied()
})
.collect::<Vec<_>>();
let family_capture_complete = members.len() == expected_members;
let present = members
.iter()
.filter(|gradient| gradient.state == GradientState::Present)
.count();
let attached = members
.iter()
.filter(|gradient| gradient.state != GradientState::Missing)
.count();
let non_finite = members
.iter()
.filter(|gradient| gradient.state == GradientState::NonFinite)
.count();
if non_finite > 0 {
error(
issues,
"gradient_family_non_finite",
format!(
"gradient family {:?} contains {non_finite} non-finite gradients",
family.family
),
);
}
match family.expectation {
GradientFamilyExpectation::Active
if (!failed || family_capture_complete) && present < family.min_present => error(
issues,
"gradient_active_family_below_minimum",
format!(
"active gradient family {:?} has {present} present gradients; requires at least {}",
family.family, family.min_present
),
),
GradientFamilyExpectation::Inactive if attached > 0 => error(
issues,
"gradient_inactive_family_leakage",
format!(
"inactive gradient family {:?} has {attached} attached gradients",
family.family
),
),
GradientFamilyExpectation::DataConditional
if (!failed || family_capture_complete)
&& present > 0
&& present < family.min_present =>
{
error(
issues,
"gradient_conditional_family_below_minimum",
format!(
"data-conditional gradient family {:?} was attached but has {present} present gradients; requires at least {}",
family.family, family.min_present
),
)
}
_ => {}
}
}
}
fn step_count(doc: &TraceDocument, step: ExecutionStep) -> usize {
doc.spans
.iter()
.filter(|span| span.step == Some(step))
.count()
}
fn error(issues: &mut Vec<HealthIssue>, code: &str, message: impl Into<String>) {
issues.push(HealthIssue {
severity: HealthSeverity::Error,
code: code.into(),
message: message.into(),
});
}
fn warning(issues: &mut Vec<HealthIssue>, code: &str, message: impl Into<String>) {
issues.push(HealthIssue {
severity: HealthSeverity::Warning,
code: code.into(),
message: message.into(),
});
}
fn required_semantic_label_issue(
issues: &mut Vec<HealthIssue>,
failed: bool,
code: &str,
message: impl Into<String>,
) {
if failed {
warning(issues, code, message);
} else {
error(issues, code, message);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::capability::CaptureContract;
use crate::trace::{
MemoryCategory, MemoryEvent, RunOutcome, SpanKind, SpanRecord, TerminalEvent, TimingMode,
TraceRunMeta, SCHEMA,
};
fn failed_document() -> TraceDocument {
TraceDocument {
schema: SCHEMA.into(),
run: TraceRunMeta {
run_id: "failed".into(),
correlation_id: "failed/run".into(),
entrypoint: "demo".into(),
phase: ExecutionPhase::Infer,
timestamp: "2026-08-19T00:00:00Z".into(),
capture_step: 1,
warmup_steps: 0,
device: "cpu".into(),
measured_region_device_synchronized: false,
timing_mode: TimingMode::Host,
capture_contract: CaptureContract::default(),
comparison_identity: None,
tags: Default::default(),
candle_version: None,
},
spans: vec![SpanRecord {
id: "root".into(),
parent_id: None,
name: "demo".into(),
kind: SpanKind::Function,
measured: false,
start_ns: 0,
closed: false,
duration_ns: 0,
step: None,
}],
ops: vec![],
tensors: vec![],
tensor_stats: vec![],
memory: vec![],
device_memory: vec![],
device_intervals: vec![],
gradients: vec![],
edges: vec![],
terminal: TerminalEvent {
outcome: RunOutcome::Failed,
timestamp_ns: 10,
reason: Some("boom".into()),
},
}
}
#[test]
fn failed_capture_is_diagnosable_without_becoming_complete() {
let health = analyze_health(&failed_document());
assert!(health.structurally_valid);
assert!(!health.capture_complete);
assert!(health
.issues
.iter()
.any(|issue| issue.code == "capture_failed"));
assert!(health
.issues
.iter()
.any(|issue| issue.code == "open_span" && issue.severity == HealthSeverity::Warning));
}
#[test]
fn complete_capture_requires_each_declared_semantic_label_exactly_once() {
let mut document = failed_document();
document.terminal = TerminalEvent {
outcome: RunOutcome::Complete,
timestamp_ns: 10,
reason: None,
};
document.spans[0].measured = true;
document.spans[0].closed = true;
document.spans[0].duration_ns = 5;
document.run.capture_contract.required_semantic_labels = vec!["missing".into()];
let health = analyze_health(&document);
assert!(!health.structurally_valid);
assert!(health.issues.iter().any(|issue| {
issue.code == "required_semantic_label_cardinality"
&& issue.severity == HealthSeverity::Error
}));
document.run.capture_contract.required_semantic_labels = vec!["demo".into(), "demo".into()];
let duplicate_health = analyze_health(&document);
assert!(!duplicate_health.structurally_valid);
assert!(duplicate_health.issues.iter().any(|issue| {
issue.code == "duplicate_required_semantic_label"
&& issue.severity == HealthSeverity::Error
}));
document.run.capture_contract.required_semantic_labels = vec!["demo".into()];
let mut repeated = document.spans[0].clone();
repeated.id = "child".into();
repeated.parent_id = Some("root".into());
repeated.measured = false;
document.spans.push(repeated);
let repeated_health = analyze_health(&document);
assert!(!repeated_health.structurally_valid);
assert!(repeated_health.issues.iter().any(|issue| {
issue.code == "required_semantic_label_cardinality"
&& issue.severity == HealthSeverity::Error
&& issue.message.contains("observed 2")
}));
}
#[test]
fn failed_capture_keeps_missing_required_labels_diagnostic() {
let mut document = failed_document();
document.run.capture_contract.required_semantic_labels = vec!["missing".into()];
let health = analyze_health(&document);
assert!(health.structurally_valid);
assert!(health.issues.iter().any(|issue| {
issue.code == "required_semantic_label_cardinality"
&& issue.severity == HealthSeverity::Warning
}));
}
#[test]
fn complete_capture_rejects_an_incomplete_semantic_label_partition() {
let mut document = failed_document();
document.terminal = TerminalEvent {
outcome: RunOutcome::Complete,
timestamp_ns: 10,
reason: None,
};
document.spans[0].measured = true;
document.spans[0].closed = true;
document.spans[0].duration_ns = 5;
document.run.capture_contract.required_semantic_labels =
vec!["demo".into(), "prepare".into()];
document.run.capture_contract.gpu_expected_semantic_labels = vec!["demo".into()];
let health = analyze_health(&document);
assert!(!health.structurally_valid);
assert!(health.issues.iter().any(|issue| {
issue.code == "incomplete_semantic_label_partition"
&& issue.severity == HealthSeverity::Error
}));
}
#[test]
fn out_of_domain_run_provenance_is_a_structural_error() {
let mut document = failed_document();
document.run.capture_step = 0;
let health = analyze_health(&document);
assert!(!health.structurally_valid);
assert!(health.issues.iter().any(|issue| {
issue.code == "run_provenance_invalid" && issue.severity == HealthSeverity::Error
}));
let mut document = failed_document();
document.run.warmup_steps = document.run.capture_step;
assert!(!analyze_health(&document).structurally_valid);
let mut document = failed_document();
document.run.entrypoint = " ".into();
assert!(!analyze_health(&document).structurally_valid);
}
#[test]
fn memory_events_outside_their_closed_span_are_rejected() {
let mut document = failed_document();
document.spans[0].closed = true;
document.spans[0].start_ns = 10;
document.spans[0].duration_ns = 20;
document.memory = vec![MemoryEvent {
timestamp_ns: 50,
storage_id: "late".into(),
tensor_id: "late".into(),
span_id: "root".into(),
op_name: None,
device: "cpu".into(),
bytes: 8,
action: MemoryAction::Alloc,
shape: vec![8],
dtype: "u8".into(),
category: MemoryCategory::Activation,
}];
let health = analyze_health(&document);
assert!(!health.structurally_valid);
assert!(health.issues.iter().any(|issue| {
issue.code == "memory_outside_span" && issue.severity == HealthSeverity::Error
}));
}
#[test]
fn distinct_tensor_aliases_share_one_live_storage() {
let mut document = failed_document();
let memory = |timestamp_ns, tensor_id: &str, action| MemoryEvent {
timestamp_ns,
storage_id: "shared".into(),
tensor_id: tensor_id.into(),
span_id: "root".into(),
op_name: None,
device: "cpu".into(),
bytes: 64,
action,
shape: vec![16],
dtype: "f32".into(),
category: MemoryCategory::Activation,
};
document.memory = vec![
memory(1, "base", MemoryAction::Alloc),
memory(2, "view", MemoryAction::Alloc),
memory(3, "view", MemoryAction::Free),
];
let health = analyze_health(&document);
assert!(health.structurally_valid);
assert!(!health
.issues
.iter()
.any(|issue| issue.code == "duplicate_allocation"));
}
}