1use std::path::Path;
4
5use anyhow::{ensure, Context, Result};
6use serde::{Deserialize, Serialize};
7
8use crate::activation::{ActivationProfile, ByteNanoseconds};
9use crate::capability::{
10 CapabilityKind, CapabilityLevel, CapabilityState, CoverageLevel, EvidenceCapabilities,
11};
12use crate::graph::{build_from_trace, ExecutionGraph};
13use crate::nsight::{GpuEvidenceStatus, NsightEvidence, ProvenanceBindingState};
14use crate::timing::{analyze_timing, TimingProfile};
15use crate::trace::memory::{analyze_memory, MemoryProfile};
16use crate::trace::{
17 analyze_health, parse_trace, HealthSeverity, TensorStatsEvent, TraceDocument, TraceHealth,
18 TraceRunMeta,
19};
20
21pub const SCHEMA: &str = "candle-graph/evidence/5";
22
23#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
24pub struct EvidencePacket {
25 #[serde(deserialize_with = "deserialize_schema")]
26 pub schema: String,
27 pub provenance: TraceRunMeta,
28 pub health: TraceHealth,
29 pub capabilities: EvidenceCapabilities,
30 pub findings: Vec<EvidenceFinding>,
31 pub facts: Vec<EvidenceFact>,
32 pub gaps: Vec<String>,
33 #[serde(default, skip_serializing_if = "Option::is_none")]
35 pub graph: Option<ExecutionGraph>,
36 #[serde(default, skip_serializing_if = "Vec::is_empty")]
38 pub tensor_stats: Vec<TensorStatsEvent>,
39 pub timing: TimingProfile,
40 pub memory: MemoryProfile,
41 pub activations: ActivationProfile,
43 pub gpu: NsightEvidence,
44}
45
46fn deserialize_schema<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
47where
48 D: serde::Deserializer<'de>,
49{
50 let schema = String::deserialize(deserializer)?;
51 if schema != SCHEMA {
52 return Err(serde::de::Error::custom(format_args!(
53 "unsupported evidence schema {schema:?}; expected {SCHEMA:?}"
54 )));
55 }
56 Ok(schema)
57}
58
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
60pub struct EvidenceFinding {
61 pub code: String,
62 pub summary: String,
63 pub source: String,
64 pub requires: Vec<CapabilityKind>,
65 pub qualification: CapabilityLevel,
66}
67
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69pub struct EvidenceFact {
70 pub code: String,
71 pub label: String,
72 pub value: FactValue,
73 pub source: String,
74 pub capability: CapabilityKind,
75}
76
77#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
78#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
79pub enum FactValue {
80 DurationNs(u64),
81 Bytes(u64),
82 Count(u64),
83 Text(String),
84}
85
86pub fn build_evidence(trace: &Path, nsight_dir: Option<&Path>) -> Result<EvidencePacket> {
87 let document =
88 parse_trace(trace).with_context(|| format!("parse trace {}", trace.display()))?;
89 let contract = &document.run.capture_contract;
90 let required_application_labels = contract.required_semantic_labels.clone();
91 let gpu_expected_semantic_labels = contract.resolved_gpu_expected_semantic_labels();
92 let cpu_only_semantic_labels = contract.resolved_cpu_only_semantic_labels();
93 let gpu = NsightEvidence::load_optional_with_semantic_contract(
94 nsight_dir,
95 &required_application_labels,
96 &gpu_expected_semantic_labels,
97 &cpu_only_semantic_labels,
98 );
99 EvidencePacket::from_document(document, gpu)
100}
101
102impl EvidencePacket {
103 pub fn validate_schema(&self) -> Result<()> {
106 ensure!(
107 self.schema == SCHEMA,
108 "unsupported evidence schema {:?}; expected {:?}",
109 self.schema,
110 SCHEMA
111 );
112 if let Some(graph) = &self.graph {
113 ensure!(
114 graph.schema == crate::graph::SCHEMA,
115 "unsupported graph schema {:?}; expected {:?}",
116 graph.schema,
117 crate::graph::SCHEMA
118 );
119 }
120 Ok(())
121 }
122
123 pub fn from_document(document: TraceDocument, mut gpu: NsightEvidence) -> Result<Self> {
124 gpu.bind_to_trace(&document.run.run_id, &document.run.correlation_id);
125 let health = analyze_health(&document);
126 let timing = analyze_timing(&document);
127 let memory = analyze_memory(&document);
128 let mut activations = ActivationProfile::from_trace(&document, &timing, &memory);
129 let capabilities = assess_capabilities(
130 &document,
131 &health,
132 &timing,
133 &memory,
134 &gpu,
135 activations.operations.len(),
136 );
137 activations.qualify(&capabilities);
138 let graph = (health.capture_complete && health.structurally_valid)
139 .then(|| build_from_trace(&document))
140 .transpose()?;
141 let tensor_stats = document.tensor_stats.clone();
142 let mut findings = Vec::new();
143 let mut facts = Vec::new();
144
145 if capabilities.outer_wall_time.is_available() {
146 if let Some(duration_ns) = document
147 .spans
148 .iter()
149 .find(|span| span.measured && span.closed)
150 .map(|span| span.duration_ns)
151 {
152 facts.push(EvidenceFact {
153 code: "outer_wall_time".into(),
154 label: "Measured region wall time".into(),
155 value: FactValue::DurationNs(duration_ns),
156 source: "measured_span".into(),
157 capability: CapabilityKind::OuterWallTime,
158 });
159 }
160 }
161 if capabilities.gradient_coverage.is_available() {
162 if let Some(contract) = document.run.capture_contract.gradient_contract.as_ref() {
163 facts.extend([
164 EvidenceFact {
165 code: "gradient_manifest_sha256".into(),
166 label: "Gradient manifest SHA-256".into(),
167 value: FactValue::Text(contract.manifest_sha256.clone()),
168 source: "capture_contract.gradient_contract".into(),
169 capability: CapabilityKind::Gradients,
170 },
171 EvidenceFact {
172 code: "gradient_manifest_entries".into(),
173 label: "Expected gradient parameters".into(),
174 value: FactValue::Count(contract.expected.len() as u64),
175 source: "capture_contract.gradient_contract".into(),
176 capability: CapabilityKind::Gradients,
177 },
178 EvidenceFact {
179 code: "gradient_family_expectations".into(),
180 label: "Gradient family expectations".into(),
181 value: FactValue::Count(contract.families.len() as u64),
182 source: "capture_contract.gradient_contract".into(),
183 capability: CapabilityKind::Gradients,
184 },
185 ]);
186 }
187 }
188 if let Some(graph) = &graph {
189 if let Some(span) = graph.summary.slowest_host_spans.first() {
190 findings.push(EvidenceFinding {
191 code: "largest_observed_host_self_time".into(),
192 summary: format!(
193 "`{}` has the largest observed measured-scope host self-time ({:.2} ms overlap-clipped, {:.2} ms full self-time; `{}` span duration {:.2} ms of {:.2} ms full)",
194 span.name,
195 span.measured_overlap_self_time_ns as f64 / 1_000_000.0,
196 span.host_self_time_ns as f64 / 1_000_000.0,
197 span.scope.as_str(),
198 span.measured_overlap_duration_ns as f64 / 1_000_000.0,
199 span.full_duration_ns as f64 / 1_000_000.0,
200 ),
201 source: span.id.clone(),
202 requires: vec![CapabilityKind::NestedHostTime],
203 qualification: capabilities.nested_host_time.level,
204 });
205 }
206 if let Some(span) = graph.summary.slowest_device_spans.first() {
207 findings.push(EvidenceFinding {
208 code: "largest_observed_device_busy_time".into(),
209 summary: format!("`{}` has the largest observed device-busy interval union on `{}` ({:.2} ms)", span.name, span.device, span.device_busy_ns as f64 / 1_000_000.0),
210 source: span.id.clone(),
211 requires: vec![CapabilityKind::NestedDeviceTime],
212 qualification: capabilities.nested_device_time.level,
213 });
214 }
215 let non_present = graph
216 .gradients
217 .iter()
218 .filter(|gradient| {
219 !matches!(gradient.state, crate::graph::GradientRecordState::Present)
220 })
221 .count();
222 if !graph.gradients.is_empty() {
223 facts.push(EvidenceFact {
224 code: "gradient_observations".into(),
225 label: "Non-present gradient observations".into(),
226 value: FactValue::Count(non_present as u64),
227 source: "trace.gradient".into(),
228 capability: CapabilityKind::Gradients,
229 });
230 }
231 }
232 if capabilities.logical_memory_coverage.is_available() {
233 if let Some(logical) = &memory.logical {
234 if let Some(peak) = &logical.peak {
235 facts.push(EvidenceFact {
236 code: "logical_peak_live_bytes".into(),
237 label: "Peak live logical storage".into(),
238 value: FactValue::Bytes(peak.live_bytes),
239 source: "logical_storage_lifetimes".into(),
240 capability: CapabilityKind::LogicalMemory,
241 });
242 }
243 }
244 }
245 if capabilities.activation_coverage.is_available() {
246 facts.push(EvidenceFact {
247 code: "activation_operations".into(),
248 label: "Observed activation-producing operations".into(),
249 value: FactValue::Count(activations.operations.len() as u64),
250 source: "trace activation category links".into(),
251 capability: CapabilityKind::Activations,
252 });
253 if activations.host_time.is_available() {
254 if let Some(operation) =
255 activations.top_by(|operation| Some(operation.observed_host_duration_ns))
256 {
257 findings.push(EvidenceFinding {
258 code: "largest_activation_host_time".into(),
259 summary: format!(
260 "`{}` has the largest observed activation-operation host duration ({:.2} ms); this is not device time",
261 operation.name,
262 operation.observed_host_duration_ns as f64 / 1_000_000.0,
263 ),
264 source: operation.id.clone(),
265 requires: vec![
266 CapabilityKind::Activations,
267 CapabilityKind::NestedHostTime,
268 ],
269 qualification: activations.host_time.level,
270 });
271 }
272 }
273 }
274 if health.capture_complete
275 && health.structurally_valid
276 && capabilities.gpu_correlation.is_available()
277 && capabilities.provenance_binding.is_available()
278 {
279 if let Some(kernel) = gpu.kernels.first() {
280 findings.push(EvidenceFinding {
281 code: "largest_nsight_kernel_total".into(),
282 summary: format!(
283 "`{}` has the largest normalized Nsight kernel total ({:.2} ms)",
284 kernel.name,
285 kernel.total_ns as f64 / 1_000_000.0
286 ),
287 source: "nsight.cuda_gpu_kern_sum".into(),
288 requires: vec![
289 CapabilityKind::GpuCorrelation,
290 CapabilityKind::ProvenanceBinding,
291 ],
292 qualification: weakest_level(
293 capabilities.gpu_correlation.level,
294 capabilities.provenance_binding.level,
295 ),
296 });
297 }
298 }
299
300 let mut gaps = health
301 .gaps()
302 .map(|issue| issue.message.clone())
303 .collect::<Vec<_>>();
304 for state in [
305 &capabilities.structural_trace,
306 &capabilities.outer_wall_time,
307 &capabilities.nested_host_time,
308 &capabilities.nested_device_time,
309 &capabilities.operation_coverage,
310 &capabilities.activation_coverage,
311 &capabilities.tensor_coverage,
312 &capabilities.gradient_coverage,
313 &capabilities.logical_memory_coverage,
314 &capabilities.physical_memory_coverage,
315 &capabilities.gpu_correlation,
316 &capabilities.provenance_binding,
317 ] {
318 if !state.is_complete() {
319 gaps.push(state.reason.clone());
320 }
321 }
322 gaps.sort();
323 gaps.dedup();
324
325 Ok(Self {
326 schema: SCHEMA.into(),
327 provenance: document.run,
328 health,
329 capabilities,
330 findings,
331 facts,
332 gaps,
333 graph,
334 tensor_stats,
335 timing,
336 memory,
337 activations,
338 gpu,
339 })
340 }
341
342 pub fn markdown(&self) -> String {
343 let status = if !self.health.structurally_valid {
344 "STRUCTURALLY INVALID"
345 } else if !self.health.capture_complete {
346 "FAILED CAPTURE"
347 } else {
348 "COMPLETE CAPTURE"
349 };
350 let mut output = format!(
351 "# candle-graph evidence\n\n- Status: **{status}**\n- Entrypoint: `{}`\n- Run: `{}`\n- Phase: `{}`\n- Device: `{}`\n\n## Capability matrix\n\n| Capability | Level | Source | Reason |\n| --- | --- | --- | --- |\n",
352 self.provenance.entrypoint, self.provenance.run_id, self.provenance.phase.as_str(), self.provenance.device,
353 );
354 for (name, state) in capability_rows(&self.capabilities) {
355 output.push_str(&format!(
356 "| {name} | `{:?}` | `{}` | {} |\n",
357 state.level,
358 state.source,
359 state.reason.replace('|', "\\|")
360 ));
361 }
362 output.push_str("\n## Qualified findings\n\n");
363 if self.findings.is_empty() {
364 output.push_str("No findings met their evidence prerequisites.\n");
365 } else {
366 for finding in &self.findings {
367 let requirements = finding
368 .requires
369 .iter()
370 .map(|requirement| format!("`{requirement:?}`"))
371 .collect::<Vec<_>>()
372 .join(", ");
373 output.push_str(&format!(
374 "- [`{:?}`] {} (requires {requirements})\n",
375 finding.qualification, finding.summary
376 ));
377 }
378 }
379 output.push_str("\n## Activation hotspots\n\n");
380 output.push_str(&format!(
381 "- Coverage: `{:?}` — {}\n",
382 self.activations.coverage.level, self.activations.coverage.reason
383 ));
384 output.push_str(
385 "- Metrics are independent: host duration, per-clock device busy time, dense output footprint, and logical storage are never combined.\n",
386 );
387 if self.activations.operations.is_empty() {
388 output
389 .push_str("- No category-linked activation-producing operations were observed.\n");
390 } else {
391 if let Some(operation) = self
392 .activations
393 .top_by(|operation| Some(operation.observed_host_duration_ns))
394 {
395 output.push_str(&format!(
396 "- Largest observed host duration: `{}` ({:.2} ms; qualification `{:?}`).\n",
397 operation.name,
398 operation.observed_host_duration_ns as f64 / 1_000_000.0,
399 self.activations.host_time.level,
400 ));
401 }
402 if let Some(operation) = self
403 .activations
404 .top_by(|operation| operation.dense_output_bytes)
405 {
406 output.push_str(&format!(
407 "- Largest dense output footprint: `{}` ({} bytes; qualification `{:?}`).\n",
408 operation.name,
409 operation.dense_output_bytes.unwrap_or_default(),
410 self.activations.dense_output.level,
411 ));
412 }
413 if let Some(operation) = self
414 .activations
415 .top_by(|operation| operation.logical_allocated_bytes)
416 {
417 output.push_str(&format!(
418 "- Largest attributed logical allocation: `{}` ({} bytes; qualification `{:?}`).\n",
419 operation.name,
420 operation.logical_allocated_bytes.unwrap_or_default(),
421 self.activations.logical_memory.level,
422 ));
423 }
424 if let Some(operation) = self
425 .activations
426 .top_by(|operation| operation.logical_byte_nanoseconds)
427 {
428 output.push_str(&format!(
429 "- Largest logical space-time footprint: `{}` ({} byte-ns; qualification `{:?}`).\n",
430 operation.name,
431 operation
432 .logical_byte_nanoseconds
433 .unwrap_or(ByteNanoseconds(0))
434 .0,
435 self.activations.logical_memory.level,
436 ));
437 }
438 }
439 output.push_str("\n## Evidence gaps\n\n");
440 if self.gaps.is_empty() {
441 output.push_str("No declared capability gaps.\n");
442 } else {
443 for gap in &self.gaps {
444 output.push_str(&format!("- {gap}\n"));
445 }
446 }
447 output
448 }
449}
450
451fn assess_capabilities(
452 document: &TraceDocument,
453 health: &TraceHealth,
454 timing: &TimingProfile,
455 memory: &MemoryProfile,
456 gpu: &NsightEvidence,
457 activation_observations: usize,
458) -> EvidenceCapabilities {
459 let trace_validation_source = || format!("{} validation", crate::trace::SCHEMA);
460 let structural_trace = if health.structurally_valid {
461 CapabilityState::from_coverage(
462 CoverageLevel::Complete,
463 trace_validation_source(),
464 "span and event invariants passed",
465 )
466 } else {
467 CapabilityState::invalid(
468 trace_validation_source(),
469 "one or more structural invariants failed",
470 )
471 };
472 let measured = document
473 .spans
474 .iter()
475 .filter(|span| span.measured && span.closed)
476 .count();
477 let outer_wall_time = if !health.structurally_valid {
478 CapabilityState::invalid(
479 trace_validation_source(),
480 "outer wall time is not qualified for a structurally invalid trace",
481 )
482 } else if !health.capture_complete {
483 CapabilityState::unavailable("capture did not complete")
484 } else if measured == 1 {
485 CapabilityState::from_coverage(
486 CoverageLevel::Complete,
487 "measured span",
488 "one closed measured region was observed",
489 )
490 } else {
491 CapabilityState::invalid(
492 "measured span",
493 "exactly one closed measured region is required",
494 )
495 };
496 let nested_host_time = if !health.structurally_valid {
497 CapabilityState::invalid(
498 trace_validation_source(),
499 "nested host attribution requires a structurally valid trace",
500 )
501 } else if health.capture_complete {
502 CapabilityState::from_coverage(
503 CoverageLevel::Complete,
504 "span wall intervals",
505 "measured-subtree and concurrent-overlap host intervals are structurally valid",
506 )
507 } else {
508 CapabilityState::unavailable(
509 "measured-scope host attribution requires a complete, valid capture",
510 )
511 };
512 let nested_device_time = observed_coverage(
513 timing.device_coverage,
514 document.device_intervals.len(),
515 "device intervals",
516 "device timing",
517 );
518 let operation_coverage = observed_coverage(
519 document.run.capture_contract.operations,
520 document.ops.len(),
521 "trace op events",
522 "operation coverage",
523 );
524 let activation_coverage = if let Some(reason) = document
525 .run
526 .capture_contract
527 .activation_contract_violation()
528 {
529 CapabilityState::invalid("capture contract", reason)
530 } else {
531 observed_coverage(
532 document.run.capture_contract.activations,
533 activation_observations,
534 "activation category links",
535 "activation-operation coverage",
536 )
537 };
538 let tensor_coverage = observed_coverage(
539 document.run.capture_contract.tensors,
540 document.tensors.len(),
541 "trace tensor events",
542 "tensor coverage",
543 );
544 let gradient_coverage = assess_gradient_coverage(document, health);
545 let logical_memory_coverage = observed_coverage(
546 document.run.capture_contract.logical_memory,
547 document.memory.len(),
548 "storage lifetime events",
549 "logical memory coverage",
550 );
551 let physical_memory_coverage = observed_coverage(
552 document.run.capture_contract.physical_memory,
553 document.device_memory.len(),
554 "device memory samples",
555 "physical memory coverage",
556 );
557 let provenance_binding = match (gpu.status, gpu.provenance.binding) {
558 (GpuEvidenceStatus::Unavailable, _) => {
559 CapabilityState::unavailable("no Nsight artifacts were supplied")
560 }
561 (GpuEvidenceStatus::Failed, _) => {
562 CapabilityState::invalid("Nsight artifact manifest", "artifact loading failed")
563 }
564 (_, ProvenanceBindingState::Bound) => CapabilityState::from_coverage(
565 CoverageLevel::Complete,
566 "capture-manifest.json",
567 "manifest IDs and artifact hashes match this trace",
568 ),
569 (_, ProvenanceBindingState::Partial) => CapabilityState::from_coverage(
570 CoverageLevel::Partial,
571 "Nsight artifact hashes",
572 "artifacts are hashed, but trace binding is incomplete",
573 ),
574 (_, ProvenanceBindingState::Mismatch) => CapabilityState::invalid(
575 "capture-manifest.json",
576 "manifest IDs or artifact hashes do not match",
577 ),
578 };
579 let required_labels = &document.run.capture_contract.required_semantic_labels;
580 let required_application_labels_present = required_labels
581 .iter()
582 .collect::<std::collections::BTreeSet<_>>()
583 .len()
584 == required_labels.len()
585 && required_labels.iter().all(|required| {
586 document
587 .spans
588 .iter()
589 .filter(|span| span.name == *required)
590 .count()
591 == 1
592 });
593 let gpu_correlation = if !required_application_labels_present {
594 CapabilityState::invalid(
595 "trace semantic labels",
596 "one or more labels required for GPU correlation are absent from the application trace",
597 )
598 } else {
599 match gpu.status {
600 GpuEvidenceStatus::Available if gpu.correlation.complete => {
601 CapabilityState::from_coverage(
602 CoverageLevel::Complete,
603 "Nsight NVTX projection",
604 "application and projected semantic labels matched",
605 )
606 }
607 GpuEvidenceStatus::Available => CapabilityState::from_coverage(
608 CoverageLevel::Partial,
609 "Nsight NVTX projection",
610 gpu.correlation
611 .reason
612 .clone()
613 .unwrap_or_else(|| "correlation is incomplete".into()),
614 ),
615 GpuEvidenceStatus::Unavailable => CapabilityState::unavailable(
616 gpu.reason
617 .clone()
618 .unwrap_or_else(|| "Nsight evidence was not supplied".into()),
619 ),
620 GpuEvidenceStatus::Failed => CapabilityState::invalid(
621 "Nsight normalization",
622 gpu.reason
623 .clone()
624 .unwrap_or_else(|| "Nsight normalization failed".into()),
625 ),
626 }
627 };
628 debug_assert_eq!(memory.logical.is_some(), !document.memory.is_empty());
629 debug_assert_eq!(
630 memory.physical.is_some(),
631 !document.device_memory.is_empty()
632 );
633 let mut capabilities = EvidenceCapabilities {
634 structural_trace,
635 outer_wall_time,
636 nested_host_time,
637 nested_device_time,
638 operation_coverage,
639 activation_coverage,
640 tensor_coverage,
641 gradient_coverage,
642 logical_memory_coverage,
643 physical_memory_coverage,
644 gpu_correlation,
645 provenance_binding,
646 };
647 if !health.structurally_valid {
648 for state in [
649 &mut capabilities.nested_device_time,
650 &mut capabilities.operation_coverage,
651 &mut capabilities.activation_coverage,
652 &mut capabilities.tensor_coverage,
653 &mut capabilities.gradient_coverage,
654 &mut capabilities.logical_memory_coverage,
655 &mut capabilities.gpu_correlation,
656 ] {
657 if state.is_available() {
658 *state = CapabilityState::invalid(
659 trace_validation_source(),
660 "structurally invalid trace cannot qualify trace-linked evidence",
661 );
662 }
663 }
664 }
665 if !health.capture_complete {
666 for state in [
667 &mut capabilities.nested_device_time,
668 &mut capabilities.operation_coverage,
669 &mut capabilities.activation_coverage,
670 &mut capabilities.tensor_coverage,
671 &mut capabilities.gradient_coverage,
672 &mut capabilities.logical_memory_coverage,
673 &mut capabilities.physical_memory_coverage,
674 &mut capabilities.gpu_correlation,
675 &mut capabilities.provenance_binding,
676 ] {
677 if state.level == CapabilityLevel::Complete {
678 state.level = CapabilityLevel::Partial;
679 state.reason = format!(
680 "{}; failed capture means observations are diagnostic only",
681 state.reason
682 );
683 }
684 }
685 }
686 capabilities
687}
688
689fn assess_gradient_coverage(document: &TraceDocument, health: &TraceHealth) -> CapabilityState {
690 let declared = document.run.capture_contract.gradients;
691 if declared != CoverageLevel::Complete {
692 return observed_coverage(
693 declared,
694 document.gradients.len(),
695 "trace gradient events",
696 "gradient coverage",
697 );
698 }
699
700 let Some(contract) = document.run.capture_contract.gradient_contract.as_ref() else {
701 return CapabilityState::invalid(
702 "exact gradient contract",
703 "complete gradient coverage requires a digest-bound parameter manifest",
704 );
705 };
706 if let Err(error) = contract.validate() {
707 return CapabilityState::invalid(
708 "exact gradient contract",
709 format!("gradient contract validation failed: {error}"),
710 );
711 }
712
713 let gradient_issues = health
714 .issues
715 .iter()
716 .filter(|issue| {
717 issue.code.starts_with("gradient_")
718 || matches!(
719 issue.code.as_str(),
720 "duplicate_gradient_event_id" | "empty_gradient_event_id"
721 )
722 })
723 .collect::<Vec<_>>();
724 if gradient_issues
725 .iter()
726 .any(|issue| issue.severity == HealthSeverity::Error)
727 {
728 return CapabilityState::invalid(
729 "exact gradient contract",
730 "gradient events did not satisfy the exact manifest and family contract",
731 );
732 }
733 if !gradient_issues.is_empty() {
734 return CapabilityState::from_coverage(
735 CoverageLevel::Partial,
736 "exact gradient contract",
737 "capture ended before every manifest and family expectation could be validated",
738 );
739 }
740
741 CapabilityState::from_coverage(
742 CoverageLevel::Complete,
743 "exact gradient contract",
744 format!(
745 "{} manifest entries and {} family expectations validated against {}",
746 contract.expected.len(),
747 contract.families.len(),
748 contract.manifest_sha256
749 ),
750 )
751}
752
753fn observed_coverage(
754 declared: CoverageLevel,
755 observations: usize,
756 source: &str,
757 label: &str,
758) -> CapabilityState {
759 match (declared, observations) {
760 (CoverageLevel::Complete, 0) => CapabilityState::invalid(
761 source,
762 format!("producer declared complete {label}, but emitted no observations"),
763 ),
764 (CoverageLevel::Complete, _) => CapabilityState::from_coverage(
765 CoverageLevel::Complete,
766 source,
767 format!("producer declared complete {label}"),
768 ),
769 (CoverageLevel::Partial, 0) => CapabilityState::unavailable(format!(
770 "producer declared partial {label}, but this run emitted no observations"
771 )),
772 (CoverageLevel::Partial, _) => CapabilityState::from_coverage(
773 CoverageLevel::Partial,
774 source,
775 format!("producer declared partial {label}"),
776 ),
777 (CoverageLevel::None, 0) => {
778 CapabilityState::unavailable(format!("producer did not declare or emit {label}"))
779 }
780 (CoverageLevel::None, _) => CapabilityState::from_coverage(
781 CoverageLevel::Partial,
782 source,
783 format!("observations exist, but producer did not declare complete {label}"),
784 ),
785 }
786}
787
788fn capability_rows(capabilities: &EvidenceCapabilities) -> [(&'static str, &CapabilityState); 12] {
789 [
790 ("Structural trace", &capabilities.structural_trace),
791 ("Outer wall time", &capabilities.outer_wall_time),
792 ("Measured-scope host time", &capabilities.nested_host_time),
793 ("Nested device time", &capabilities.nested_device_time),
794 ("Operations", &capabilities.operation_coverage),
795 ("Activation operations", &capabilities.activation_coverage),
796 ("Tensors", &capabilities.tensor_coverage),
797 ("Gradients", &capabilities.gradient_coverage),
798 ("Logical memory", &capabilities.logical_memory_coverage),
799 ("Physical memory", &capabilities.physical_memory_coverage),
800 ("GPU correlation", &capabilities.gpu_correlation),
801 ("Provenance binding", &capabilities.provenance_binding),
802 ]
803}
804
805fn weakest_level(left: CapabilityLevel, right: CapabilityLevel) -> CapabilityLevel {
806 use CapabilityLevel::{Complete, Invalid, Partial, Unavailable};
807 match (left, right) {
808 (Invalid, _) | (_, Invalid) => Invalid,
809 (Unavailable, _) | (_, Unavailable) => Unavailable,
810 (Partial, _) | (_, Partial) => Partial,
811 (Complete, Complete) => Complete,
812 }
813}
814
815#[cfg(test)]
816mod tests {
817 use super::*;
818 use crate::capability::{CaptureContract, MeasurementScope};
819 use crate::nsight::NsightEvidence;
820 use crate::trace::{
821 GradientEvent, GradientState, OpEvent, RunOutcome, SpanKind, SpanRecord, TerminalEvent,
822 TimingMode, TraceRunMeta, SCHEMA as TRACE_SCHEMA,
823 };
824
825 #[test]
826 fn failed_capture_downgrades_observed_capabilities_and_emits_no_findings() {
827 let document = TraceDocument {
828 schema: TRACE_SCHEMA.into(),
829 run: TraceRunMeta {
830 run_id: "failed".into(),
831 correlation_id: "failed/run".into(),
832 entrypoint: "demo".into(),
833 phase: crate::ExecutionPhase::Infer,
834 timestamp: "2026-08-19T00:00:00Z".into(),
835 capture_step: 1,
836 warmup_steps: 0,
837 device: "cpu".into(),
838 measured_region_device_synchronized: false,
839 timing_mode: TimingMode::Host,
840 capture_contract: CaptureContract {
841 measurement_scope: MeasurementScope::ProfiledWork,
842 operations: CoverageLevel::Complete,
843 ..CaptureContract::default()
844 },
845 comparison_identity: None,
846 tags: Default::default(),
847 candle_version: None,
848 },
849 spans: vec![SpanRecord {
850 id: "root".into(),
851 parent_id: None,
852 name: "demo".into(),
853 kind: SpanKind::Function,
854 measured: false,
855 start_ns: 0,
856 closed: false,
857 duration_ns: 0,
858 step: None,
859 }],
860 ops: vec![OpEvent {
861 span_id: "root".into(),
862 op_name: "add".into(),
863 inputs: vec![],
864 output: None,
865 shape: vec![1],
866 dtype: "f32".into(),
867 device: "cpu".into(),
868 duration_ns: 1,
869 timestamp_ns: 1,
870 output_dense_bytes: Some(4),
871 input_dense_bytes: 0,
872 }],
873 tensors: vec![],
874 tensor_stats: vec![],
875 memory: vec![],
876 device_memory: vec![],
877 device_intervals: vec![],
878 gradients: vec![],
879 edges: vec![],
880 terminal: TerminalEvent {
881 outcome: RunOutcome::Failed,
882 timestamp_ns: 2,
883 reason: Some("interrupted".into()),
884 },
885 };
886 let packet =
887 EvidencePacket::from_document(document, NsightEvidence::unavailable("not captured"))
888 .unwrap();
889 assert_eq!(
890 packet.capabilities.operation_coverage.level,
891 CapabilityLevel::Partial
892 );
893 assert_eq!(
894 packet.capabilities.structural_trace.source,
895 format!("{TRACE_SCHEMA} validation")
896 );
897 assert!(packet.findings.is_empty());
898 assert!(packet.graph.is_none());
899 }
900
901 #[test]
902 fn tensor_and_gradient_capabilities_enforce_declared_coverage() {
903 let document = TraceDocument {
904 schema: TRACE_SCHEMA.into(),
905 run: TraceRunMeta {
906 run_id: "typed-coverage".into(),
907 correlation_id: "typed/coverage".into(),
908 entrypoint: "demo".into(),
909 phase: crate::ExecutionPhase::Train,
910 timestamp: "2026-08-19T00:00:00Z".into(),
911 capture_step: 1,
912 warmup_steps: 0,
913 device: "cpu".into(),
914 measured_region_device_synchronized: false,
915 timing_mode: TimingMode::Host,
916 capture_contract: CaptureContract {
917 measurement_scope: MeasurementScope::ProfiledWork,
918 tensors: CoverageLevel::Complete,
919 gradients: CoverageLevel::None,
920 ..CaptureContract::default()
921 },
922 comparison_identity: None,
923 tags: Default::default(),
924 candle_version: None,
925 },
926 spans: vec![SpanRecord {
927 id: "root".into(),
928 parent_id: None,
929 name: "demo".into(),
930 kind: SpanKind::Function,
931 measured: true,
932 start_ns: 0,
933 closed: true,
934 duration_ns: 1,
935 step: None,
936 }],
937 ops: vec![],
938 tensors: vec![],
939 tensor_stats: vec![],
940 memory: vec![],
941 device_memory: vec![],
942 device_intervals: vec![],
943 gradients: vec![GradientEvent {
944 event_id: "gradient-1".into(),
945 root: "parameters".into(),
946 key: "weight".into(),
947 state: GradientState::Present,
948 norm: Some(1.0),
949 }],
950 edges: vec![],
951 terminal: TerminalEvent {
952 outcome: RunOutcome::Complete,
953 timestamp_ns: 1,
954 reason: None,
955 },
956 };
957 let packet =
958 EvidencePacket::from_document(document, NsightEvidence::unavailable("not captured"))
959 .unwrap();
960 assert_eq!(
961 packet.capabilities.tensor_coverage.level,
962 CapabilityLevel::Invalid
963 );
964 assert_eq!(
965 packet.capabilities.gradient_coverage.level,
966 CapabilityLevel::Partial
967 );
968 }
969
970 #[test]
971 fn older_capability_packets_default_new_typed_fields() {
972 let json = serde_json::json!({
973 "structural_trace": CapabilityState::default(),
974 "outer_wall_time": CapabilityState::default(),
975 "nested_host_time": CapabilityState::default(),
976 "nested_device_time": CapabilityState::default(),
977 "operation_coverage": CapabilityState::default(),
978 "logical_memory_coverage": CapabilityState::default(),
979 "physical_memory_coverage": CapabilityState::default(),
980 "gpu_correlation": CapabilityState::default(),
981 "provenance_binding": CapabilityState::default()
982 });
983 let capabilities: EvidenceCapabilities = serde_json::from_value(json).unwrap();
984 assert_eq!(
985 capabilities.tensor_coverage.level,
986 CapabilityLevel::Unavailable
987 );
988 assert_eq!(
989 capabilities.gradient_coverage.level,
990 CapabilityLevel::Unavailable
991 );
992 }
993}