1use std::path::Path;
4
5use anyhow::{ensure, Context, Result};
6use serde::{Deserialize, Serialize};
7
8use crate::capability::{
9 CapabilityKind, CapabilityLevel, CapabilityState, CoverageLevel, EvidenceCapabilities,
10};
11use crate::graph::{build_from_trace, ExecutionGraph};
12use crate::nsight::{GpuEvidenceStatus, NsightEvidence, ProvenanceBindingState};
13use crate::timing::{analyze_timing, TimingProfile};
14use crate::trace::memory::{analyze_memory, MemoryProfile};
15use crate::trace::{
16 analyze_health, parse_trace, HealthSeverity, TensorStatsEvent, TraceDocument, TraceHealth,
17 TraceRunMeta,
18};
19
20pub const SCHEMA: &str = "candle-graph/evidence/4";
21
22#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
23pub struct EvidencePacket {
24 #[serde(deserialize_with = "deserialize_schema")]
25 pub schema: String,
26 pub provenance: TraceRunMeta,
27 pub health: TraceHealth,
28 pub capabilities: EvidenceCapabilities,
29 pub findings: Vec<EvidenceFinding>,
30 pub facts: Vec<EvidenceFact>,
31 pub gaps: Vec<String>,
32 #[serde(default, skip_serializing_if = "Option::is_none")]
34 pub graph: Option<ExecutionGraph>,
35 #[serde(default, skip_serializing_if = "Vec::is_empty")]
37 pub tensor_stats: Vec<TensorStatsEvent>,
38 pub timing: TimingProfile,
39 pub memory: MemoryProfile,
40 pub gpu: NsightEvidence,
41}
42
43fn deserialize_schema<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
44where
45 D: serde::Deserializer<'de>,
46{
47 let schema = String::deserialize(deserializer)?;
48 if schema != SCHEMA {
49 return Err(serde::de::Error::custom(format_args!(
50 "unsupported evidence schema {schema:?}; expected {SCHEMA:?}"
51 )));
52 }
53 Ok(schema)
54}
55
56#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
57pub struct EvidenceFinding {
58 pub code: String,
59 pub summary: String,
60 pub source: String,
61 pub requires: Vec<CapabilityKind>,
62 pub qualification: CapabilityLevel,
63}
64
65#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
66pub struct EvidenceFact {
67 pub code: String,
68 pub label: String,
69 pub value: FactValue,
70 pub source: String,
71 pub capability: CapabilityKind,
72}
73
74#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
75#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
76pub enum FactValue {
77 DurationNs(u64),
78 Bytes(u64),
79 Count(u64),
80 Text(String),
81}
82
83pub fn build_evidence(trace: &Path, nsight_dir: Option<&Path>) -> Result<EvidencePacket> {
84 let document =
85 parse_trace(trace).with_context(|| format!("parse trace {}", trace.display()))?;
86 let contract = &document.run.capture_contract;
87 let required_application_labels = contract.required_semantic_labels.clone();
88 let gpu_expected_semantic_labels = contract.resolved_gpu_expected_semantic_labels();
89 let cpu_only_semantic_labels = contract.resolved_cpu_only_semantic_labels();
90 let gpu = NsightEvidence::load_optional_with_semantic_contract(
91 nsight_dir,
92 &required_application_labels,
93 &gpu_expected_semantic_labels,
94 &cpu_only_semantic_labels,
95 );
96 EvidencePacket::from_document(document, gpu)
97}
98
99impl EvidencePacket {
100 pub fn validate_schema(&self) -> Result<()> {
103 ensure!(
104 self.schema == SCHEMA,
105 "unsupported evidence schema {:?}; expected {:?}",
106 self.schema,
107 SCHEMA
108 );
109 if let Some(graph) = &self.graph {
110 ensure!(
111 graph.schema == crate::graph::SCHEMA,
112 "unsupported graph schema {:?}; expected {:?}",
113 graph.schema,
114 crate::graph::SCHEMA
115 );
116 }
117 Ok(())
118 }
119
120 pub fn from_document(document: TraceDocument, mut gpu: NsightEvidence) -> Result<Self> {
121 gpu.bind_to_trace(&document.run.run_id, &document.run.correlation_id);
122 let health = analyze_health(&document);
123 let timing = analyze_timing(&document);
124 let memory = analyze_memory(&document);
125 let capabilities = assess_capabilities(&document, &health, &timing, &memory, &gpu);
126 let graph = (health.capture_complete && health.structurally_valid)
127 .then(|| build_from_trace(&document))
128 .transpose()?;
129 let tensor_stats = document.tensor_stats.clone();
130 let mut findings = Vec::new();
131 let mut facts = Vec::new();
132
133 if capabilities.outer_wall_time.is_available() {
134 if let Some(duration_ns) = document
135 .spans
136 .iter()
137 .find(|span| span.measured && span.closed)
138 .map(|span| span.duration_ns)
139 {
140 facts.push(EvidenceFact {
141 code: "outer_wall_time".into(),
142 label: "Measured region wall time".into(),
143 value: FactValue::DurationNs(duration_ns),
144 source: "measured_span".into(),
145 capability: CapabilityKind::OuterWallTime,
146 });
147 }
148 }
149 if capabilities.gradient_coverage.is_available() {
150 if let Some(contract) = document.run.capture_contract.gradient_contract.as_ref() {
151 facts.extend([
152 EvidenceFact {
153 code: "gradient_manifest_sha256".into(),
154 label: "Gradient manifest SHA-256".into(),
155 value: FactValue::Text(contract.manifest_sha256.clone()),
156 source: "capture_contract.gradient_contract".into(),
157 capability: CapabilityKind::Gradients,
158 },
159 EvidenceFact {
160 code: "gradient_manifest_entries".into(),
161 label: "Expected gradient parameters".into(),
162 value: FactValue::Count(contract.expected.len() as u64),
163 source: "capture_contract.gradient_contract".into(),
164 capability: CapabilityKind::Gradients,
165 },
166 EvidenceFact {
167 code: "gradient_family_expectations".into(),
168 label: "Gradient family expectations".into(),
169 value: FactValue::Count(contract.families.len() as u64),
170 source: "capture_contract.gradient_contract".into(),
171 capability: CapabilityKind::Gradients,
172 },
173 ]);
174 }
175 }
176 if let Some(graph) = &graph {
177 if let Some(span) = graph.summary.slowest_host_spans.first() {
178 findings.push(EvidenceFinding {
179 code: "largest_observed_host_self_time".into(),
180 summary: format!(
181 "`{}` has the largest observed measured-scope host self-time ({:.2} ms overlap-clipped, {:.2} ms full self-time; `{}` span duration {:.2} ms of {:.2} ms full)",
182 span.name,
183 span.measured_overlap_self_time_ns as f64 / 1_000_000.0,
184 span.host_self_time_ns as f64 / 1_000_000.0,
185 span.scope.as_str(),
186 span.measured_overlap_duration_ns as f64 / 1_000_000.0,
187 span.full_duration_ns as f64 / 1_000_000.0,
188 ),
189 source: span.id.clone(),
190 requires: vec![CapabilityKind::NestedHostTime],
191 qualification: capabilities.nested_host_time.level,
192 });
193 }
194 if let Some(span) = graph.summary.slowest_device_spans.first() {
195 findings.push(EvidenceFinding {
196 code: "largest_observed_device_busy_time".into(),
197 summary: format!("`{}` has the largest observed device-busy interval union on `{}` ({:.2} ms)", span.name, span.device, span.device_busy_ns as f64 / 1_000_000.0),
198 source: span.id.clone(),
199 requires: vec![CapabilityKind::NestedDeviceTime],
200 qualification: capabilities.nested_device_time.level,
201 });
202 }
203 let non_present = graph
204 .gradients
205 .iter()
206 .filter(|gradient| {
207 !matches!(gradient.state, crate::graph::GradientRecordState::Present)
208 })
209 .count();
210 if !graph.gradients.is_empty() {
211 facts.push(EvidenceFact {
212 code: "gradient_observations".into(),
213 label: "Non-present gradient observations".into(),
214 value: FactValue::Count(non_present as u64),
215 source: "trace.gradient".into(),
216 capability: CapabilityKind::Gradients,
217 });
218 }
219 }
220 if capabilities.logical_memory_coverage.is_available() {
221 if let Some(logical) = &memory.logical {
222 if let Some(peak) = &logical.peak {
223 facts.push(EvidenceFact {
224 code: "logical_peak_live_bytes".into(),
225 label: "Peak live logical storage".into(),
226 value: FactValue::Bytes(peak.live_bytes),
227 source: "logical_storage_lifetimes".into(),
228 capability: CapabilityKind::LogicalMemory,
229 });
230 }
231 }
232 }
233 if health.capture_complete
234 && health.structurally_valid
235 && capabilities.gpu_correlation.is_available()
236 && capabilities.provenance_binding.is_available()
237 {
238 if let Some(kernel) = gpu.kernels.first() {
239 findings.push(EvidenceFinding {
240 code: "largest_nsight_kernel_total".into(),
241 summary: format!(
242 "`{}` has the largest normalized Nsight kernel total ({:.2} ms)",
243 kernel.name,
244 kernel.total_ns as f64 / 1_000_000.0
245 ),
246 source: "nsight.cuda_gpu_kern_sum".into(),
247 requires: vec![
248 CapabilityKind::GpuCorrelation,
249 CapabilityKind::ProvenanceBinding,
250 ],
251 qualification: weakest_level(
252 capabilities.gpu_correlation.level,
253 capabilities.provenance_binding.level,
254 ),
255 });
256 }
257 }
258
259 let mut gaps = health
260 .gaps()
261 .map(|issue| issue.message.clone())
262 .collect::<Vec<_>>();
263 for state in [
264 &capabilities.structural_trace,
265 &capabilities.outer_wall_time,
266 &capabilities.nested_host_time,
267 &capabilities.nested_device_time,
268 &capabilities.operation_coverage,
269 &capabilities.tensor_coverage,
270 &capabilities.gradient_coverage,
271 &capabilities.logical_memory_coverage,
272 &capabilities.physical_memory_coverage,
273 &capabilities.gpu_correlation,
274 &capabilities.provenance_binding,
275 ] {
276 if !state.is_complete() {
277 gaps.push(state.reason.clone());
278 }
279 }
280 gaps.sort();
281 gaps.dedup();
282
283 Ok(Self {
284 schema: SCHEMA.into(),
285 provenance: document.run,
286 health,
287 capabilities,
288 findings,
289 facts,
290 gaps,
291 graph,
292 tensor_stats,
293 timing,
294 memory,
295 gpu,
296 })
297 }
298
299 pub fn markdown(&self) -> String {
300 let status = if !self.health.structurally_valid {
301 "STRUCTURALLY INVALID"
302 } else if !self.health.capture_complete {
303 "FAILED CAPTURE"
304 } else {
305 "COMPLETE CAPTURE"
306 };
307 let mut output = format!(
308 "# candle-graph evidence\n\n- Status: **{status}**\n- Entrypoint: `{}`\n- Run: `{}`\n- Phase: `{}`\n- Device: `{}`\n\n## Capability matrix\n\n| Capability | Level | Source | Reason |\n| --- | --- | --- | --- |\n",
309 self.provenance.entrypoint, self.provenance.run_id, self.provenance.phase.as_str(), self.provenance.device,
310 );
311 for (name, state) in capability_rows(&self.capabilities) {
312 output.push_str(&format!(
313 "| {name} | `{:?}` | `{}` | {} |\n",
314 state.level,
315 state.source,
316 state.reason.replace('|', "\\|")
317 ));
318 }
319 output.push_str("\n## Qualified findings\n\n");
320 if self.findings.is_empty() {
321 output.push_str("No findings met their evidence prerequisites.\n");
322 } else {
323 for finding in &self.findings {
324 let requirements = finding
325 .requires
326 .iter()
327 .map(|requirement| format!("`{requirement:?}`"))
328 .collect::<Vec<_>>()
329 .join(", ");
330 output.push_str(&format!(
331 "- [`{:?}`] {} (requires {requirements})\n",
332 finding.qualification, finding.summary
333 ));
334 }
335 }
336 output.push_str("\n## Evidence gaps\n\n");
337 if self.gaps.is_empty() {
338 output.push_str("No declared capability gaps.\n");
339 } else {
340 for gap in &self.gaps {
341 output.push_str(&format!("- {gap}\n"));
342 }
343 }
344 output
345 }
346}
347
348fn assess_capabilities(
349 document: &TraceDocument,
350 health: &TraceHealth,
351 timing: &TimingProfile,
352 memory: &MemoryProfile,
353 gpu: &NsightEvidence,
354) -> EvidenceCapabilities {
355 let trace_validation_source = || format!("{} validation", crate::trace::SCHEMA);
356 let structural_trace = if health.structurally_valid {
357 CapabilityState::from_coverage(
358 CoverageLevel::Complete,
359 trace_validation_source(),
360 "span and event invariants passed",
361 )
362 } else {
363 CapabilityState::invalid(
364 trace_validation_source(),
365 "one or more structural invariants failed",
366 )
367 };
368 let measured = document
369 .spans
370 .iter()
371 .filter(|span| span.measured && span.closed)
372 .count();
373 let outer_wall_time = if !health.structurally_valid {
374 CapabilityState::invalid(
375 trace_validation_source(),
376 "outer wall time is not qualified for a structurally invalid trace",
377 )
378 } else if !health.capture_complete {
379 CapabilityState::unavailable("capture did not complete")
380 } else if measured == 1 {
381 CapabilityState::from_coverage(
382 CoverageLevel::Complete,
383 "measured span",
384 "one closed measured region was observed",
385 )
386 } else {
387 CapabilityState::invalid(
388 "measured span",
389 "exactly one closed measured region is required",
390 )
391 };
392 let nested_host_time = if !health.structurally_valid {
393 CapabilityState::invalid(
394 trace_validation_source(),
395 "nested host attribution requires a structurally valid trace",
396 )
397 } else if health.capture_complete {
398 CapabilityState::from_coverage(
399 CoverageLevel::Complete,
400 "span wall intervals",
401 "measured-subtree and concurrent-overlap host intervals are structurally valid",
402 )
403 } else {
404 CapabilityState::unavailable(
405 "measured-scope host attribution requires a complete, valid capture",
406 )
407 };
408 let nested_device_time = observed_coverage(
409 timing.device_coverage,
410 document.device_intervals.len(),
411 "device intervals",
412 "device timing",
413 );
414 let operation_coverage = observed_coverage(
415 document.run.capture_contract.operations,
416 document.ops.len(),
417 "trace op events",
418 "operation coverage",
419 );
420 let tensor_coverage = observed_coverage(
421 document.run.capture_contract.tensors,
422 document.tensors.len(),
423 "trace tensor events",
424 "tensor coverage",
425 );
426 let gradient_coverage = assess_gradient_coverage(document, health);
427 let logical_memory_coverage = observed_coverage(
428 document.run.capture_contract.logical_memory,
429 document.memory.len(),
430 "storage lifetime events",
431 "logical memory coverage",
432 );
433 let physical_memory_coverage = observed_coverage(
434 document.run.capture_contract.physical_memory,
435 document.device_memory.len(),
436 "device memory samples",
437 "physical memory coverage",
438 );
439 let provenance_binding = match (gpu.status, gpu.provenance.binding) {
440 (GpuEvidenceStatus::Unavailable, _) => {
441 CapabilityState::unavailable("no Nsight artifacts were supplied")
442 }
443 (GpuEvidenceStatus::Failed, _) => {
444 CapabilityState::invalid("Nsight artifact manifest", "artifact loading failed")
445 }
446 (_, ProvenanceBindingState::Bound) => CapabilityState::from_coverage(
447 CoverageLevel::Complete,
448 "capture-manifest.json",
449 "manifest IDs and artifact hashes match this trace",
450 ),
451 (_, ProvenanceBindingState::Partial) => CapabilityState::from_coverage(
452 CoverageLevel::Partial,
453 "Nsight artifact hashes",
454 "artifacts are hashed, but trace binding is incomplete",
455 ),
456 (_, ProvenanceBindingState::Mismatch) => CapabilityState::invalid(
457 "capture-manifest.json",
458 "manifest IDs or artifact hashes do not match",
459 ),
460 };
461 let required_labels = &document.run.capture_contract.required_semantic_labels;
462 let required_application_labels_present = required_labels
463 .iter()
464 .collect::<std::collections::BTreeSet<_>>()
465 .len()
466 == required_labels.len()
467 && required_labels.iter().all(|required| {
468 document
469 .spans
470 .iter()
471 .filter(|span| span.name == *required)
472 .count()
473 == 1
474 });
475 let gpu_correlation = if !required_application_labels_present {
476 CapabilityState::invalid(
477 "trace semantic labels",
478 "one or more labels required for GPU correlation are absent from the application trace",
479 )
480 } else {
481 match gpu.status {
482 GpuEvidenceStatus::Available if gpu.correlation.complete => {
483 CapabilityState::from_coverage(
484 CoverageLevel::Complete,
485 "Nsight NVTX projection",
486 "application and projected semantic labels matched",
487 )
488 }
489 GpuEvidenceStatus::Available => CapabilityState::from_coverage(
490 CoverageLevel::Partial,
491 "Nsight NVTX projection",
492 gpu.correlation
493 .reason
494 .clone()
495 .unwrap_or_else(|| "correlation is incomplete".into()),
496 ),
497 GpuEvidenceStatus::Unavailable => CapabilityState::unavailable(
498 gpu.reason
499 .clone()
500 .unwrap_or_else(|| "Nsight evidence was not supplied".into()),
501 ),
502 GpuEvidenceStatus::Failed => CapabilityState::invalid(
503 "Nsight normalization",
504 gpu.reason
505 .clone()
506 .unwrap_or_else(|| "Nsight normalization failed".into()),
507 ),
508 }
509 };
510 debug_assert_eq!(memory.logical.is_some(), !document.memory.is_empty());
511 debug_assert_eq!(
512 memory.physical.is_some(),
513 !document.device_memory.is_empty()
514 );
515 let mut capabilities = EvidenceCapabilities {
516 structural_trace,
517 outer_wall_time,
518 nested_host_time,
519 nested_device_time,
520 operation_coverage,
521 tensor_coverage,
522 gradient_coverage,
523 logical_memory_coverage,
524 physical_memory_coverage,
525 gpu_correlation,
526 provenance_binding,
527 };
528 if !health.structurally_valid {
529 for state in [
530 &mut capabilities.nested_device_time,
531 &mut capabilities.operation_coverage,
532 &mut capabilities.tensor_coverage,
533 &mut capabilities.gradient_coverage,
534 &mut capabilities.logical_memory_coverage,
535 &mut capabilities.gpu_correlation,
536 ] {
537 if state.is_available() {
538 *state = CapabilityState::invalid(
539 trace_validation_source(),
540 "structurally invalid trace cannot qualify trace-linked evidence",
541 );
542 }
543 }
544 }
545 if !health.capture_complete {
546 for state in [
547 &mut capabilities.nested_device_time,
548 &mut capabilities.operation_coverage,
549 &mut capabilities.tensor_coverage,
550 &mut capabilities.gradient_coverage,
551 &mut capabilities.logical_memory_coverage,
552 &mut capabilities.physical_memory_coverage,
553 &mut capabilities.gpu_correlation,
554 &mut capabilities.provenance_binding,
555 ] {
556 if state.level == CapabilityLevel::Complete {
557 state.level = CapabilityLevel::Partial;
558 state.reason = format!(
559 "{}; failed capture means observations are diagnostic only",
560 state.reason
561 );
562 }
563 }
564 }
565 capabilities
566}
567
568fn assess_gradient_coverage(document: &TraceDocument, health: &TraceHealth) -> CapabilityState {
569 let declared = document.run.capture_contract.gradients;
570 if declared != CoverageLevel::Complete {
571 return observed_coverage(
572 declared,
573 document.gradients.len(),
574 "trace gradient events",
575 "gradient coverage",
576 );
577 }
578
579 let Some(contract) = document.run.capture_contract.gradient_contract.as_ref() else {
580 return CapabilityState::invalid(
581 "exact gradient contract",
582 "complete gradient coverage requires a digest-bound parameter manifest",
583 );
584 };
585 if let Err(error) = contract.validate() {
586 return CapabilityState::invalid(
587 "exact gradient contract",
588 format!("gradient contract validation failed: {error}"),
589 );
590 }
591
592 let gradient_issues = health
593 .issues
594 .iter()
595 .filter(|issue| {
596 issue.code.starts_with("gradient_")
597 || matches!(
598 issue.code.as_str(),
599 "duplicate_gradient_event_id" | "empty_gradient_event_id"
600 )
601 })
602 .collect::<Vec<_>>();
603 if gradient_issues
604 .iter()
605 .any(|issue| issue.severity == HealthSeverity::Error)
606 {
607 return CapabilityState::invalid(
608 "exact gradient contract",
609 "gradient events did not satisfy the exact manifest and family contract",
610 );
611 }
612 if !gradient_issues.is_empty() {
613 return CapabilityState::from_coverage(
614 CoverageLevel::Partial,
615 "exact gradient contract",
616 "capture ended before every manifest and family expectation could be validated",
617 );
618 }
619
620 CapabilityState::from_coverage(
621 CoverageLevel::Complete,
622 "exact gradient contract",
623 format!(
624 "{} manifest entries and {} family expectations validated against {}",
625 contract.expected.len(),
626 contract.families.len(),
627 contract.manifest_sha256
628 ),
629 )
630}
631
632fn observed_coverage(
633 declared: CoverageLevel,
634 observations: usize,
635 source: &str,
636 label: &str,
637) -> CapabilityState {
638 match (declared, observations) {
639 (CoverageLevel::Complete, 0) => CapabilityState::invalid(
640 source,
641 format!("producer declared complete {label}, but emitted no observations"),
642 ),
643 (CoverageLevel::Complete, _) => CapabilityState::from_coverage(
644 CoverageLevel::Complete,
645 source,
646 format!("producer declared complete {label}"),
647 ),
648 (CoverageLevel::Partial, 0) => CapabilityState::unavailable(format!(
649 "producer declared partial {label}, but this run emitted no observations"
650 )),
651 (CoverageLevel::Partial, _) => CapabilityState::from_coverage(
652 CoverageLevel::Partial,
653 source,
654 format!("producer declared partial {label}"),
655 ),
656 (CoverageLevel::None, 0) => {
657 CapabilityState::unavailable(format!("producer did not declare or emit {label}"))
658 }
659 (CoverageLevel::None, _) => CapabilityState::from_coverage(
660 CoverageLevel::Partial,
661 source,
662 format!("observations exist, but producer did not declare complete {label}"),
663 ),
664 }
665}
666
667fn capability_rows(capabilities: &EvidenceCapabilities) -> [(&'static str, &CapabilityState); 11] {
668 [
669 ("Structural trace", &capabilities.structural_trace),
670 ("Outer wall time", &capabilities.outer_wall_time),
671 ("Measured-scope host time", &capabilities.nested_host_time),
672 ("Nested device time", &capabilities.nested_device_time),
673 ("Operations", &capabilities.operation_coverage),
674 ("Tensors", &capabilities.tensor_coverage),
675 ("Gradients", &capabilities.gradient_coverage),
676 ("Logical memory", &capabilities.logical_memory_coverage),
677 ("Physical memory", &capabilities.physical_memory_coverage),
678 ("GPU correlation", &capabilities.gpu_correlation),
679 ("Provenance binding", &capabilities.provenance_binding),
680 ]
681}
682
683fn weakest_level(left: CapabilityLevel, right: CapabilityLevel) -> CapabilityLevel {
684 use CapabilityLevel::{Complete, Invalid, Partial, Unavailable};
685 match (left, right) {
686 (Invalid, _) | (_, Invalid) => Invalid,
687 (Unavailable, _) | (_, Unavailable) => Unavailable,
688 (Partial, _) | (_, Partial) => Partial,
689 (Complete, Complete) => Complete,
690 }
691}
692
693#[cfg(test)]
694mod tests {
695 use super::*;
696 use crate::capability::{CaptureContract, MeasurementScope};
697 use crate::nsight::NsightEvidence;
698 use crate::trace::{
699 GradientEvent, GradientState, OpEvent, RunOutcome, SpanKind, SpanRecord, TerminalEvent,
700 TimingMode, TraceRunMeta, SCHEMA as TRACE_SCHEMA,
701 };
702
703 #[test]
704 fn failed_capture_downgrades_observed_capabilities_and_emits_no_findings() {
705 let document = TraceDocument {
706 schema: TRACE_SCHEMA.into(),
707 run: TraceRunMeta {
708 run_id: "failed".into(),
709 correlation_id: "failed/run".into(),
710 entrypoint: "demo".into(),
711 phase: crate::ExecutionPhase::Infer,
712 timestamp: "2026-08-19T00:00:00Z".into(),
713 capture_step: 1,
714 warmup_steps: 0,
715 device: "cpu".into(),
716 measured_region_device_synchronized: false,
717 timing_mode: TimingMode::Host,
718 capture_contract: CaptureContract {
719 measurement_scope: MeasurementScope::ProfiledWork,
720 operations: CoverageLevel::Complete,
721 ..CaptureContract::default()
722 },
723 comparison_identity: None,
724 tags: Default::default(),
725 candle_version: None,
726 },
727 spans: vec![SpanRecord {
728 id: "root".into(),
729 parent_id: None,
730 name: "demo".into(),
731 kind: SpanKind::Function,
732 measured: false,
733 start_ns: 0,
734 closed: false,
735 duration_ns: 0,
736 step: None,
737 }],
738 ops: vec![OpEvent {
739 span_id: "root".into(),
740 op_name: "add".into(),
741 inputs: vec![],
742 output: None,
743 shape: vec![1],
744 dtype: "f32".into(),
745 device: "cpu".into(),
746 duration_ns: 1,
747 timestamp_ns: 1,
748 output_dense_bytes: Some(4),
749 input_dense_bytes: 0,
750 }],
751 tensors: vec![],
752 tensor_stats: vec![],
753 memory: vec![],
754 device_memory: vec![],
755 device_intervals: vec![],
756 gradients: vec![],
757 edges: vec![],
758 terminal: TerminalEvent {
759 outcome: RunOutcome::Failed,
760 timestamp_ns: 2,
761 reason: Some("interrupted".into()),
762 },
763 };
764 let packet =
765 EvidencePacket::from_document(document, NsightEvidence::unavailable("not captured"))
766 .unwrap();
767 assert_eq!(
768 packet.capabilities.operation_coverage.level,
769 CapabilityLevel::Partial
770 );
771 assert_eq!(
772 packet.capabilities.structural_trace.source,
773 format!("{TRACE_SCHEMA} validation")
774 );
775 assert!(packet.findings.is_empty());
776 assert!(packet.graph.is_none());
777 }
778
779 #[test]
780 fn tensor_and_gradient_capabilities_enforce_declared_coverage() {
781 let document = TraceDocument {
782 schema: TRACE_SCHEMA.into(),
783 run: TraceRunMeta {
784 run_id: "typed-coverage".into(),
785 correlation_id: "typed/coverage".into(),
786 entrypoint: "demo".into(),
787 phase: crate::ExecutionPhase::Train,
788 timestamp: "2026-08-19T00:00:00Z".into(),
789 capture_step: 1,
790 warmup_steps: 0,
791 device: "cpu".into(),
792 measured_region_device_synchronized: false,
793 timing_mode: TimingMode::Host,
794 capture_contract: CaptureContract {
795 measurement_scope: MeasurementScope::ProfiledWork,
796 tensors: CoverageLevel::Complete,
797 gradients: CoverageLevel::None,
798 ..CaptureContract::default()
799 },
800 comparison_identity: None,
801 tags: Default::default(),
802 candle_version: None,
803 },
804 spans: vec![SpanRecord {
805 id: "root".into(),
806 parent_id: None,
807 name: "demo".into(),
808 kind: SpanKind::Function,
809 measured: true,
810 start_ns: 0,
811 closed: true,
812 duration_ns: 1,
813 step: None,
814 }],
815 ops: vec![],
816 tensors: vec![],
817 tensor_stats: vec![],
818 memory: vec![],
819 device_memory: vec![],
820 device_intervals: vec![],
821 gradients: vec![GradientEvent {
822 event_id: "gradient-1".into(),
823 root: "parameters".into(),
824 key: "weight".into(),
825 state: GradientState::Present,
826 norm: Some(1.0),
827 }],
828 edges: vec![],
829 terminal: TerminalEvent {
830 outcome: RunOutcome::Complete,
831 timestamp_ns: 1,
832 reason: None,
833 },
834 };
835 let packet =
836 EvidencePacket::from_document(document, NsightEvidence::unavailable("not captured"))
837 .unwrap();
838 assert_eq!(
839 packet.capabilities.tensor_coverage.level,
840 CapabilityLevel::Invalid
841 );
842 assert_eq!(
843 packet.capabilities.gradient_coverage.level,
844 CapabilityLevel::Partial
845 );
846 }
847
848 #[test]
849 fn older_capability_packets_default_new_typed_fields() {
850 let json = serde_json::json!({
851 "structural_trace": CapabilityState::default(),
852 "outer_wall_time": CapabilityState::default(),
853 "nested_host_time": CapabilityState::default(),
854 "nested_device_time": CapabilityState::default(),
855 "operation_coverage": CapabilityState::default(),
856 "logical_memory_coverage": CapabilityState::default(),
857 "physical_memory_coverage": CapabilityState::default(),
858 "gpu_correlation": CapabilityState::default(),
859 "provenance_binding": CapabilityState::default()
860 });
861 let capabilities: EvidenceCapabilities = serde_json::from_value(json).unwrap();
862 assert_eq!(
863 capabilities.tensor_coverage.level,
864 CapabilityLevel::Unavailable
865 );
866 assert_eq!(
867 capabilities.gradient_coverage.level,
868 CapabilityLevel::Unavailable
869 );
870 }
871}