1use crate::lab::config::LabConfig;
33use crate::lab::runtime::{CrashpackLink, LabRuntime, SporkHarnessReport};
34use crate::lab::spork_harness::{ScenarioRunnerError, SporkScenarioConfig, SporkScenarioRunner};
35use crate::trace::{TraceBuffer, TraceBufferHandle, TraceEvent};
36use serde::{Deserialize, Serialize};
37use std::collections::BTreeMap;
38
39#[must_use]
43pub fn find_divergence(a: &[TraceEvent], b: &[TraceEvent]) -> Option<TraceDivergence> {
44 let a_events = a;
45 let b_events = b;
46
47 for (i, (a_event, b_event)) in a_events.iter().zip(b_events.iter()).enumerate() {
48 if !events_match(a_event, b_event) {
49 return Some(TraceDivergence {
50 position: i,
51 event_a: (*a_event).clone(),
52 event_b: (*b_event).clone(),
53 });
54 }
55 }
56
57 if a_events.len() != b_events.len() {
59 let position = a_events.len().min(b_events.len());
60 #[allow(clippy::map_unwrap_or)]
61 return Some(TraceDivergence {
62 position,
63 event_a: a_events
64 .get(position)
65 .map(|e| (*e).clone())
66 .unwrap_or_else(|| {
67 TraceEvent::user_trace(0, crate::types::Time::ZERO, "<end of trace A>")
68 }),
69 event_b: b_events
70 .get(position)
71 .map(|e| (*e).clone())
72 .unwrap_or_else(|| {
73 TraceEvent::user_trace(0, crate::types::Time::ZERO, "<end of trace B>")
74 }),
75 });
76 }
77
78 None
79}
80
81fn events_match(a: &TraceEvent, b: &TraceEvent) -> bool {
83 a.kind == b.kind && a.time == b.time && a.logical_time == b.logical_time && a.data == b.data
84}
85
86#[derive(Debug, Clone)]
88pub struct TraceDivergence {
89 pub position: usize,
91 pub event_a: TraceEvent,
93 pub event_b: TraceEvent,
95}
96
97impl std::fmt::Display for TraceDivergence {
98 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
99 write!(
100 f,
101 "Divergence at position {}:\n A: {}\n B: {}",
102 self.position, self.event_a, self.event_b
103 )
104 }
105}
106
107#[derive(Debug, Clone, PartialEq, Eq)]
109pub struct TraceSummary {
110 pub event_count: usize,
112 pub spawn_count: usize,
114 pub complete_count: usize,
116 pub cancel_count: usize,
118}
119
120impl TraceSummary {
121 #[must_use]
123 pub fn from_buffer(buffer: &TraceBuffer) -> Self {
124 use crate::trace::event::TraceEventKind;
125
126 let mut summary = Self {
127 event_count: 0,
128 spawn_count: 0,
129 complete_count: 0,
130 cancel_count: 0,
131 };
132
133 for event in buffer.iter() {
134 summary.event_count += 1;
135 match event.kind {
136 TraceEventKind::Spawn => summary.spawn_count += 1,
137 TraceEventKind::Complete => summary.complete_count += 1,
138 TraceEventKind::CancelRequest | TraceEventKind::CancelAck => {
139 summary.cancel_count += 1;
140 }
141 _ => {}
142 }
143 }
144
145 summary
146 }
147}
148
149#[derive(Debug)]
151pub struct ReplayValidation {
152 pub matched: bool,
154 pub original_certificate: u64,
156 pub replay_certificate: u64,
158 pub divergence: Option<TraceDivergence>,
160 pub original_steps: u64,
162 pub replay_steps: u64,
164}
165
166impl ReplayValidation {
167 #[must_use]
169 pub fn is_valid(&self) -> bool {
170 self.matched && self.divergence.is_none()
171 }
172}
173
174impl std::fmt::Display for ReplayValidation {
175 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176 if self.is_valid() {
177 write!(
178 f,
179 "Replay OK: {} steps, certificate {:#018x}",
180 self.replay_steps, self.replay_certificate
181 )
182 } else {
183 write!(f, "Replay DIVERGED:")?;
184 if self.original_certificate != self.replay_certificate {
185 write!(
186 f,
187 "\n Certificate mismatch: original={:#018x} replay={:#018x}",
188 self.original_certificate, self.replay_certificate
189 )?;
190 }
191 if let Some(ref div) = self.divergence {
192 write!(f, "\n {div}")?;
193 }
194 if self.original_steps != self.replay_steps {
195 write!(
196 f,
197 "\n Step count mismatch: original={} replay={}",
198 self.original_steps, self.replay_steps
199 )?;
200 }
201 Ok(())
202 }
203 }
204}
205
206pub fn validate_replay<F>(seed: u64, worker_count: usize, test: F) -> ReplayValidation
213where
214 F: Fn(&mut LabRuntime),
215{
216 let run = |s: u64| -> (u64, u64, TraceBufferHandle) {
217 let mut config = LabConfig::new(s);
218 config = config.worker_count(worker_count);
219 let mut runtime = LabRuntime::new(config);
220 test(&mut runtime);
221 let steps = runtime.steps();
222 let cert = runtime.certificate().hash();
223 let trace = runtime.trace().clone();
224 (steps, cert, trace)
225 };
226
227 let (steps_a, cert_a, trace_a) = run(seed);
228 let (steps_b, cert_b, trace_b) = run(seed);
229
230 let events_a = trace_a.snapshot();
231 let events_b = trace_b.snapshot();
232 let divergence = find_divergence(&events_a, &events_b);
233 let matched = cert_a == cert_b && steps_a == steps_b;
234
235 ReplayValidation {
236 matched,
237 original_certificate: cert_a,
238 replay_certificate: cert_b,
239 divergence,
240 original_steps: steps_a,
241 replay_steps: steps_b,
242 }
243}
244
245pub fn validate_replay_multi<F>(
247 seeds: &[u64],
248 worker_count: usize,
249 test: F,
250) -> Vec<ReplayValidation>
251where
252 F: Fn(&mut LabRuntime),
253{
254 seeds
255 .iter()
256 .map(|&seed| validate_replay(seed, worker_count, &test))
257 .collect()
258}
259
260#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct ExplorationRunSummary {
263 pub seed: u64,
265 pub schedule_hash: u64,
267 pub trace_fingerprint: u64,
269}
270
271#[derive(Debug, Clone, PartialEq, Eq)]
273pub struct ExplorationFingerprintClass {
274 pub trace_fingerprint: u64,
276 pub run_count: usize,
278 pub seeds: Vec<u64>,
280 pub schedule_hashes: Vec<u64>,
282}
283
284#[derive(Debug, Clone, PartialEq, Eq)]
286pub struct ExplorationReport {
287 pub runs: Vec<ExplorationRunSummary>,
289 pub fingerprint_classes: Vec<ExplorationFingerprintClass>,
291}
292
293impl ExplorationReport {
294 #[must_use]
296 pub fn unique_fingerprint_count(&self) -> usize {
297 self.fingerprint_classes.len()
298 }
299}
300
301#[derive(Debug, Clone, PartialEq, Eq)]
303pub struct SporkExplorationRunSummary {
304 pub seed: u64,
306 pub schedule_hash: u64,
308 pub trace_fingerprint: u64,
310 pub passed: bool,
312 pub crashpack_link: Option<CrashpackLink>,
314}
315
316#[derive(Debug, Clone, PartialEq, Eq)]
318pub struct SporkExplorationReport {
319 pub runs: Vec<SporkExplorationRunSummary>,
321 pub fingerprint_classes: Vec<ExplorationFingerprintClass>,
323}
324
325impl SporkExplorationReport {
326 #[must_use]
328 pub fn unique_fingerprint_count(&self) -> usize {
329 self.fingerprint_classes.len()
330 }
331
332 #[must_use]
334 pub fn failure_count(&self) -> usize {
335 self.runs.iter().filter(|run| !run.passed).count()
336 }
337
338 #[must_use]
340 pub fn all_failures_linked_to_crashpacks(&self) -> bool {
341 self.runs
342 .iter()
343 .filter(|run| !run.passed)
344 .all(|run| run.crashpack_link.is_some())
345 }
346}
347
348#[must_use]
350pub fn classify_fingerprint_classes(
351 runs: &[ExplorationRunSummary],
352) -> Vec<ExplorationFingerprintClass> {
353 let mut grouped: BTreeMap<u64, (usize, Vec<u64>, Vec<u64>)> = BTreeMap::new();
354
355 for run in runs {
356 let entry = grouped
357 .entry(run.trace_fingerprint)
358 .or_insert_with(|| (0, Vec::new(), Vec::new()));
359 entry.0 += 1;
360 entry.1.push(run.seed);
361 entry.2.push(run.schedule_hash);
362 }
363
364 grouped
365 .into_iter()
366 .map(
367 |(trace_fingerprint, (run_count, mut seeds, mut schedule_hashes))| {
368 seeds.sort_unstable();
369 seeds.dedup();
370 schedule_hashes.sort_unstable();
371 schedule_hashes.dedup();
372 ExplorationFingerprintClass {
373 trace_fingerprint,
374 run_count,
375 seeds,
376 schedule_hashes,
377 }
378 },
379 )
380 .collect()
381}
382
383pub fn explore_seed_space<F>(seeds: &[u64], worker_count: usize, test: F) -> ExplorationReport
388where
389 F: Fn(&mut LabRuntime),
390{
391 let mut runs: Vec<ExplorationRunSummary> = seeds
392 .iter()
393 .map(|&seed| {
394 let mut config = LabConfig::new(seed);
395 config = config.worker_count(worker_count);
396 let mut runtime = LabRuntime::new(config);
397 test(&mut runtime);
398
399 let trace_events = runtime.trace().snapshot();
400 let normalized = normalize_for_replay(&trace_events);
401 let trace_fingerprint =
402 crate::trace::canonicalize::trace_fingerprint(&normalized.normalized);
403
404 ExplorationRunSummary {
405 seed,
406 schedule_hash: runtime.certificate().hash(),
407 trace_fingerprint,
408 }
409 })
410 .collect();
411
412 runs.sort_by_key(|run| run.seed);
413 let fingerprint_classes = classify_fingerprint_classes(&runs);
414 ExplorationReport {
415 runs,
416 fingerprint_classes,
417 }
418}
419
420#[must_use]
422pub fn summarize_spork_reports(reports: &[SporkHarnessReport]) -> SporkExplorationReport {
423 let mut runs: Vec<SporkExplorationRunSummary> = reports
424 .iter()
425 .map(|report| {
426 let passed = report.passed();
427 SporkExplorationRunSummary {
428 seed: report.seed(),
429 schedule_hash: report.run.trace_certificate.schedule_hash,
430 trace_fingerprint: report.trace_fingerprint(),
431 passed,
432 crashpack_link: if passed {
433 None
434 } else {
435 report.crashpack_link()
436 },
437 }
438 })
439 .collect();
440
441 runs.sort_by_key(|run| (run.seed, run.schedule_hash, run.trace_fingerprint));
442
443 let class_input: Vec<ExplorationRunSummary> = runs
444 .iter()
445 .map(|run| ExplorationRunSummary {
446 seed: run.seed,
447 schedule_hash: run.schedule_hash,
448 trace_fingerprint: run.trace_fingerprint,
449 })
450 .collect();
451
452 SporkExplorationReport {
453 runs,
454 fingerprint_classes: classify_fingerprint_classes(&class_input),
455 }
456}
457
458pub fn explore_spork_seed_space<F>(seeds: &[u64], mut run_for_seed: F) -> SporkExplorationReport
464where
465 F: FnMut(u64) -> SporkHarnessReport,
466{
467 let reports: Vec<SporkHarnessReport> = seeds.iter().map(|&seed| run_for_seed(seed)).collect();
468 summarize_spork_reports(&reports)
469}
470
471pub fn explore_scenario_runner_seed_space(
478 runner: &SporkScenarioRunner,
479 scenario_id: &str,
480 base_config: &SporkScenarioConfig,
481 seeds: &[u64],
482) -> Result<SporkExplorationReport, ScenarioRunnerError> {
483 let mut reports = Vec::with_capacity(seeds.len());
484 for &seed in seeds {
485 let mut config = base_config.clone();
486 config.seed = seed;
487 let result = runner.run_with_config(scenario_id, Some(config))?;
488 reports.push(result.report);
489 }
490 Ok(summarize_spork_reports(&reports))
491}
492
493pub const DIVERGENCE_CORPUS_SCHEMA_VERSION: &str = "lab-live-divergence-corpus-v1";
495
496#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
498#[serde(rename_all = "snake_case")]
499pub enum DivergenceBundleLevel {
500 Full,
502 Reduced,
504}
505
506#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
508#[serde(rename_all = "snake_case")]
509pub enum DifferentialPolicyClass {
510 RuntimeSemanticBug,
512 LabModelOrMappingBug,
514 ArtifactSchemaViolation,
516 InsufficientObservability,
518 UnsupportedSurface,
520 SchedulerNoiseSuspected,
522 IrreproducibleDivergence,
524}
525
526impl DifferentialPolicyClass {
527 #[must_use]
529 pub fn as_str(self) -> &'static str {
530 match self {
531 Self::RuntimeSemanticBug => "runtime_semantic_bug",
532 Self::LabModelOrMappingBug => "lab_model_or_mapping_bug",
533 Self::ArtifactSchemaViolation => "artifact_schema_violation",
534 Self::InsufficientObservability => "insufficient_observability",
535 Self::UnsupportedSurface => "unsupported_surface",
536 Self::SchedulerNoiseSuspected => "scheduler_noise_suspected",
537 Self::IrreproducibleDivergence => "irreproducible_divergence",
538 }
539 }
540
541 #[must_use]
543 pub fn bundle_level(self) -> DivergenceBundleLevel {
544 match self {
545 Self::RuntimeSemanticBug
546 | Self::LabModelOrMappingBug
547 | Self::ArtifactSchemaViolation
548 | Self::IrreproducibleDivergence => DivergenceBundleLevel::Full,
549 Self::InsufficientObservability
550 | Self::UnsupportedSurface
551 | Self::SchedulerNoiseSuspected => DivergenceBundleLevel::Reduced,
552 }
553 }
554}
555
556impl std::fmt::Display for DifferentialPolicyClass {
557 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
558 f.write_str(self.as_str())
559 }
560}
561
562#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
564#[serde(rename_all = "snake_case")]
565pub enum RegressionPromotionState {
566 Investigating,
568 Minimized,
570 PromotedRegression,
572 KnownOpen,
574 Rejected,
576}
577
578#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
580#[serde(rename_all = "snake_case")]
581pub enum DivergenceShrinkStatus {
582 NotRequested,
584 Pending,
586 PreservedSemanticClass,
588 Rejected,
590}
591
592#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
594pub struct DivergenceArtifactBundle {
595 pub bundle_root: String,
597 pub differential_summary_path: String,
599 pub differential_event_log_path: String,
601 pub differential_failures_path: String,
603 pub differential_deviations_path: String,
605 pub differential_repro_manifest_path: String,
607 pub lab_normalized_path: String,
609 pub live_normalized_path: String,
611}
612
613impl DivergenceArtifactBundle {
614 #[must_use]
616 pub fn under(root: impl Into<String>) -> Self {
617 let bundle_root = root.into().trim_end_matches('/').to_string();
618 let join = |name: &str| format!("{bundle_root}/{name}");
619 Self {
620 bundle_root: bundle_root.clone(),
621 differential_summary_path: join("differential_summary.json"),
622 differential_event_log_path: join("differential_event_log.jsonl"),
623 differential_failures_path: join("differential_failures.json"),
624 differential_deviations_path: join("differential_deviations.json"),
625 differential_repro_manifest_path: join("differential_repro_manifest.json"),
626 lab_normalized_path: join("lab_normalized.json"),
627 live_normalized_path: join("live_normalized.json"),
628 }
629 }
630}
631
632#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
634pub struct DivergenceRetentionMetadata {
635 pub bundle_level: DivergenceBundleLevel,
637 pub local_retention_days: u16,
639 pub ci_retention_days: u16,
641 pub redaction_mode: String,
643}
644
645impl DivergenceRetentionMetadata {
646 #[must_use]
648 pub fn for_policy_class(policy_class: DifferentialPolicyClass) -> Self {
649 Self {
650 bundle_level: policy_class.bundle_level(),
651 local_retention_days: 14,
652 ci_retention_days: 30,
653 redaction_mode: "metadata_only".to_string(),
654 }
655 }
656}
657
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
660pub struct DivergenceFirstSeenContext {
661 pub runner_profile: String,
663 pub attempt_index: u32,
665 pub rerun_count: u32,
667}
668
669#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
671pub struct DivergenceMinimizationLineage {
672 pub original_seed: u64,
674 #[serde(skip_serializing_if = "Option::is_none")]
676 pub minimized_seed: Option<u64>,
677 #[serde(skip_serializing_if = "Option::is_none")]
679 pub shrinker: Option<String>,
680 pub shrink_status: DivergenceShrinkStatus,
682 pub preserved_divergence_class: bool,
684 pub preserved_policy_class: bool,
686}
687
688impl DivergenceMinimizationLineage {
689 #[must_use]
691 pub fn from_seed_lineage(lineage: &crate::lab::dual_run::SeedLineageRecord) -> Self {
692 Self {
693 original_seed: lineage.canonical_seed,
694 minimized_seed: None,
695 shrinker: None,
696 shrink_status: DivergenceShrinkStatus::NotRequested,
697 preserved_divergence_class: true,
698 preserved_policy_class: true,
699 }
700 }
701
702 #[must_use]
704 pub fn with_minimized_seed(
705 mut self,
706 seed: u64,
707 shrinker: impl Into<String>,
708 preserved_divergence_class: bool,
709 preserved_policy_class: bool,
710 ) -> Self {
711 self.minimized_seed = Some(seed);
712 self.shrinker = Some(shrinker.into());
713 self.shrink_status = if preserved_divergence_class && preserved_policy_class {
714 DivergenceShrinkStatus::PreservedSemanticClass
715 } else {
716 DivergenceShrinkStatus::Rejected
717 };
718 self.preserved_divergence_class = preserved_divergence_class;
719 self.preserved_policy_class = preserved_policy_class;
720 self
721 }
722}
723
724#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
726pub struct DivergenceCorpusEntry {
727 pub schema_version: String,
729 pub entry_id: String,
731 pub scenario_id: String,
733 pub surface_id: String,
735 pub surface_contract_version: String,
737 pub divergence_class: String,
739 pub policy_class: DifferentialPolicyClass,
741 pub first_seen: DivergenceFirstSeenContext,
743 pub seed_lineage: crate::lab::dual_run::SeedLineageRecord,
745 #[serde(default, skip_serializing_if = "Vec::is_empty")]
747 pub mismatch_fields: Vec<String>,
748 pub artifact_bundle: DivergenceArtifactBundle,
750 pub minimization_lineage: DivergenceMinimizationLineage,
752 pub regression_promotion_state: RegressionPromotionState,
754 pub retention: DivergenceRetentionMetadata,
756 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
758 pub metadata: BTreeMap<String, String>,
759}
760
761impl DivergenceCorpusEntry {
762 #[must_use]
764 pub fn from_dual_run_result(
765 result: &crate::lab::dual_run::DualRunResult,
766 runner_profile: impl Into<String>,
767 divergence_class: impl Into<String>,
768 policy_class: DifferentialPolicyClass,
769 bundle_root: impl Into<String>,
770 ) -> Self {
771 let seed_lineage = result.seed_lineage.clone();
772 let entry_id = Self::entry_id_for(
773 &result.verdict.scenario_id,
774 &seed_lineage.seed_lineage_id,
775 policy_class,
776 );
777 let mut mismatch_fields: Vec<String> = result
778 .verdict
779 .mismatches
780 .iter()
781 .map(|mismatch| mismatch.field.clone())
782 .collect();
783 mismatch_fields.sort_unstable();
784 mismatch_fields.dedup();
785
786 let mut metadata = BTreeMap::new();
787 if let Some(path) = result.lab.provenance.artifact_path.as_deref() {
788 metadata.insert("lab_artifact_path".to_string(), path.to_string());
789 }
790 if let Some(path) = result.live.provenance.artifact_path.as_deref() {
791 metadata.insert("live_artifact_path".to_string(), path.to_string());
792 }
793 if let Some(cmd) = result.lab.provenance.repro_command.as_deref() {
794 metadata.insert("lab_repro_command".to_string(), cmd.to_string());
795 }
796 if let Some(cmd) = result.live.provenance.repro_command.as_deref() {
797 metadata.insert("live_repro_command".to_string(), cmd.to_string());
798 }
799 if !result.lab_invariant_violations.is_empty() {
800 metadata.insert(
801 "lab_invariant_violations".to_string(),
802 result.lab_invariant_violations.join(","),
803 );
804 }
805 if !result.live_invariant_violations.is_empty() {
806 metadata.insert(
807 "live_invariant_violations".to_string(),
808 result.live_invariant_violations.join(","),
809 );
810 }
811
812 Self {
813 schema_version: DIVERGENCE_CORPUS_SCHEMA_VERSION.to_string(),
814 entry_id,
815 scenario_id: result.verdict.scenario_id.clone(),
816 surface_id: result.verdict.surface_id.clone(),
817 surface_contract_version: result.lab.surface_contract_version.clone(),
818 divergence_class: divergence_class.into(),
819 policy_class,
820 first_seen: DivergenceFirstSeenContext {
821 runner_profile: runner_profile.into(),
822 attempt_index: 0,
823 rerun_count: 0,
824 },
825 seed_lineage: seed_lineage.clone(),
826 mismatch_fields,
827 artifact_bundle: DivergenceArtifactBundle::under(bundle_root),
828 minimization_lineage: DivergenceMinimizationLineage::from_seed_lineage(&seed_lineage),
829 regression_promotion_state: RegressionPromotionState::Investigating,
830 retention: DivergenceRetentionMetadata::for_policy_class(policy_class),
831 metadata,
832 }
833 }
834
835 #[must_use]
837 pub fn entry_id_for(
838 scenario_id: &str,
839 seed_lineage_id: &str,
840 policy_class: DifferentialPolicyClass,
841 ) -> String {
842 format!(
843 "{}.{}.{}",
844 sanitize_registry_component(scenario_id),
845 sanitize_registry_component(seed_lineage_id),
846 policy_class.as_str()
847 )
848 }
849
850 #[must_use]
852 pub fn default_bundle_root(&self) -> String {
853 format!(
854 "artifacts/differential/{}/{}/{}",
855 sanitize_registry_component(&self.surface_id),
856 sanitize_registry_component(&self.scenario_id),
857 sanitize_registry_component(&self.seed_lineage.seed_lineage_id)
858 )
859 }
860
861 #[must_use]
863 pub fn with_first_seen_attempt(mut self, attempt_index: u32, rerun_count: u32) -> Self {
864 self.first_seen.attempt_index = attempt_index;
865 self.first_seen.rerun_count = rerun_count;
866 self
867 }
868
869 #[must_use]
871 pub fn with_minimization_lineage(mut self, lineage: DivergenceMinimizationLineage) -> Self {
872 self.minimization_lineage = lineage;
873 self.regression_promotion_state = if self.minimization_lineage.minimized_seed.is_some() {
874 RegressionPromotionState::Minimized
875 } else {
876 self.regression_promotion_state
877 };
878 self
879 }
880
881 #[must_use]
883 pub fn promote_to_regression(mut self, promoted_scenario_id: impl Into<String>) -> Self {
884 self.regression_promotion_state = RegressionPromotionState::PromotedRegression;
885 self.metadata.insert(
886 "promoted_scenario_id".to_string(),
887 promoted_scenario_id.into(),
888 );
889 self
890 }
891}
892
893#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
895pub struct DivergenceCorpusRegistry {
896 pub schema_version: String,
898 pub entries: Vec<DivergenceCorpusEntry>,
900}
901
902impl DivergenceCorpusRegistry {
903 #[must_use]
905 pub fn new() -> Self {
906 Self {
907 schema_version: DIVERGENCE_CORPUS_SCHEMA_VERSION.to_string(),
908 entries: Vec::new(),
909 }
910 }
911
912 pub fn upsert(&mut self, entry: DivergenceCorpusEntry) {
914 if let Some(existing) = self
915 .entries
916 .iter_mut()
917 .find(|existing| existing.entry_id == entry.entry_id)
918 {
919 *existing = entry;
920 } else {
921 self.entries.push(entry);
922 self.entries
923 .sort_by(|left, right| left.entry_id.cmp(&right.entry_id));
924 }
925 }
926}
927
928impl Default for DivergenceCorpusRegistry {
929 fn default() -> Self {
930 Self::new()
931 }
932}
933
934pub const DIFFERENTIAL_SUMMARY_SCHEMA_VERSION: &str = "lab-live-differential-summary-v1";
936pub const DIFFERENTIAL_FAILURES_SCHEMA_VERSION: &str = "lab-live-differential-failures-v1";
938pub const DIFFERENTIAL_DEVIATIONS_SCHEMA_VERSION: &str = "lab-live-differential-deviations-v1";
940pub const DIFFERENTIAL_REPRO_MANIFEST_SCHEMA_VERSION: &str =
942 "lab-live-differential-repro-manifest-v1";
943
944#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
946pub struct DifferentialCrashpackReference {
947 pub path: String,
949 pub id: String,
951 pub fingerprint: u64,
953 pub replay_command: String,
955}
956
957impl DifferentialCrashpackReference {
958 #[must_use]
960 pub fn from_runtime_link(link: &CrashpackLink) -> Self {
961 Self {
962 path: link.path.clone(),
963 id: link.id.clone(),
964 fingerprint: link.fingerprint,
965 replay_command: link.replay.command_line.clone(),
966 }
967 }
968
969 #[must_use]
972 pub fn from_provenance(provenance: &crate::lab::dual_run::ReplayMetadata) -> Option<Self> {
973 let path = provenance.artifact_path.as_ref()?;
974 let file_name = path.rsplit('/').next().unwrap_or(path);
975 if !file_name.contains("crashpack") {
976 return None;
977 }
978 let fingerprint = provenance.trace_fingerprint?;
979 Some(Self {
980 path: path.clone(),
981 id: format!(
982 "crashpack-{:016x}-{:016x}",
983 provenance.effective_seed, fingerprint
984 ),
985 fingerprint,
986 replay_command: provenance
987 .repro_command
988 .clone()
989 .unwrap_or_else(|| provenance.default_repro_command()),
990 })
991 }
992}
993
994#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
996pub struct DifferentialFailureArtifact {
997 pub runtime_kind: String,
999 pub normalized_record_path: String,
1001 #[serde(skip_serializing_if = "Option::is_none")]
1003 pub artifact_path: Option<String>,
1004 pub repro_command: String,
1006 #[serde(skip_serializing_if = "Option::is_none")]
1008 pub crashpack_link: Option<DifferentialCrashpackReference>,
1009 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1011 pub invariant_violations: Vec<String>,
1012}
1013
1014impl DifferentialFailureArtifact {
1015 #[must_use]
1016 fn from_observable(
1017 observable: &crate::lab::dual_run::NormalizedObservable,
1018 normalized_record_path: impl Into<String>,
1019 invariant_violations: &[String],
1020 ) -> Self {
1021 let repro_command = observable
1022 .provenance
1023 .repro_command
1024 .clone()
1025 .unwrap_or_else(|| observable.provenance.default_repro_command());
1026
1027 Self {
1028 runtime_kind: observable.runtime_kind.to_string(),
1029 normalized_record_path: normalized_record_path.into(),
1030 artifact_path: observable.provenance.artifact_path.clone(),
1031 repro_command,
1032 crashpack_link: DifferentialCrashpackReference::from_provenance(&observable.provenance),
1033 invariant_violations: invariant_violations.to_vec(),
1034 }
1035 }
1036}
1037
1038#[derive(Debug, Clone, Serialize, Deserialize)]
1040pub struct DifferentialSummaryDocument {
1041 pub schema_version: String,
1043 pub entry_id: String,
1045 pub scenario_id: String,
1047 pub surface_id: String,
1049 pub surface_contract_version: String,
1051 pub verdict_summary: String,
1053 pub policy_summary: String,
1055 pub divergence_class: String,
1057 pub policy_class: DifferentialPolicyClass,
1059 pub regression_promotion_state: RegressionPromotionState,
1061 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1063 pub mismatch_fields: Vec<String>,
1064 pub mismatch_count: usize,
1066 pub passed: bool,
1068 pub lab_invariant_violation_count: usize,
1070 pub live_invariant_violation_count: usize,
1072 pub bundle_level: DivergenceBundleLevel,
1074 pub bundle_root: String,
1076}
1077
1078#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1080pub struct DifferentialFailuresDocument {
1081 pub schema_version: String,
1083 pub entry_id: String,
1085 pub scenario_id: String,
1087 pub surface_id: String,
1089 pub failure_artifacts: Vec<DifferentialFailureArtifact>,
1091}
1092
1093#[derive(Debug, Clone, Serialize, Deserialize)]
1095pub struct DifferentialDeviationsDocument {
1096 pub schema_version: String,
1098 pub entry_id: String,
1100 pub scenario_id: String,
1102 pub surface_id: String,
1104 pub policy_summary: String,
1106 pub provisional_class: String,
1108 #[serde(skip_serializing_if = "Option::is_none")]
1110 pub suggested_final_class: Option<String>,
1111 pub explanation: String,
1113 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1115 pub mismatches: Vec<crate::lab::dual_run::SemanticMismatch>,
1116 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1118 pub lab_invariant_violations: Vec<String>,
1119 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1121 pub live_invariant_violations: Vec<String>,
1122}
1123
1124#[derive(Debug, Clone, Serialize, Deserialize)]
1126pub struct DifferentialReproManifest {
1127 pub schema_version: String,
1129 pub entry_id: String,
1131 pub scenario_id: String,
1133 pub surface_id: String,
1135 pub surface_contract_version: String,
1137 pub divergence_class: String,
1139 pub policy_class: DifferentialPolicyClass,
1141 pub regression_promotion_state: RegressionPromotionState,
1143 pub rerun_decision: crate::lab::dual_run::RerunDecision,
1145 pub first_seen: DivergenceFirstSeenContext,
1147 pub seed_lineage: crate::lab::dual_run::SeedLineageRecord,
1149 pub minimization_lineage: DivergenceMinimizationLineage,
1151 pub bundle_root: String,
1153 pub summary_path: String,
1155 pub deviations_path: String,
1157 pub failure_artifacts_path: String,
1159 pub lab_normalized_path: String,
1161 pub live_normalized_path: String,
1163 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1165 pub repro_commands: Vec<String>,
1166 #[serde(skip_serializing_if = "Option::is_none")]
1168 pub promoted_scenario_id: Option<String>,
1169}
1170
1171#[derive(Debug, Clone, Serialize, Deserialize)]
1173pub struct DifferentialBundleArtifacts {
1174 pub summary: DifferentialSummaryDocument,
1176 pub failures: DifferentialFailuresDocument,
1178 pub deviations: DifferentialDeviationsDocument,
1180 pub repro_manifest: DifferentialReproManifest,
1182 pub lab_normalized: crate::lab::dual_run::NormalizedObservable,
1184 pub live_normalized: crate::lab::dual_run::NormalizedObservable,
1186}
1187
1188impl DifferentialBundleArtifacts {
1189 #[must_use]
1192 pub fn from_dual_run_result(
1193 entry: &DivergenceCorpusEntry,
1194 result: &crate::lab::dual_run::DualRunResult,
1195 ) -> Self {
1196 let failure_artifacts = vec![
1197 DifferentialFailureArtifact::from_observable(
1198 &result.lab,
1199 entry.artifact_bundle.lab_normalized_path.clone(),
1200 &result.lab_invariant_violations,
1201 ),
1202 DifferentialFailureArtifact::from_observable(
1203 &result.live,
1204 entry.artifact_bundle.live_normalized_path.clone(),
1205 &result.live_invariant_violations,
1206 ),
1207 ];
1208 let mut repro_commands: Vec<String> = failure_artifacts
1209 .iter()
1210 .map(|artifact| artifact.repro_command.clone())
1211 .collect();
1212 repro_commands.sort_unstable();
1213 repro_commands.dedup();
1214
1215 let summary = DifferentialSummaryDocument {
1216 schema_version: DIFFERENTIAL_SUMMARY_SCHEMA_VERSION.to_string(),
1217 entry_id: entry.entry_id.clone(),
1218 scenario_id: entry.scenario_id.clone(),
1219 surface_id: entry.surface_id.clone(),
1220 surface_contract_version: entry.surface_contract_version.clone(),
1221 verdict_summary: result.verdict.summary(),
1222 policy_summary: result.policy.summary(),
1223 divergence_class: entry.divergence_class.clone(),
1224 policy_class: entry.policy_class,
1225 regression_promotion_state: entry.regression_promotion_state,
1226 mismatch_fields: entry.mismatch_fields.clone(),
1227 mismatch_count: entry.mismatch_fields.len(),
1228 passed: result.passed(),
1229 lab_invariant_violation_count: result.lab_invariant_violations.len(),
1230 live_invariant_violation_count: result.live_invariant_violations.len(),
1231 bundle_level: entry.retention.bundle_level,
1232 bundle_root: entry.artifact_bundle.bundle_root.clone(),
1233 };
1234
1235 let failures = DifferentialFailuresDocument {
1236 schema_version: DIFFERENTIAL_FAILURES_SCHEMA_VERSION.to_string(),
1237 entry_id: entry.entry_id.clone(),
1238 scenario_id: entry.scenario_id.clone(),
1239 surface_id: entry.surface_id.clone(),
1240 failure_artifacts,
1241 };
1242
1243 let deviations = DifferentialDeviationsDocument {
1244 schema_version: DIFFERENTIAL_DEVIATIONS_SCHEMA_VERSION.to_string(),
1245 entry_id: entry.entry_id.clone(),
1246 scenario_id: entry.scenario_id.clone(),
1247 surface_id: entry.surface_id.clone(),
1248 policy_summary: result.policy.summary(),
1249 provisional_class: result.policy.provisional_class.to_string(),
1250 suggested_final_class: result
1251 .policy
1252 .suggested_final_class
1253 .map(|class| class.to_string()),
1254 explanation: result.policy.explanation.clone(),
1255 mismatches: result.verdict.mismatches.clone(),
1256 lab_invariant_violations: result.lab_invariant_violations.clone(),
1257 live_invariant_violations: result.live_invariant_violations.clone(),
1258 };
1259
1260 let repro_manifest = DifferentialReproManifest {
1261 schema_version: DIFFERENTIAL_REPRO_MANIFEST_SCHEMA_VERSION.to_string(),
1262 entry_id: entry.entry_id.clone(),
1263 scenario_id: entry.scenario_id.clone(),
1264 surface_id: entry.surface_id.clone(),
1265 surface_contract_version: entry.surface_contract_version.clone(),
1266 divergence_class: entry.divergence_class.clone(),
1267 policy_class: entry.policy_class,
1268 regression_promotion_state: entry.regression_promotion_state,
1269 rerun_decision: result.policy.rerun_decision,
1270 first_seen: entry.first_seen.clone(),
1271 seed_lineage: entry.seed_lineage.clone(),
1272 minimization_lineage: entry.minimization_lineage.clone(),
1273 bundle_root: entry.artifact_bundle.bundle_root.clone(),
1274 summary_path: entry.artifact_bundle.differential_summary_path.clone(),
1275 deviations_path: entry.artifact_bundle.differential_deviations_path.clone(),
1276 failure_artifacts_path: entry.artifact_bundle.differential_failures_path.clone(),
1277 lab_normalized_path: entry.artifact_bundle.lab_normalized_path.clone(),
1278 live_normalized_path: entry.artifact_bundle.live_normalized_path.clone(),
1279 repro_commands,
1280 promoted_scenario_id: entry.metadata.get("promoted_scenario_id").cloned(),
1281 };
1282
1283 Self {
1284 summary,
1285 failures,
1286 deviations,
1287 repro_manifest,
1288 lab_normalized: result.lab.clone(),
1289 live_normalized: result.live.clone(),
1290 }
1291 }
1292}
1293
1294fn sanitize_registry_component(input: &str) -> String {
1295 let mut out = String::with_capacity(input.len());
1296 for ch in input.chars() {
1297 if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' {
1298 out.push(ch);
1299 } else {
1300 out.push('_');
1301 }
1302 }
1303 out.trim_matches('_').to_string()
1304}
1305
1306#[derive(Debug, Clone)]
1312pub struct NormalizationResult {
1313 pub normalized: Vec<TraceEvent>,
1315 pub original_switches: usize,
1317 pub normalized_switches: usize,
1319 pub algorithm: String,
1321}
1322
1323impl NormalizationResult {
1324 #[must_use]
1326 pub fn switch_reduction(&self) -> usize {
1327 self.original_switches
1328 .saturating_sub(self.normalized_switches)
1329 }
1330
1331 #[must_use]
1333 #[allow(clippy::cast_precision_loss)]
1334 pub fn switch_reduction_pct(&self) -> f64 {
1335 if self.original_switches == 0 {
1336 0.0
1337 } else {
1338 (self.switch_reduction() as f64 / self.original_switches as f64) * 100.0
1339 }
1340 }
1341}
1342
1343impl std::fmt::Display for NormalizationResult {
1344 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1345 write!(
1346 f,
1347 "Normalized {} events: {} → {} switches ({:.1}% reduction, {})",
1348 self.normalized.len(),
1349 self.original_switches,
1350 self.normalized_switches,
1351 self.switch_reduction_pct(),
1352 self.algorithm
1353 )
1354 }
1355}
1356
1357#[must_use]
1376pub fn normalize_for_replay(events: &[TraceEvent]) -> NormalizationResult {
1377 normalize_for_replay_with_config(events, &crate::trace::GeodesicConfig::default())
1378}
1379
1380#[must_use]
1387pub fn normalize_for_replay_with_config(
1388 events: &[TraceEvent],
1389 config: &crate::trace::GeodesicConfig,
1390) -> NormalizationResult {
1391 let original_switches = crate::trace::trace_switch_cost(events);
1392 let (normalized, geodesic_result) = crate::trace::normalize_trace(events, config);
1393
1394 NormalizationResult {
1395 normalized,
1396 original_switches,
1397 normalized_switches: geodesic_result.switch_count,
1398 algorithm: format!("{:?}", geodesic_result.algorithm),
1399 }
1400}
1401
1402#[must_use]
1410pub fn compare_normalized(a: &[TraceEvent], b: &[TraceEvent]) -> Option<TraceDivergence> {
1411 let norm_a = normalize_for_replay(a);
1412 let norm_b = normalize_for_replay(b);
1413 find_divergence(&norm_a.normalized, &norm_b.normalized)
1414}
1415
1416#[must_use]
1420pub fn traces_equivalent(a: &[TraceEvent], b: &[TraceEvent]) -> bool {
1421 compare_normalized(a, b).is_none()
1422}
1423
1424#[cfg(test)]
1425mod tests {
1426 use super::*;
1427 use crate::app::AppSpec;
1428 use crate::lab::SporkScenarioSpec;
1429 use crate::trace::event::{TraceData, TraceEventKind};
1430 use crate::types::Budget;
1431 use crate::types::Time;
1432
1433 fn init_test(name: &str) {
1434 crate::test_utils::init_test_logging();
1435 crate::test_phase!(name);
1436 }
1437
1438 fn trace_message_contains(event: &TraceEvent, needle: &str) -> bool {
1439 matches!(&event.data, TraceData::Message(message) if message.contains(needle))
1440 }
1441
1442 #[test]
1443 fn identical_traces_no_divergence() {
1444 init_test("identical_traces_no_divergence");
1445 let a = vec![TraceEvent::new(
1446 1,
1447 Time::ZERO,
1448 TraceEventKind::UserTrace,
1449 TraceData::None,
1450 )];
1451 let b = vec![TraceEvent::new(
1452 1,
1453 Time::ZERO,
1454 TraceEventKind::UserTrace,
1455 TraceData::None,
1456 )];
1457
1458 let div = find_divergence(&a, &b);
1459 let ok = div.is_none();
1460 crate::assert_with_log!(ok, "no divergence", true, ok);
1461 crate::test_complete!("identical_traces_no_divergence");
1462 }
1463
1464 #[test]
1465 fn trace_seq_only_difference_no_divergence() {
1466 init_test("trace_seq_only_difference_no_divergence");
1467 let a = vec![TraceEvent::new(
1468 1,
1469 Time::ZERO,
1470 TraceEventKind::UserTrace,
1471 TraceData::Message("same".to_string()),
1472 )];
1473 let b = vec![TraceEvent::new(
1474 99,
1475 Time::ZERO,
1476 TraceEventKind::UserTrace,
1477 TraceData::Message("same".to_string()),
1478 )];
1479
1480 let div = find_divergence(&a, &b);
1481 let ok = div.is_none();
1482 crate::assert_with_log!(ok, "seq-only differences ignored", true, ok);
1483 crate::test_complete!("trace_seq_only_difference_no_divergence");
1484 }
1485
1486 #[test]
1487 fn different_traces_find_divergence() {
1488 init_test("different_traces_find_divergence");
1489 let a = vec![TraceEvent::new(
1490 1,
1491 Time::ZERO,
1492 TraceEventKind::Spawn,
1493 TraceData::None,
1494 )];
1495 let b = vec![TraceEvent::new(
1496 1,
1497 Time::ZERO,
1498 TraceEventKind::Complete,
1499 TraceData::None,
1500 )];
1501
1502 let div = find_divergence(&a, &b);
1503 let some = div.is_some();
1504 crate::assert_with_log!(some, "divergence", true, some);
1505 let pos = div.expect("divergence").position;
1506 crate::assert_with_log!(pos == 0, "position", 0, pos);
1507 crate::test_complete!("different_traces_find_divergence");
1508 }
1509
1510 #[test]
1511 fn different_traces_find_divergence_data() {
1512 init_test("different_traces_find_divergence_data");
1513 let a = vec![TraceEvent::new(
1514 1,
1515 Time::ZERO,
1516 TraceEventKind::UserTrace,
1517 TraceData::Message("a".to_string()),
1518 )];
1519 let b = vec![TraceEvent::new(
1520 1,
1521 Time::ZERO,
1522 TraceEventKind::UserTrace,
1523 TraceData::Message("b".to_string()),
1524 )];
1525
1526 let div = find_divergence(&a, &b);
1527 let some = div.is_some();
1528 crate::assert_with_log!(some, "divergence", true, some);
1529 let pos = div.expect("divergence").position;
1530 crate::assert_with_log!(pos == 0, "position", 0, pos);
1531 crate::test_complete!("different_traces_find_divergence_data");
1532 }
1533
1534 #[test]
1537 fn replay_single_task_deterministic() {
1538 use crate::types::Budget;
1539 let validation = validate_replay(42, 1, |runtime| {
1540 let region = runtime.state.create_root_region(Budget::INFINITE);
1541 let (t, _) = runtime
1542 .state
1543 .create_task(region, Budget::INFINITE, async { 1 })
1544 .expect("t");
1545 runtime.scheduler.lock().schedule(t, 0);
1546 runtime.run_until_quiescent();
1547 });
1548
1549 assert!(validation.is_valid(), "Replay failed: {validation}");
1550 assert_eq!(
1551 validation.original_certificate,
1552 validation.replay_certificate
1553 );
1554 assert_eq!(validation.original_steps, validation.replay_steps);
1555 }
1556
1557 #[test]
1558 fn replay_two_tasks_deterministic() {
1559 use crate::types::Budget;
1560 let validation = validate_replay(0, 1, |runtime| {
1561 let region = runtime.state.create_root_region(Budget::INFINITE);
1562 let (t1, _) = runtime
1563 .state
1564 .create_task(region, Budget::INFINITE, async {})
1565 .expect("t1");
1566 let (t2, _) = runtime
1567 .state
1568 .create_task(region, Budget::INFINITE, async {})
1569 .expect("t2");
1570 {
1571 let mut sched = runtime.scheduler.lock();
1572 sched.schedule(t1, 0);
1573 sched.schedule(t2, 0);
1574 }
1575 runtime.run_until_quiescent();
1576 });
1577
1578 assert!(validation.is_valid(), "Replay failed: {validation}");
1579 }
1580
1581 #[test]
1582 fn replay_multi_seeds_all_deterministic() {
1583 use crate::types::Budget;
1584 let seeds: Vec<u64> = (0..10).collect();
1585 let results = validate_replay_multi(&seeds, 1, |runtime| {
1586 let region = runtime.state.create_root_region(Budget::INFINITE);
1587 let (t, _) = runtime
1588 .state
1589 .create_task(region, Budget::INFINITE, async { 42 })
1590 .expect("t");
1591 runtime.scheduler.lock().schedule(t, 0);
1592 runtime.run_until_quiescent();
1593 });
1594
1595 for (i, v) in results.iter().enumerate() {
1596 assert!(v.is_valid(), "Seed {} replay failed: {v}", seeds[i]);
1597 }
1598 }
1599
1600 #[test]
1601 fn replay_validation_display_ok() {
1602 let v = ReplayValidation {
1603 matched: true,
1604 original_certificate: 0x1234,
1605 replay_certificate: 0x1234,
1606 divergence: None,
1607 original_steps: 5,
1608 replay_steps: 5,
1609 };
1610 let s = format!("{v}");
1611 assert!(s.contains("Replay OK"));
1612 }
1613
1614 #[test]
1615 fn replay_validation_display_diverged() {
1616 let v = ReplayValidation {
1617 matched: false,
1618 original_certificate: 0x1234,
1619 replay_certificate: 0x5678,
1620 divergence: None,
1621 original_steps: 5,
1622 replay_steps: 5,
1623 };
1624 let s = format!("{v}");
1625 assert!(s.contains("DIVERGED"));
1626 assert!(s.contains("Certificate mismatch"));
1627 }
1628
1629 #[test]
1632 fn normalization_single_owner_no_switches() {
1633 init_test("normalization_single_owner_no_switches");
1634 let events = vec![
1636 TraceEvent::new(
1637 1,
1638 Time::from_nanos(0),
1639 TraceEventKind::Spawn,
1640 TraceData::None,
1641 ),
1642 TraceEvent::new(
1643 2,
1644 Time::from_nanos(1),
1645 TraceEventKind::Poll,
1646 TraceData::None,
1647 ),
1648 TraceEvent::new(
1649 3,
1650 Time::from_nanos(2),
1651 TraceEventKind::Complete,
1652 TraceData::None,
1653 ),
1654 ];
1655 let result = normalize_for_replay(&events);
1659 assert_eq!(result.switch_reduction(), 0);
1661 crate::test_complete!("normalization_single_owner_no_switches");
1662 }
1663
1664 #[test]
1665 fn normalization_result_display() {
1666 init_test("normalization_result_display");
1667 let result = NormalizationResult {
1668 normalized: vec![],
1669 original_switches: 10,
1670 normalized_switches: 3,
1671 algorithm: "Greedy".to_string(),
1672 };
1673
1674 let display = format!("{result}");
1675 assert!(display.contains("10 → 3 switches"));
1676 assert!(display.contains("70.0% reduction"));
1677 assert!(display.contains("Greedy"));
1678 crate::test_complete!("normalization_result_display");
1679 }
1680
1681 #[test]
1682 fn normalization_result_zero_switches() {
1683 init_test("normalization_result_zero_switches");
1684 let result = NormalizationResult {
1685 normalized: vec![],
1686 original_switches: 0,
1687 normalized_switches: 0,
1688 algorithm: "Trivial".to_string(),
1689 };
1690
1691 let pct = result.switch_reduction_pct();
1693 assert!((pct - 0.0).abs() < f64::EPSILON);
1694 crate::test_complete!("normalization_result_zero_switches");
1695 }
1696
1697 #[test]
1698 fn traces_equivalent_identical() {
1699 init_test("traces_equivalent_identical");
1700 let events = vec![
1701 TraceEvent::new(
1702 1,
1703 Time::from_nanos(0),
1704 TraceEventKind::Spawn,
1705 TraceData::None,
1706 ),
1707 TraceEvent::new(
1708 2,
1709 Time::from_nanos(1),
1710 TraceEventKind::Complete,
1711 TraceData::None,
1712 ),
1713 ];
1714
1715 let equivalent = traces_equivalent(&events, &events);
1716 crate::assert_with_log!(equivalent, "identical traces equivalent", true, equivalent);
1717 crate::test_complete!("traces_equivalent_identical");
1718 }
1719
1720 #[test]
1721 fn traces_equivalent_ignores_sequence_numbers() {
1722 init_test("traces_equivalent_ignores_sequence_numbers");
1723 let a = vec![TraceEvent::new(
1724 1,
1725 Time::from_nanos(0),
1726 TraceEventKind::Spawn,
1727 TraceData::None,
1728 )];
1729 let b = vec![TraceEvent::new(
1730 42,
1731 Time::from_nanos(0),
1732 TraceEventKind::Spawn,
1733 TraceData::None,
1734 )];
1735
1736 let equivalent = traces_equivalent(&a, &b);
1737 crate::assert_with_log!(
1738 equivalent,
1739 "seq-only differences still equivalent",
1740 true,
1741 equivalent
1742 );
1743 crate::test_complete!("traces_equivalent_ignores_sequence_numbers");
1744 }
1745
1746 #[test]
1747 fn traces_equivalent_different_kinds() {
1748 init_test("traces_equivalent_different_kinds");
1749 let a = vec![TraceEvent::new(
1750 1,
1751 Time::from_nanos(0),
1752 TraceEventKind::Spawn,
1753 TraceData::None,
1754 )];
1755 let b = vec![TraceEvent::new(
1756 1,
1757 Time::from_nanos(0),
1758 TraceEventKind::Complete,
1759 TraceData::None,
1760 )];
1761
1762 let equivalent = traces_equivalent(&a, &b);
1763 crate::assert_with_log!(
1764 !equivalent,
1765 "different kinds not equivalent",
1766 false,
1767 equivalent
1768 );
1769 crate::test_complete!("traces_equivalent_different_kinds");
1770 }
1771
1772 #[test]
1773 fn compare_normalized_returns_divergence() {
1774 init_test("compare_normalized_returns_divergence");
1775 let a = vec![TraceEvent::new(
1776 1,
1777 Time::from_nanos(0),
1778 TraceEventKind::Spawn,
1779 TraceData::None,
1780 )];
1781 let b = vec![TraceEvent::new(
1782 1,
1783 Time::from_nanos(0),
1784 TraceEventKind::Complete,
1785 TraceData::None,
1786 )];
1787
1788 let divergence = compare_normalized(&a, &b);
1789 let has_div = divergence.is_some();
1790 crate::assert_with_log!(has_div, "divergence found", true, has_div);
1791 crate::test_complete!("compare_normalized_returns_divergence");
1792 }
1793
1794 #[test]
1795 fn normalize_with_config_custom_beam() {
1796 use crate::trace::GeodesicConfig;
1797
1798 init_test("normalize_with_config_custom_beam");
1799 let events = vec![
1800 TraceEvent::new(
1801 1,
1802 Time::from_nanos(0),
1803 TraceEventKind::Spawn,
1804 TraceData::None,
1805 ),
1806 TraceEvent::new(
1807 2,
1808 Time::from_nanos(1),
1809 TraceEventKind::Poll,
1810 TraceData::None,
1811 ),
1812 ];
1813
1814 let config = GeodesicConfig {
1815 exact_threshold: 0,
1816 beam_threshold: 1,
1817 beam_width: 4,
1818 step_budget: 100,
1819 };
1820
1821 let result = normalize_for_replay_with_config(&events, &config);
1822 assert!(!result.algorithm.is_empty());
1824 crate::test_complete!("normalize_with_config_custom_beam");
1825 }
1826
1827 #[test]
1828 fn classify_fingerprint_classes_is_deterministic() {
1829 init_test("classify_fingerprint_classes_is_deterministic");
1830
1831 let runs = vec![
1832 ExplorationRunSummary {
1833 seed: 9,
1834 schedule_hash: 0xB,
1835 trace_fingerprint: 0xAA,
1836 },
1837 ExplorationRunSummary {
1838 seed: 3,
1839 schedule_hash: 0xA,
1840 trace_fingerprint: 0xBB,
1841 },
1842 ExplorationRunSummary {
1843 seed: 7,
1844 schedule_hash: 0xC,
1845 trace_fingerprint: 0xAA,
1846 },
1847 ExplorationRunSummary {
1848 seed: 7,
1849 schedule_hash: 0xC,
1850 trace_fingerprint: 0xAA,
1851 },
1852 ];
1853
1854 let classes = classify_fingerprint_classes(&runs);
1855 assert_eq!(classes.len(), 2);
1856 assert_eq!(classes[0].trace_fingerprint, 0xAA);
1857 assert_eq!(classes[0].run_count, 3);
1858 assert_eq!(classes[0].seeds, vec![7, 9]);
1859 assert_eq!(classes[0].schedule_hashes, vec![0xB, 0xC]);
1860 assert_eq!(classes[1].trace_fingerprint, 0xBB);
1861 assert_eq!(classes[1].run_count, 1);
1862 assert_eq!(classes[1].seeds, vec![3]);
1863 assert_eq!(classes[1].schedule_hashes, vec![0xA]);
1864
1865 crate::test_complete!("classify_fingerprint_classes_is_deterministic");
1866 }
1867
1868 #[test]
1869 fn explore_seed_space_is_deterministic_for_same_inputs() {
1870 init_test("explore_seed_space_is_deterministic_for_same_inputs");
1871
1872 let seeds = [11_u64, 13_u64, 11_u64];
1873 let scenario = |runtime: &mut LabRuntime| {
1874 let region = runtime.state.create_root_region(Budget::INFINITE);
1875 let (task, _) = runtime
1876 .state
1877 .create_task(region, Budget::INFINITE, async {})
1878 .expect("task");
1879 runtime.scheduler.lock().schedule(task, 0);
1880 runtime.run_until_quiescent();
1881 };
1882
1883 let a = explore_seed_space(&seeds, 1, scenario);
1884 let b = explore_seed_space(&seeds, 1, scenario);
1885
1886 assert_eq!(a, b, "same seeds and scenario must produce same report");
1887 assert_eq!(a.runs.len(), seeds.len());
1888 assert!(a.unique_fingerprint_count() >= 1);
1889
1890 crate::test_complete!("explore_seed_space_is_deterministic_for_same_inputs");
1891 }
1892
1893 fn make_spork_report(seed: u64, failing: bool) -> SporkHarnessReport {
1894 use crate::record::ObligationKind;
1895
1896 let config = LabConfig::new(seed).panic_on_leak(false);
1897 let mut runtime = LabRuntime::new(config);
1898 let region = runtime.state.create_root_region(Budget::INFINITE);
1899 let (task, _) = runtime
1900 .state
1901 .create_task(region, Budget::INFINITE, async {})
1902 .expect("create task");
1903 runtime.scheduler.lock().schedule(task, 0);
1904 if failing {
1908 runtime
1909 .state
1910 .create_obligation(
1911 ObligationKind::SendPermit,
1912 task,
1913 region,
1914 Some("intentional failure for exploration".to_string()),
1915 )
1916 .expect("create failing obligation");
1917 }
1918 runtime.run_until_quiescent();
1919
1920 runtime.spork_report("spork_exploration", Vec::new())
1921 }
1922
1923 #[test]
1924 fn summarize_spork_reports_links_failures_to_crashpacks() {
1925 init_test("summarize_spork_reports_links_failures_to_crashpacks");
1926
1927 let passing = make_spork_report(31, false);
1928 let failing = make_spork_report(32, true);
1929
1930 let summary = summarize_spork_reports(&[failing, passing]);
1931 assert_eq!(summary.runs.len(), 2);
1932 assert_eq!(summary.failure_count(), 1);
1933 assert!(summary.unique_fingerprint_count() >= 1);
1934 assert!(
1935 summary.all_failures_linked_to_crashpacks(),
1936 "failed runs must include crashpack linkage metadata"
1937 );
1938
1939 let failed_run = summary
1940 .runs
1941 .iter()
1942 .find(|run| !run.passed)
1943 .expect("one failing run expected");
1944 let crashpack = failed_run
1945 .crashpack_link
1946 .as_ref()
1947 .expect("failing run should have crashpack link");
1948 assert!(
1949 crashpack.path.starts_with("crashpack-"),
1950 "unexpected crashpack path: {}",
1951 crashpack.path
1952 );
1953
1954 crate::test_complete!("summarize_spork_reports_links_failures_to_crashpacks");
1955 }
1956
1957 #[test]
1958 fn explore_spork_seed_space_is_deterministic() {
1959 init_test("explore_spork_seed_space_is_deterministic");
1960
1961 let seeds = [42_u64, 41_u64, 42_u64];
1962
1963 let run_for_seed = |seed: u64| make_spork_report(seed, seed.is_multiple_of(2));
1964 let a = explore_spork_seed_space(&seeds, run_for_seed);
1965
1966 let run_for_seed = |seed: u64| make_spork_report(seed, seed.is_multiple_of(2));
1967 let b = explore_spork_seed_space(&seeds, run_for_seed);
1968
1969 assert_eq!(a, b, "same seeds must produce deterministic report");
1970 assert_eq!(a.runs.len(), seeds.len());
1971 assert_eq!(a.failure_count(), 2);
1972 assert!(a.unique_fingerprint_count() >= 1);
1973 assert!(a.all_failures_linked_to_crashpacks());
1974
1975 crate::test_complete!("explore_spork_seed_space_is_deterministic");
1976 }
1977
1978 #[test]
1979 fn scenario_runner_exploration_has_deterministic_fingerprints() {
1980 init_test("scenario_runner_exploration_has_deterministic_fingerprints");
1981
1982 let mut runner = SporkScenarioRunner::new();
1983 runner
1984 .register(
1985 SporkScenarioSpec::new("replay.scenario", |_| AppSpec::new("replay_app"))
1986 .with_default_config(SporkScenarioConfig::default()),
1987 )
1988 .expect("register scenario");
1989
1990 let base_config = SporkScenarioConfig::default();
1991 let seeds = [12_u64, 13_u64, 12_u64];
1992
1993 let a =
1994 explore_scenario_runner_seed_space(&runner, "replay.scenario", &base_config, &seeds)
1995 .expect("exploration A");
1996 let b =
1997 explore_scenario_runner_seed_space(&runner, "replay.scenario", &base_config, &seeds)
1998 .expect("exploration B");
1999
2000 assert_eq!(a, b, "scenario exploration must be deterministic");
2001 assert_eq!(a.runs.len(), seeds.len());
2002 assert!(a.unique_fingerprint_count() >= 1);
2003
2004 let seed_12: Vec<_> = a.runs.iter().filter(|run| run.seed == 12).collect();
2006 assert_eq!(seed_12.len(), 2);
2007 assert_eq!(seed_12[0].trace_fingerprint, seed_12[1].trace_fingerprint);
2008
2009 crate::test_complete!("scenario_runner_exploration_has_deterministic_fingerprints");
2010 }
2011
2012 fn make_dual_run_divergence_result() -> crate::lab::dual_run::DualRunResult {
2013 use crate::lab::dual_run::{
2014 CancellationRecord, DualRunHarness, LoserDrainRecord, ObligationBalanceRecord,
2015 RegionCloseRecord, ResourceSurfaceRecord, TerminalOutcome,
2016 };
2017
2018 fn base_semantics() -> crate::lab::dual_run::NormalizedSemantics {
2019 crate::lab::dual_run::NormalizedSemantics {
2020 terminal_outcome: TerminalOutcome::ok(),
2021 cancellation: CancellationRecord::none(),
2022 loser_drain: LoserDrainRecord::not_applicable(),
2023 region_close: RegionCloseRecord::quiescent(),
2024 obligation_balance: ObligationBalanceRecord::zero(),
2025 resource_surface: ResourceSurfaceRecord::empty("test.surface"),
2026 }
2027 }
2028
2029 let mut result = DualRunHarness::phase1(
2030 "divergence.registry.case",
2031 "test.surface",
2032 "v1",
2033 "Divergence corpus registry coverage",
2034 0xD1,
2035 )
2036 .lab(|_config| base_semantics())
2037 .live(|_seed, _entropy| {
2038 let mut sem = base_semantics();
2039 sem.obligation_balance = ObligationBalanceRecord {
2040 reserved: 1,
2041 committed: 0,
2042 aborted: 0,
2043 leaked: 1,
2044 unresolved: 0,
2045 balanced: false,
2046 };
2047 sem
2048 })
2049 .run();
2050
2051 let mut lab_provenance = result
2052 .lab
2053 .provenance
2054 .clone()
2055 .with_artifact_path("crashpack-divergence.registry.case.json")
2056 .with_repro_command("cargo test divergence.registry.case -- --nocapture");
2057 if lab_provenance.trace_fingerprint.is_none() {
2058 lab_provenance.trace_fingerprint = Some(0xC0DE_CAFE);
2059 }
2060 result.lab.provenance = lab_provenance;
2061
2062 let mut live_provenance = result
2063 .live
2064 .provenance
2065 .clone()
2066 .with_artifact_path("artifacts/live/divergence.registry.case.json")
2067 .with_repro_command("cargo test divergence.registry.case -- --nocapture --live");
2068 if live_provenance.trace_fingerprint.is_none() {
2069 live_provenance.trace_fingerprint = Some(0xBEEF_BAAD);
2070 }
2071 result.live.provenance = live_provenance;
2072 result
2073 }
2074
2075 #[test]
2076 fn divergence_artifact_bundle_uses_stable_bundle_layout() {
2077 init_test("divergence_artifact_bundle_uses_stable_bundle_layout");
2078
2079 let bundle = DivergenceArtifactBundle::under("artifacts/differential/run-001");
2080 assert_eq!(
2081 bundle.differential_summary_path,
2082 "artifacts/differential/run-001/differential_summary.json"
2083 );
2084 assert_eq!(
2085 bundle.live_normalized_path,
2086 "artifacts/differential/run-001/live_normalized.json"
2087 );
2088
2089 crate::test_complete!("divergence_artifact_bundle_uses_stable_bundle_layout");
2090 }
2091
2092 #[test]
2093 fn divergence_retention_defaults_follow_policy_class() {
2094 init_test("divergence_retention_defaults_follow_policy_class");
2095
2096 let full = DivergenceRetentionMetadata::for_policy_class(
2097 DifferentialPolicyClass::RuntimeSemanticBug,
2098 );
2099 assert_eq!(full.bundle_level, DivergenceBundleLevel::Full);
2100 assert_eq!(full.local_retention_days, 14);
2101 assert_eq!(full.ci_retention_days, 30);
2102 assert_eq!(full.redaction_mode, "metadata_only");
2103
2104 let reduced = DivergenceRetentionMetadata::for_policy_class(
2105 DifferentialPolicyClass::UnsupportedSurface,
2106 );
2107 assert_eq!(reduced.bundle_level, DivergenceBundleLevel::Reduced);
2108
2109 crate::test_complete!("divergence_retention_defaults_follow_policy_class");
2110 }
2111
2112 #[test]
2113 fn divergence_corpus_entry_tracks_lineage_and_promotion_state() {
2114 init_test("divergence_corpus_entry_tracks_lineage_and_promotion_state");
2115
2116 let result = make_dual_run_divergence_result();
2117 assert!(!result.passed(), "test fixture must produce a divergence");
2118
2119 let entry = DivergenceCorpusEntry::from_dual_run_result(
2120 &result,
2121 "pilot_surface",
2122 "obligation_balance_mismatch",
2123 DifferentialPolicyClass::RuntimeSemanticBug,
2124 "artifacts/differential/test-run",
2125 )
2126 .with_first_seen_attempt(2, 1)
2127 .with_minimization_lineage(
2128 DivergenceMinimizationLineage::from_seed_lineage(&result.seed_lineage)
2129 .with_minimized_seed(0x2A, "prefix_shrinker", true, true),
2130 )
2131 .promote_to_regression("regression.test.surface.obligation_leak.seed_2a");
2132
2133 assert_eq!(
2134 entry.policy_class,
2135 DifferentialPolicyClass::RuntimeSemanticBug
2136 );
2137 assert_eq!(entry.first_seen.runner_profile, "pilot_surface");
2138 assert_eq!(entry.first_seen.attempt_index, 2);
2139 assert_eq!(entry.first_seen.rerun_count, 1);
2140 assert_eq!(
2141 entry.minimization_lineage.shrink_status,
2142 DivergenceShrinkStatus::PreservedSemanticClass
2143 );
2144 assert_eq!(
2145 entry.regression_promotion_state,
2146 RegressionPromotionState::PromotedRegression
2147 );
2148 assert_eq!(
2149 entry.metadata.get("promoted_scenario_id"),
2150 Some(&"regression.test.surface.obligation_leak.seed_2a".to_string())
2151 );
2152 assert!(
2153 entry
2154 .mismatch_fields
2155 .contains(&"semantics.obligation_balance.balanced".to_string()),
2156 "mismatch fields should retain the semantic mismatch path"
2157 );
2158 assert!(
2159 entry
2160 .artifact_bundle
2161 .differential_repro_manifest_path
2162 .ends_with("differential_repro_manifest.json")
2163 );
2164 assert_eq!(
2165 entry.artifact_bundle.bundle_root,
2166 "artifacts/differential/test-run"
2167 );
2168
2169 crate::test_complete!("divergence_corpus_entry_tracks_lineage_and_promotion_state");
2170 }
2171
2172 #[test]
2173 fn divergence_registry_upsert_is_deterministic() {
2174 init_test("divergence_registry_upsert_is_deterministic");
2175
2176 let result = make_dual_run_divergence_result();
2177 let entry = DivergenceCorpusEntry::from_dual_run_result(
2178 &result,
2179 "nightly",
2180 "obligation_balance_mismatch",
2181 DifferentialPolicyClass::RuntimeSemanticBug,
2182 "artifacts/differential/nightly-case",
2183 );
2184
2185 let mut registry = DivergenceCorpusRegistry::new();
2186 registry.upsert(entry.clone());
2187 registry.upsert(entry.promote_to_regression("regression.promoted"));
2188
2189 assert_eq!(registry.schema_version, DIVERGENCE_CORPUS_SCHEMA_VERSION);
2190 assert_eq!(registry.entries.len(), 1);
2191 assert_eq!(
2192 registry.entries[0].regression_promotion_state,
2193 RegressionPromotionState::PromotedRegression
2194 );
2195
2196 crate::test_complete!("divergence_registry_upsert_is_deterministic");
2197 }
2198
2199 #[test]
2200 fn differential_bundle_artifacts_capture_repro_and_minimization_lineage() {
2201 init_test("differential_bundle_artifacts_capture_repro_and_minimization_lineage");
2202
2203 let result = make_dual_run_divergence_result();
2204 let entry = DivergenceCorpusEntry::from_dual_run_result(
2205 &result,
2206 "nightly",
2207 "obligation_balance_mismatch",
2208 DifferentialPolicyClass::RuntimeSemanticBug,
2209 "artifacts/differential/nightly/divergence.registry.case",
2210 )
2211 .with_first_seen_attempt(3, 2)
2212 .with_minimization_lineage(
2213 DivergenceMinimizationLineage::from_seed_lineage(&result.seed_lineage)
2214 .with_minimized_seed(0x2A, "prefix_shrinker", true, true),
2215 )
2216 .promote_to_regression("regression.test.surface.obligation_leak.seed_2a");
2217
2218 let bundle = DifferentialBundleArtifacts::from_dual_run_result(&entry, &result);
2219 assert_eq!(
2220 bundle.summary.schema_version,
2221 DIFFERENTIAL_SUMMARY_SCHEMA_VERSION
2222 );
2223 assert_eq!(
2224 bundle.summary.bundle_root,
2225 "artifacts/differential/nightly/divergence.registry.case"
2226 );
2227 assert_eq!(bundle.failures.failure_artifacts.len(), 2);
2228 assert_eq!(
2229 bundle.failures.failure_artifacts[0].runtime_kind,
2230 "lab".to_string()
2231 );
2232 assert_eq!(
2233 bundle.failures.failure_artifacts[0]
2234 .crashpack_link
2235 .as_ref()
2236 .map(|link| link.path.as_str()),
2237 Some("crashpack-divergence.registry.case.json")
2238 );
2239 assert_eq!(
2240 bundle.repro_manifest.promoted_scenario_id.as_deref(),
2241 Some("regression.test.surface.obligation_leak.seed_2a")
2242 );
2243 assert_eq!(
2244 bundle.repro_manifest.minimization_lineage.shrink_status,
2245 DivergenceShrinkStatus::PreservedSemanticClass
2246 );
2247 assert_eq!(
2248 bundle.repro_manifest.failure_artifacts_path,
2249 "artifacts/differential/nightly/divergence.registry.case/differential_failures.json"
2250 );
2251 assert!(
2252 bundle
2253 .repro_manifest
2254 .repro_commands
2255 .contains(&"cargo test divergence.registry.case -- --nocapture".to_string())
2256 );
2257 assert!(
2258 bundle
2259 .deviations
2260 .mismatches
2261 .iter()
2262 .any(|mismatch| mismatch.field == "semantics.obligation_balance.balanced")
2263 );
2264
2265 crate::test_complete!(
2266 "differential_bundle_artifacts_capture_repro_and_minimization_lineage"
2267 );
2268 }
2269
2270 #[test]
2271 fn inferred_crashpack_reference_requires_crashpack_like_path() {
2272 init_test("inferred_crashpack_reference_requires_crashpack_like_path");
2273
2274 let result = make_dual_run_divergence_result();
2275 let lab_link = DifferentialCrashpackReference::from_provenance(&result.lab.provenance);
2276 let live_link = DifferentialCrashpackReference::from_provenance(&result.live.provenance);
2277
2278 assert!(
2279 lab_link.is_some(),
2280 "crashpack-like lab artifact should infer linkage"
2281 );
2282 assert!(
2283 live_link.is_none(),
2284 "non-crashpack live artifact should not infer crashpack linkage"
2285 );
2286
2287 crate::test_complete!("inferred_crashpack_reference_requires_crashpack_like_path");
2288 }
2289
2290 #[derive(Debug, Clone)]
2296 struct ReplayMetamorphicConfig {
2297 worker_count: usize,
2299 checkpoint_count: usize,
2301 task_count: usize,
2303 }
2304
2305 impl Default for ReplayMetamorphicConfig {
2306 fn default() -> Self {
2307 Self {
2308 worker_count: 4,
2309 checkpoint_count: 5,
2310 task_count: 8,
2311 }
2312 }
2313 }
2314
2315 fn create_fork_join_test_scenario(
2317 config: &ReplayMetamorphicConfig,
2318 rng_seed: u64,
2319 ) -> impl Fn(&mut LabRuntime) + Clone {
2320 let task_count = config.task_count;
2321 move |runtime: &mut LabRuntime| {
2322 use crate::util::det_rng::DetRng;
2324 let mut rng = DetRng::new(rng_seed);
2325
2326 for i in 0..task_count {
2328 let _task_seed = rng.next_u64();
2329 runtime.trace().record_event(|id| {
2332 crate::trace::TraceEvent::user_trace(
2333 id,
2334 runtime.now(),
2335 format!("fork_task_{}", i),
2336 )
2337 });
2338 }
2339
2340 for i in 0..task_count {
2342 runtime.trace().record_event(|id| {
2343 crate::trace::TraceEvent::user_trace(
2344 id,
2345 runtime.now(),
2346 format!("join_task_{}", i),
2347 )
2348 });
2349 }
2350 }
2351 }
2352
2353 #[test]
2358 fn metamorphic_checkpoint_replay_equivalence() {
2359 init_test("metamorphic_checkpoint_replay_equivalence");
2360
2361 let seed = std::time::SystemTime::now()
2362 .duration_since(std::time::UNIX_EPOCH)
2363 .unwrap()
2364 .as_nanos() as u64;
2365
2366 let config = ReplayMetamorphicConfig::default();
2367
2368 let test_scenario = create_fork_join_test_scenario(&config, seed);
2370
2371 let mut original_config = LabConfig::new(seed);
2373 original_config = original_config.worker_count(config.worker_count);
2374 let mut original_runtime = LabRuntime::new(original_config);
2375 test_scenario(&mut original_runtime);
2376 let original_trace = original_runtime.trace().snapshot();
2377 let original_certificate = original_runtime.certificate().hash();
2378
2379 for checkpoint_idx in 0..config.checkpoint_count.min(original_trace.len()) {
2382 let mut replay_config = LabConfig::new(seed);
2383 replay_config = replay_config.worker_count(config.worker_count);
2384 let mut replay_runtime = LabRuntime::new(replay_config);
2385
2386 for event in &original_trace[..checkpoint_idx] {
2388 replay_runtime.trace().push_event(event.clone());
2389 }
2390
2391 test_scenario(&mut replay_runtime);
2393 let replay_trace = replay_runtime.trace().snapshot();
2394 let replay_certificate = replay_runtime.certificate().hash();
2395
2396 assert_eq!(
2398 original_certificate, replay_certificate,
2399 "Checkpoint {} replay diverged in certificate hash",
2400 checkpoint_idx
2401 );
2402
2403 if replay_trace.len() >= original_trace.len() {
2406 for (i, (orig_event, replay_event)) in
2407 original_trace.iter().zip(replay_trace.iter()).enumerate()
2408 {
2409 if i >= checkpoint_idx {
2410 assert!(
2411 events_match(orig_event, replay_event),
2412 "Event {} after checkpoint {} doesn't match: {:?} vs {:?}",
2413 i,
2414 checkpoint_idx,
2415 orig_event,
2416 replay_event
2417 );
2418 }
2419 }
2420 }
2421 }
2422
2423 crate::test_complete!("metamorphic_checkpoint_replay_equivalence");
2424 }
2425
2426 #[test]
2431 fn metamorphic_parallel_scope_fork_join_determinism() {
2432 init_test("metamorphic_parallel_scope_fork_join_determinism");
2433
2434 let seed = std::time::SystemTime::now()
2435 .duration_since(std::time::UNIX_EPOCH)
2436 .unwrap()
2437 .as_nanos() as u64;
2438
2439 let config = ReplayMetamorphicConfig::default();
2440
2441 let test_scenario = create_fork_join_test_scenario(&config, seed);
2443
2444 let mut executions = Vec::new();
2445
2446 for _run_idx in 0..5 {
2448 let mut runtime_config = LabConfig::new(seed); runtime_config = runtime_config.worker_count(config.worker_count);
2450 let mut runtime = LabRuntime::new(runtime_config);
2451
2452 test_scenario(&mut runtime);
2453
2454 let trace = runtime.trace().snapshot();
2455 let certificate = runtime.certificate().hash();
2456 let steps = runtime.steps();
2457
2458 executions.push((trace, certificate, steps));
2459 }
2460
2461 for (run_idx, (trace, certificate, steps)) in executions.iter().enumerate().skip(1) {
2463 assert_eq!(
2464 executions[0].1, *certificate,
2465 "Run {} has different certificate than run 0",
2466 run_idx
2467 );
2468 assert_eq!(
2469 executions[0].2, *steps,
2470 "Run {} has different step count than run 0",
2471 run_idx
2472 );
2473
2474 let divergence = find_divergence(&executions[0].0, trace);
2476 assert!(
2477 divergence.is_none(),
2478 "Run {} diverged from run 0: {:?}",
2479 run_idx,
2480 divergence
2481 );
2482 }
2483
2484 for (run_idx, (trace, _, _)) in executions.iter().enumerate() {
2486 let mut fork_events = Vec::new();
2487 let mut join_events = Vec::new();
2488
2489 for event in trace {
2490 if matches!(&event.data, crate::trace::event::TraceData::Message(msg) if msg.contains("fork_task_"))
2491 {
2492 fork_events.push(event.clone());
2493 } else if matches!(&event.data, crate::trace::event::TraceData::Message(msg) if msg.contains("join_task_"))
2494 {
2495 join_events.push(event.clone());
2496 }
2497 }
2498
2499 if let (Some(last_fork), Some(first_join)) = (fork_events.last(), join_events.first()) {
2501 assert!(
2502 last_fork.time <= first_join.time,
2503 "Run {}: Fork events should complete before join events start",
2504 run_idx
2505 );
2506 }
2507 }
2508
2509 crate::test_complete!("metamorphic_parallel_scope_fork_join_determinism");
2510 }
2511
2512 #[test]
2517 fn metamorphic_panic_replay_cause_chain_consistency() {
2518 init_test("metamorphic_panic_replay_cause_chain_consistency");
2519
2520 let seed = std::time::SystemTime::now()
2521 .duration_since(std::time::UNIX_EPOCH)
2522 .unwrap()
2523 .as_nanos() as u64;
2524
2525 let config = ReplayMetamorphicConfig::default();
2526
2527 let panic_scenario = move |runtime: &mut LabRuntime| {
2529 use crate::util::det_rng::DetRng;
2530 let mut rng = DetRng::new(seed);
2531
2532 for i in 0..config.task_count {
2534 if rng.next_u64() % 4 == 0 {
2535 runtime.trace().record_event(|id| {
2537 crate::trace::TraceEvent::user_trace(
2538 id,
2539 runtime.now(),
2540 format!("panic_task_{}", i),
2541 )
2542 });
2543 } else {
2544 runtime.trace().record_event(|id| {
2545 crate::trace::TraceEvent::user_trace(
2546 id,
2547 runtime.now(),
2548 format!("normal_task_{}", i),
2549 )
2550 });
2551 }
2552 }
2553 };
2554
2555 let mut original_config = LabConfig::new(seed);
2557 original_config = original_config.worker_count(config.worker_count);
2558 let mut original_runtime = LabRuntime::new(original_config);
2559 panic_scenario(&mut original_runtime);
2560 let original_trace = original_runtime.trace().snapshot();
2561
2562 let mut replay_config = LabConfig::new(seed);
2564 replay_config = replay_config.worker_count(config.worker_count);
2565 let mut replay_runtime = LabRuntime::new(replay_config);
2566 panic_scenario(&mut replay_runtime);
2567 let replay_trace = replay_runtime.trace().snapshot();
2568
2569 let original_panics: Vec<_> = original_trace
2571 .iter()
2572 .filter(|event| trace_message_contains(event, "panic_"))
2573 .collect();
2574 let replay_panics: Vec<_> = replay_trace
2575 .iter()
2576 .filter(|event| trace_message_contains(event, "panic_"))
2577 .collect();
2578
2579 assert_eq!(
2580 original_panics.len(),
2581 replay_panics.len(),
2582 "Panic count should match between original and replay"
2583 );
2584
2585 for (original_panic, replay_panic) in original_panics.iter().zip(replay_panics.iter()) {
2586 assert!(
2587 events_match(original_panic, replay_panic),
2588 "Panic events should match: {:?} vs {:?}",
2589 original_panic,
2590 replay_panic
2591 );
2592 }
2593
2594 let divergence = find_divergence(&original_trace, &replay_trace);
2596 assert!(
2597 divergence.is_none(),
2598 "Panic replay diverged: {:?}",
2599 divergence
2600 );
2601
2602 crate::test_complete!("metamorphic_panic_replay_cause_chain_consistency");
2603 }
2604
2605 #[test]
2610 fn metamorphic_cross_region_trace_ordering_preservation() {
2611 init_test("metamorphic_cross_region_trace_ordering_preservation");
2612
2613 let seed = std::time::SystemTime::now()
2614 .duration_since(std::time::UNIX_EPOCH)
2615 .unwrap()
2616 .as_nanos() as u64;
2617
2618 let config = ReplayMetamorphicConfig::default();
2619
2620 let multi_region_scenario = move |runtime: &mut LabRuntime| {
2622 use crate::util::det_rng::DetRng;
2623 let _rng = DetRng::new(seed);
2624
2625 let region_count = 3;
2626
2627 for region_id in 0..region_count {
2629 for task_id in 0..config.task_count / region_count {
2630 let now = runtime.now();
2631 runtime.trace().record_event(|id| {
2632 crate::trace::TraceEvent::user_trace(
2633 id,
2634 now,
2635 format!("region_{}_task_{}", region_id, task_id),
2636 )
2637 });
2638 }
2639 }
2640 };
2641
2642 let execution_contexts = [
2644 ("single_worker", 1),
2645 ("dual_worker", 2),
2646 ("multi_worker", 4),
2647 ];
2648
2649 let mut context_traces = Vec::new();
2650
2651 for (context_name, worker_count) in &execution_contexts {
2652 let mut runtime_config = LabConfig::new(seed);
2653 runtime_config = runtime_config.worker_count(*worker_count);
2654 let mut runtime = LabRuntime::new(runtime_config);
2655
2656 multi_region_scenario(&mut runtime);
2657 let trace = runtime.trace().snapshot();
2658 context_traces.push((context_name, trace));
2659 }
2660
2661 for (context_name, trace) in &context_traces {
2663 let mut region_events: std::collections::BTreeMap<u32, Vec<&crate::trace::TraceEvent>> =
2664 std::collections::BTreeMap::new();
2665
2666 for event in trace {
2667 if let crate::trace::event::TraceData::Message(ref data_str) = event.data {
2668 if data_str.contains("region_") {
2669 if let Some(region_start) = data_str.find("region_") {
2670 if let Some(region_end) = data_str[region_start + 7..].find('_') {
2671 if let Ok(region_id) = data_str
2672 [region_start + 7..region_start + 7 + region_end]
2673 .parse::<u32>()
2674 {
2675 region_events.entry(region_id).or_default().push(event);
2676 }
2677 }
2678 }
2679 }
2680 }
2681 }
2682
2683 assert!(
2685 !region_events.is_empty(),
2686 "Context {} should have region events",
2687 context_name
2688 );
2689
2690 for (region_id, events) in ®ion_events {
2692 for window in events.windows(2) {
2693 assert!(
2694 window[0].time <= window[1].time,
2695 "Context {}: Region {} events not in time order",
2696 context_name,
2697 region_id
2698 );
2699 }
2700 }
2701 }
2702
2703 for i in 1..context_traces.len() {
2706 let (name1, trace1) = &context_traces[0];
2707 let (name2, trace2) = &context_traces[i];
2708
2709 let logical_order1: Vec<_> = trace1
2711 .iter()
2712 .filter(|e| trace_message_contains(e, "region_"))
2713 .map(|e| &e.data)
2714 .collect();
2715 let logical_order2: Vec<_> = trace2
2716 .iter()
2717 .filter(|e| trace_message_contains(e, "region_"))
2718 .map(|e| &e.data)
2719 .collect();
2720
2721 assert_eq!(
2722 logical_order1, logical_order2,
2723 "Logical ordering differs between {} and {}",
2724 name1, name2
2725 );
2726 }
2727
2728 crate::test_complete!("metamorphic_cross_region_trace_ordering_preservation");
2729 }
2730
2731 #[test]
2736 fn metamorphic_lab_runtime_seed_determinism() {
2737 init_test("metamorphic_lab_runtime_seed_determinism");
2738
2739 const SEED: u64 = 0x1234_5678_9ABC_DEF0;
2741
2742 let config = ReplayMetamorphicConfig::default();
2743
2744 let deterministic_scenario = |runtime: &mut LabRuntime| {
2745 use crate::util::det_rng::DetRng;
2746 let mut rng = DetRng::new(SEED); for i in 0..config.task_count {
2750 let choice = rng.next_u64() % 3;
2751 let event_type = match choice {
2752 0 => "fork",
2753 1 => "work",
2754 _ => "join",
2755 };
2756
2757 runtime.trace().record_event(|id| {
2758 crate::trace::TraceEvent::user_trace(
2759 id,
2760 runtime.now(),
2761 format!("{}_{}", event_type, i),
2762 )
2763 });
2764 }
2765 };
2766
2767 let mut run_results = Vec::new();
2769
2770 for run_idx in 0..5 {
2771 let mut runtime_config = LabConfig::new(SEED);
2772 runtime_config = runtime_config.worker_count(config.worker_count);
2773 let mut runtime = LabRuntime::new(runtime_config);
2774
2775 deterministic_scenario(&mut runtime);
2776
2777 let trace = runtime.trace().snapshot();
2778 let certificate = runtime.certificate().hash();
2779 let steps = runtime.steps();
2780
2781 run_results.push((run_idx, trace, certificate, steps));
2782 }
2783
2784 for (run_idx, trace, certificate, steps) in &run_results[1..] {
2786 assert_eq!(
2787 run_results[0].2, *certificate,
2788 "Run {} certificate differs from run 0",
2789 run_idx
2790 );
2791 assert_eq!(
2792 run_results[0].3, *steps,
2793 "Run {} step count differs from run 0",
2794 run_idx
2795 );
2796
2797 let divergence = find_divergence(&run_results[0].1, trace);
2798 assert!(
2799 divergence.is_none(),
2800 "Run {} trace diverged from run 0: {:?}",
2801 run_idx,
2802 divergence
2803 );
2804 }
2805
2806 let mut different_seed_config = LabConfig::new(SEED + 1);
2808 different_seed_config = different_seed_config.worker_count(config.worker_count);
2809 let mut different_seed_runtime = LabRuntime::new(different_seed_config);
2810
2811 deterministic_scenario(&mut different_seed_runtime);
2812 let _different_trace = different_seed_runtime.trace().snapshot();
2813 let different_certificate = different_seed_runtime.certificate().hash();
2814
2815 assert_ne!(
2816 run_results[0].2, different_certificate,
2817 "Different seed should produce different certificate"
2818 );
2819
2820 crate::test_complete!("metamorphic_lab_runtime_seed_determinism");
2824 }
2825
2826 #[test]
2831 fn metamorphic_composite_replay_invariants() {
2832 init_test("metamorphic_composite_replay_invariants");
2833
2834 let seed = std::time::SystemTime::now()
2835 .duration_since(std::time::UNIX_EPOCH)
2836 .unwrap()
2837 .as_nanos() as u64;
2838
2839 let config = ReplayMetamorphicConfig::default();
2840
2841 let composite_scenario = |runtime: &mut LabRuntime| {
2843 use crate::util::det_rng::DetRng;
2844 let mut rng = DetRng::new(seed);
2845
2846 let regions = 2;
2853 let tasks_per_region = config.task_count / regions;
2854
2855 for region_id in 0..regions {
2856 for task_id in 0..tasks_per_region {
2858 let now = runtime.now();
2859 runtime.trace().record_event(|id| {
2860 crate::trace::TraceEvent::user_trace(
2861 id,
2862 now,
2863 format!("fork_region_{}_task_{}", region_id, task_id),
2864 )
2865 });
2866 }
2867
2868 for task_id in 0..tasks_per_region {
2870 let event_type = if rng.next_u64() % 10 == 0 {
2871 "panic"
2872 } else {
2873 "work"
2874 };
2875 let now = runtime.now();
2876 runtime.trace().record_event(|id| {
2877 crate::trace::TraceEvent::user_trace(
2878 id,
2879 now,
2880 format!("{}_region_{}_task_{}", event_type, region_id, task_id),
2881 )
2882 });
2883 }
2884
2885 for task_id in 0..tasks_per_region {
2887 runtime.trace().record_event(|id| {
2888 crate::trace::TraceEvent::user_trace(
2889 id,
2890 runtime.now(),
2891 format!("join_region_{}_task_{}", region_id, task_id),
2892 )
2893 });
2894 }
2895 }
2896 };
2897
2898 let replay_validation = validate_replay(seed, config.worker_count, composite_scenario);
2900
2901 assert!(
2902 replay_validation.matched,
2903 "Composite scenario replay should match original: certificates {} vs {}, steps {} vs {}",
2904 replay_validation.original_certificate,
2905 replay_validation.replay_certificate,
2906 replay_validation.original_steps,
2907 replay_validation.replay_steps
2908 );
2909
2910 assert!(
2911 replay_validation.divergence.is_none(),
2912 "Composite scenario should have no divergence: {:?}",
2913 replay_validation.divergence
2914 );
2915
2916 let test_seeds = [seed, seed + 1, seed + 42, seed + 1337, seed + 0xDEAD];
2918
2919 for &test_seed in &test_seeds {
2920 let validation = validate_replay(test_seed, config.worker_count, |runtime| {
2921 composite_scenario(runtime);
2922 });
2923
2924 assert!(
2925 validation.matched,
2926 "Seed {} composite replay failed: {:?}",
2927 test_seed, validation.divergence
2928 );
2929 }
2930
2931 let multi_validation =
2933 validate_replay_multi(&test_seeds, config.worker_count, composite_scenario);
2934
2935 for (i, validation) in multi_validation.iter().enumerate() {
2936 assert!(validation.matched, "Multi-seed run {} failed validation", i);
2937 }
2938 crate::test_complete!("metamorphic_composite_replay_invariants");
2939 }
2940}