Skip to main content

asupersync/lab/
replay.rs

1//! Replay and diff utilities for trace analysis.
2//!
3//! This module provides utilities for:
4//! - Replaying a trace to reproduce an execution
5//! - Comparing two traces to find divergences
6//! - Replay validation with certificate checking
7//! - **Trace normalization** for canonical replay ordering
8//!
9//! # Trace Normalization
10//!
11//! Use [`normalize_for_replay`] to reorder trace events into a canonical form
12//! that minimizes context switches while preserving all happens-before
13//! relationships. This is useful for:
14//!
15//! - Deterministic comparison of equivalent traces
16//! - Debugging with reduced interleaving noise
17//! - Trace minimization and simplification
18//!
19//! ```ignore
20//! use asupersync::lab::replay::{normalize_for_replay, traces_equivalent};
21//!
22//! // Normalize a trace
23//! let result = normalize_for_replay(&events);
24//! println!("{}", result); // Shows switch count reduction
25//!
26//! // Compare two traces for equivalence
27//! if traces_equivalent(&trace_a, &trace_b) {
28//!     println!("Traces are equivalent under normalization");
29//! }
30//! ```
31
32use crate::lab::config::LabConfig;
33use crate::lab::runtime::{CrashpackLink, LabRuntime, SporkHarnessReport};
34use crate::lab::spork_harness::{ScenarioRunnerError, SporkScenarioConfig, SporkScenarioRunner};
35use crate::trace::{TraceBuffer, TraceBufferHandle, TraceEvent};
36use serde::{Deserialize, Serialize};
37use std::collections::BTreeMap;
38
39/// Compares two traces and returns the first divergence point.
40///
41/// Returns `None` if the traces are equivalent.
42#[must_use]
43pub fn find_divergence(a: &[TraceEvent], b: &[TraceEvent]) -> Option<TraceDivergence> {
44    let a_events = a;
45    let b_events = b;
46
47    for (i, (a_event, b_event)) in a_events.iter().zip(b_events.iter()).enumerate() {
48        if !events_match(a_event, b_event) {
49            return Some(TraceDivergence {
50                position: i,
51                event_a: (*a_event).clone(),
52                event_b: (*b_event).clone(),
53            });
54        }
55    }
56
57    // Check for length mismatch
58    if a_events.len() != b_events.len() {
59        let position = a_events.len().min(b_events.len());
60        #[allow(clippy::map_unwrap_or)]
61        return Some(TraceDivergence {
62            position,
63            event_a: a_events
64                .get(position)
65                .map(|e| (*e).clone())
66                .unwrap_or_else(|| {
67                    TraceEvent::user_trace(0, crate::types::Time::ZERO, "<end of trace A>")
68                }),
69            event_b: b_events
70                .get(position)
71                .map(|e| (*e).clone())
72                .unwrap_or_else(|| {
73                    TraceEvent::user_trace(0, crate::types::Time::ZERO, "<end of trace B>")
74                }),
75        });
76    }
77
78    None
79}
80
81/// Checks if two events match (ignoring sequence numbers).
82fn events_match(a: &TraceEvent, b: &TraceEvent) -> bool {
83    a.kind == b.kind && a.time == b.time && a.logical_time == b.logical_time && a.data == b.data
84}
85
86/// A divergence between two traces.
87#[derive(Debug, Clone)]
88pub struct TraceDivergence {
89    /// Position in the trace where divergence occurred.
90    pub position: usize,
91    /// Event from trace A at the divergence point.
92    pub event_a: TraceEvent,
93    /// Event from trace B at the divergence point.
94    pub event_b: TraceEvent,
95}
96
97impl std::fmt::Display for TraceDivergence {
98    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
99        write!(
100            f,
101            "Divergence at position {}:\n  A: {}\n  B: {}",
102            self.position, self.event_a, self.event_b
103        )
104    }
105}
106
107/// Summary of a trace for quick comparison.
108#[derive(Debug, Clone, PartialEq, Eq)]
109pub struct TraceSummary {
110    /// Number of events.
111    pub event_count: usize,
112    /// Number of spawn events.
113    pub spawn_count: usize,
114    /// Number of complete events.
115    pub complete_count: usize,
116    /// Number of cancel events.
117    pub cancel_count: usize,
118}
119
120impl TraceSummary {
121    /// Creates a summary from a trace buffer.
122    #[must_use]
123    pub fn from_buffer(buffer: &TraceBuffer) -> Self {
124        use crate::trace::event::TraceEventKind;
125
126        let mut summary = Self {
127            event_count: 0,
128            spawn_count: 0,
129            complete_count: 0,
130            cancel_count: 0,
131        };
132
133        for event in buffer.iter() {
134            summary.event_count += 1;
135            match event.kind {
136                TraceEventKind::Spawn => summary.spawn_count += 1,
137                TraceEventKind::Complete => summary.complete_count += 1,
138                TraceEventKind::CancelRequest | TraceEventKind::CancelAck => {
139                    summary.cancel_count += 1;
140                }
141                _ => {}
142            }
143        }
144
145        summary
146    }
147}
148
149/// Result of a replay validation.
150#[derive(Debug)]
151pub struct ReplayValidation {
152    /// Whether the replay matched the original.
153    pub matched: bool,
154    /// Certificate from the original run.
155    pub original_certificate: u64,
156    /// Certificate from the replay.
157    pub replay_certificate: u64,
158    /// First trace divergence (if any).
159    pub divergence: Option<TraceDivergence>,
160    /// Steps in original.
161    pub original_steps: u64,
162    /// Steps in replay.
163    pub replay_steps: u64,
164}
165
166impl ReplayValidation {
167    /// True if both certificate and trace matched.
168    #[must_use]
169    pub fn is_valid(&self) -> bool {
170        self.matched && self.divergence.is_none()
171    }
172}
173
174impl std::fmt::Display for ReplayValidation {
175    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176        if self.is_valid() {
177            write!(
178                f,
179                "Replay OK: {} steps, certificate {:#018x}",
180                self.replay_steps, self.replay_certificate
181            )
182        } else {
183            write!(f, "Replay DIVERGED:")?;
184            if self.original_certificate != self.replay_certificate {
185                write!(
186                    f,
187                    "\n  Certificate mismatch: original={:#018x} replay={:#018x}",
188                    self.original_certificate, self.replay_certificate
189                )?;
190            }
191            if let Some(ref div) = self.divergence {
192                write!(f, "\n  {div}")?;
193            }
194            if self.original_steps != self.replay_steps {
195                write!(
196                    f,
197                    "\n  Step count mismatch: original={} replay={}",
198                    self.original_steps, self.replay_steps
199                )?;
200            }
201            Ok(())
202        }
203    }
204}
205
206/// Replay a test with the same seed and validate determinism.
207///
208/// Runs the test twice with the same seed and checks:
209/// 1. Schedule certificates match
210/// 2. Traces match (no divergence)
211/// 3. Step counts match
212pub fn validate_replay<F>(seed: u64, worker_count: usize, test: F) -> ReplayValidation
213where
214    F: Fn(&mut LabRuntime),
215{
216    let run = |s: u64| -> (u64, u64, TraceBufferHandle) {
217        let mut config = LabConfig::new(s);
218        config = config.worker_count(worker_count);
219        let mut runtime = LabRuntime::new(config);
220        test(&mut runtime);
221        let steps = runtime.steps();
222        let cert = runtime.certificate().hash();
223        let trace = runtime.trace().clone();
224        (steps, cert, trace)
225    };
226
227    let (steps_a, cert_a, trace_a) = run(seed);
228    let (steps_b, cert_b, trace_b) = run(seed);
229
230    let events_a = trace_a.snapshot();
231    let events_b = trace_b.snapshot();
232    let divergence = find_divergence(&events_a, &events_b);
233    let matched = cert_a == cert_b && steps_a == steps_b;
234
235    ReplayValidation {
236        matched,
237        original_certificate: cert_a,
238        replay_certificate: cert_b,
239        divergence,
240        original_steps: steps_a,
241        replay_steps: steps_b,
242    }
243}
244
245/// Validate replay across multiple seeds and report any failures.
246pub fn validate_replay_multi<F>(
247    seeds: &[u64],
248    worker_count: usize,
249    test: F,
250) -> Vec<ReplayValidation>
251where
252    F: Fn(&mut LabRuntime),
253{
254    seeds
255        .iter()
256        .map(|&seed| validate_replay(seed, worker_count, &test))
257        .collect()
258}
259
260/// Single seed-run summary for schedule exploration.
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct ExplorationRunSummary {
263    /// Seed used for this run.
264    pub seed: u64,
265    /// Scheduler certificate hash for this run.
266    pub schedule_hash: u64,
267    /// Canonical normalized-trace fingerprint for this run.
268    pub trace_fingerprint: u64,
269}
270
271/// Deterministic fingerprint class produced by exploration.
272#[derive(Debug, Clone, PartialEq, Eq)]
273pub struct ExplorationFingerprintClass {
274    /// Canonical normalized-trace fingerprint.
275    pub trace_fingerprint: u64,
276    /// Number of runs in this class.
277    pub run_count: usize,
278    /// Seeds observed in this class (sorted, deduplicated).
279    pub seeds: Vec<u64>,
280    /// Schedule hashes observed in this class (sorted, deduplicated).
281    pub schedule_hashes: Vec<u64>,
282}
283
284/// Deterministic schedule-exploration report.
285#[derive(Debug, Clone, PartialEq, Eq)]
286pub struct ExplorationReport {
287    /// Per-seed runs in stable order.
288    pub runs: Vec<ExplorationRunSummary>,
289    /// Unique canonical fingerprint classes in stable order.
290    pub fingerprint_classes: Vec<ExplorationFingerprintClass>,
291}
292
293impl ExplorationReport {
294    /// Number of unique canonical fingerprint classes observed.
295    #[must_use]
296    pub fn unique_fingerprint_count(&self) -> usize {
297        self.fingerprint_classes.len()
298    }
299}
300
301/// Per-run deterministic summary for Spork app exploration.
302#[derive(Debug, Clone, PartialEq, Eq)]
303pub struct SporkExplorationRunSummary {
304    /// Seed used for this run.
305    pub seed: u64,
306    /// Scheduler certificate hash for this run.
307    pub schedule_hash: u64,
308    /// Canonical trace fingerprint for this run.
309    pub trace_fingerprint: u64,
310    /// Whether all run invariants/oracles passed.
311    pub passed: bool,
312    /// Crashpack link metadata for failing runs when available.
313    pub crashpack_link: Option<CrashpackLink>,
314}
315
316/// Deterministic DPOR-style report for Spork app seed exploration.
317#[derive(Debug, Clone, PartialEq, Eq)]
318pub struct SporkExplorationReport {
319    /// Per-seed run summaries in stable order.
320    pub runs: Vec<SporkExplorationRunSummary>,
321    /// Unique canonical fingerprint classes in stable order.
322    pub fingerprint_classes: Vec<ExplorationFingerprintClass>,
323}
324
325impl SporkExplorationReport {
326    /// Number of unique canonical fingerprint classes observed.
327    #[must_use]
328    pub fn unique_fingerprint_count(&self) -> usize {
329        self.fingerprint_classes.len()
330    }
331
332    /// Number of failed runs.
333    #[must_use]
334    pub fn failure_count(&self) -> usize {
335        self.runs.iter().filter(|run| !run.passed).count()
336    }
337
338    /// True when every failed run includes crashpack linkage metadata.
339    #[must_use]
340    pub fn all_failures_linked_to_crashpacks(&self) -> bool {
341        self.runs
342            .iter()
343            .filter(|run| !run.passed)
344            .all(|run| run.crashpack_link.is_some())
345    }
346}
347
348/// Classify run summaries by canonical fingerprint into deterministic classes.
349#[must_use]
350pub fn classify_fingerprint_classes(
351    runs: &[ExplorationRunSummary],
352) -> Vec<ExplorationFingerprintClass> {
353    let mut grouped: BTreeMap<u64, (usize, Vec<u64>, Vec<u64>)> = BTreeMap::new();
354
355    for run in runs {
356        let entry = grouped
357            .entry(run.trace_fingerprint)
358            .or_insert_with(|| (0, Vec::new(), Vec::new()));
359        entry.0 += 1;
360        entry.1.push(run.seed);
361        entry.2.push(run.schedule_hash);
362    }
363
364    grouped
365        .into_iter()
366        .map(
367            |(trace_fingerprint, (run_count, mut seeds, mut schedule_hashes))| {
368                seeds.sort_unstable();
369                seeds.dedup();
370                schedule_hashes.sort_unstable();
371                schedule_hashes.dedup();
372                ExplorationFingerprintClass {
373                    trace_fingerprint,
374                    run_count,
375                    seeds,
376                    schedule_hashes,
377                }
378            },
379        )
380        .collect()
381}
382
383/// Explore a seed-space and report deterministic canonical fingerprint classes.
384///
385/// This is a DPOR-style seed exploration helper: each seed produces one schedule
386/// and one normalized-trace fingerprint; the report groups equivalent runs.
387pub fn explore_seed_space<F>(seeds: &[u64], worker_count: usize, test: F) -> ExplorationReport
388where
389    F: Fn(&mut LabRuntime),
390{
391    let mut runs: Vec<ExplorationRunSummary> = seeds
392        .iter()
393        .map(|&seed| {
394            let mut config = LabConfig::new(seed);
395            config = config.worker_count(worker_count);
396            let mut runtime = LabRuntime::new(config);
397            test(&mut runtime);
398
399            let trace_events = runtime.trace().snapshot();
400            let normalized = normalize_for_replay(&trace_events);
401            let trace_fingerprint =
402                crate::trace::canonicalize::trace_fingerprint(&normalized.normalized);
403
404            ExplorationRunSummary {
405                seed,
406                schedule_hash: runtime.certificate().hash(),
407                trace_fingerprint,
408            }
409        })
410        .collect();
411
412    runs.sort_by_key(|run| run.seed);
413    let fingerprint_classes = classify_fingerprint_classes(&runs);
414    ExplorationReport {
415        runs,
416        fingerprint_classes,
417    }
418}
419
420/// Build a deterministic Spork exploration report from completed harness reports.
421#[must_use]
422pub fn summarize_spork_reports(reports: &[SporkHarnessReport]) -> SporkExplorationReport {
423    let mut runs: Vec<SporkExplorationRunSummary> = reports
424        .iter()
425        .map(|report| {
426            let passed = report.passed();
427            SporkExplorationRunSummary {
428                seed: report.seed(),
429                schedule_hash: report.run.trace_certificate.schedule_hash,
430                trace_fingerprint: report.trace_fingerprint(),
431                passed,
432                crashpack_link: if passed {
433                    None
434                } else {
435                    report.crashpack_link()
436                },
437            }
438        })
439        .collect();
440
441    runs.sort_by_key(|run| (run.seed, run.schedule_hash, run.trace_fingerprint));
442
443    let class_input: Vec<ExplorationRunSummary> = runs
444        .iter()
445        .map(|run| ExplorationRunSummary {
446            seed: run.seed,
447            schedule_hash: run.schedule_hash,
448            trace_fingerprint: run.trace_fingerprint,
449        })
450        .collect();
451
452    SporkExplorationReport {
453        runs,
454        fingerprint_classes: classify_fingerprint_classes(&class_input),
455    }
456}
457
458/// Explore a Spork app seed-space and produce a deterministic DPOR-style report.
459///
460/// The caller provides one harness report per seed (typically by running
461/// `SporkAppHarness`/`SporkScenarioRunner` with that seed). The result is
462/// grouped by canonical fingerprint class and keeps failure-to-crashpack links.
463pub fn explore_spork_seed_space<F>(seeds: &[u64], mut run_for_seed: F) -> SporkExplorationReport
464where
465    F: FnMut(u64) -> SporkHarnessReport,
466{
467    let reports: Vec<SporkHarnessReport> = seeds.iter().map(|&seed| run_for_seed(seed)).collect();
468    summarize_spork_reports(&reports)
469}
470
471/// Run a registered Spork scenario across seeds and return deterministic
472/// exploration classes with failure-to-crashpack linkage.
473///
474/// This is the glue between `SporkScenarioRunner` and DPOR-style exploration:
475/// callers provide a scenario id and base config, and this helper handles
476/// seed fan-out + deterministic report grouping.
477pub fn explore_scenario_runner_seed_space(
478    runner: &SporkScenarioRunner,
479    scenario_id: &str,
480    base_config: &SporkScenarioConfig,
481    seeds: &[u64],
482) -> Result<SporkExplorationReport, ScenarioRunnerError> {
483    let mut reports = Vec::with_capacity(seeds.len());
484    for &seed in seeds {
485        let mut config = base_config.clone();
486        config.seed = seed;
487        let result = runner.run_with_config(scenario_id, Some(config))?;
488        reports.push(result.report);
489    }
490    Ok(summarize_spork_reports(&reports))
491}
492
493/// Schema version for the divergence corpus registry.
494pub const DIVERGENCE_CORPUS_SCHEMA_VERSION: &str = "lab-live-divergence-corpus-v1";
495
496/// Retention class for a divergence artifact bundle.
497#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
498#[serde(rename_all = "snake_case")]
499pub enum DivergenceBundleLevel {
500    /// Preserve the complete debugging bundle.
501    Full,
502    /// Preserve only the reduced summary bundle.
503    Reduced,
504}
505
506/// Final differential policy class from the divergence taxonomy.
507#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
508#[serde(rename_all = "snake_case")]
509pub enum DifferentialPolicyClass {
510    /// Stable semantic mismatch on a supported surface.
511    RuntimeSemanticBug,
512    /// Lab model, mapping, or comparator bug.
513    LabModelOrMappingBug,
514    /// Required artifact schema or evidence is missing/malformed.
515    ArtifactSchemaViolation,
516    /// The surface lacks the observability needed for a strong claim.
517    InsufficientObservability,
518    /// The surface is outside the admitted differential scope.
519    UnsupportedSurface,
520    /// The mismatch looks like scheduling noise rather than semantics.
521    SchedulerNoiseSuspected,
522    /// The mismatch could not be stabilized by rerun policy.
523    IrreproducibleDivergence,
524}
525
526impl DifferentialPolicyClass {
527    /// Stable string form shared by docs, logs, and registry entries.
528    #[must_use]
529    pub fn as_str(self) -> &'static str {
530        match self {
531            Self::RuntimeSemanticBug => "runtime_semantic_bug",
532            Self::LabModelOrMappingBug => "lab_model_or_mapping_bug",
533            Self::ArtifactSchemaViolation => "artifact_schema_violation",
534            Self::InsufficientObservability => "insufficient_observability",
535            Self::UnsupportedSurface => "unsupported_surface",
536            Self::SchedulerNoiseSuspected => "scheduler_noise_suspected",
537            Self::IrreproducibleDivergence => "irreproducible_divergence",
538        }
539    }
540
541    /// Required bundle strength from the divergence taxonomy.
542    #[must_use]
543    pub fn bundle_level(self) -> DivergenceBundleLevel {
544        match self {
545            Self::RuntimeSemanticBug
546            | Self::LabModelOrMappingBug
547            | Self::ArtifactSchemaViolation
548            | Self::IrreproducibleDivergence => DivergenceBundleLevel::Full,
549            Self::InsufficientObservability
550            | Self::UnsupportedSurface
551            | Self::SchedulerNoiseSuspected => DivergenceBundleLevel::Reduced,
552        }
553    }
554}
555
556impl std::fmt::Display for DifferentialPolicyClass {
557    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
558        f.write_str(self.as_str())
559    }
560}
561
562/// Lifecycle state for a divergence corpus entry.
563#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
564#[serde(rename_all = "snake_case")]
565pub enum RegressionPromotionState {
566    /// Newly discovered divergence under investigation.
567    Investigating,
568    /// A minimized reproducer exists and preserves the same semantics.
569    Minimized,
570    /// Promoted into a durable regression artifact.
571    PromotedRegression,
572    /// Retained as a known-open investigation instead of a regression.
573    KnownOpen,
574    /// Explicitly rejected for promotion.
575    Rejected,
576}
577
578/// Minimization/shrinker status for a divergence entry.
579#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
580#[serde(rename_all = "snake_case")]
581pub enum DivergenceShrinkStatus {
582    /// No shrinker has been requested yet.
583    NotRequested,
584    /// Shrinking is still in progress.
585    Pending,
586    /// A minimized reproducer exists and preserves the semantic class.
587    PreservedSemanticClass,
588    /// Shrinking failed to preserve the semantic class and must not replace the original.
589    Rejected,
590}
591
592/// Stable artifact layout for a retained differential bundle.
593#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
594pub struct DivergenceArtifactBundle {
595    /// Root directory for the retained bundle.
596    pub bundle_root: String,
597    /// Stable summary record path.
598    pub differential_summary_path: String,
599    /// Stable event-log path.
600    pub differential_event_log_path: String,
601    /// Stable failures path.
602    pub differential_failures_path: String,
603    /// Stable deviations path.
604    pub differential_deviations_path: String,
605    /// Stable repro manifest path.
606    pub differential_repro_manifest_path: String,
607    /// Stable lab normalized-record path.
608    pub lab_normalized_path: String,
609    /// Stable live normalized-record path.
610    pub live_normalized_path: String,
611}
612
613impl DivergenceArtifactBundle {
614    /// Build the canonical bundle layout under a root directory.
615    #[must_use]
616    pub fn under(root: impl Into<String>) -> Self {
617        let bundle_root = root.into().trim_end_matches('/').to_string();
618        let join = |name: &str| format!("{bundle_root}/{name}");
619        Self {
620            bundle_root: bundle_root.clone(),
621            differential_summary_path: join("differential_summary.json"),
622            differential_event_log_path: join("differential_event_log.jsonl"),
623            differential_failures_path: join("differential_failures.json"),
624            differential_deviations_path: join("differential_deviations.json"),
625            differential_repro_manifest_path: join("differential_repro_manifest.json"),
626            lab_normalized_path: join("lab_normalized.json"),
627            live_normalized_path: join("live_normalized.json"),
628        }
629    }
630}
631
632/// Stable retention metadata for a divergence bundle.
633#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
634pub struct DivergenceRetentionMetadata {
635    /// Required bundle strength.
636    pub bundle_level: DivergenceBundleLevel,
637    /// Default local retention window in days.
638    pub local_retention_days: u16,
639    /// Default CI retention window in days.
640    pub ci_retention_days: u16,
641    /// Default redaction policy for retained artifacts.
642    pub redaction_mode: String,
643}
644
645impl DivergenceRetentionMetadata {
646    /// Retention defaults derived from the divergence taxonomy.
647    #[must_use]
648    pub fn for_policy_class(policy_class: DifferentialPolicyClass) -> Self {
649        Self {
650            bundle_level: policy_class.bundle_level(),
651            local_retention_days: 14,
652            ci_retention_days: 30,
653            redaction_mode: "metadata_only".to_string(),
654        }
655    }
656}
657
658/// First-seen execution context for a divergence.
659#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
660pub struct DivergenceFirstSeenContext {
661    /// Named runner profile such as `smoke`, `pilot_surface`, or `nightly`.
662    pub runner_profile: String,
663    /// Attempt index within the local run.
664    pub attempt_index: u32,
665    /// Number of reruns already attempted when this entry was recorded.
666    pub rerun_count: u32,
667}
668
669/// Minimization lineage for a divergence.
670#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
671pub struct DivergenceMinimizationLineage {
672    /// Original canonical seed from the first-seen run.
673    pub original_seed: u64,
674    /// Minimized seed when one exists.
675    #[serde(skip_serializing_if = "Option::is_none")]
676    pub minimized_seed: Option<u64>,
677    /// Named shrinker or minimization pass when one exists.
678    #[serde(skip_serializing_if = "Option::is_none")]
679    pub shrinker: Option<String>,
680    /// Current shrink status.
681    pub shrink_status: DivergenceShrinkStatus,
682    /// Whether the minimized form preserved the same divergence class.
683    pub preserved_divergence_class: bool,
684    /// Whether the minimized form preserved the same policy class.
685    pub preserved_policy_class: bool,
686}
687
688impl DivergenceMinimizationLineage {
689    /// Start minimization lineage from a seed lineage record.
690    #[must_use]
691    pub fn from_seed_lineage(lineage: &crate::lab::dual_run::SeedLineageRecord) -> Self {
692        Self {
693            original_seed: lineage.canonical_seed,
694            minimized_seed: None,
695            shrinker: None,
696            shrink_status: DivergenceShrinkStatus::NotRequested,
697            preserved_divergence_class: true,
698            preserved_policy_class: true,
699        }
700    }
701
702    /// Record a minimized reproducer that preserves the same semantic meaning.
703    #[must_use]
704    pub fn with_minimized_seed(
705        mut self,
706        seed: u64,
707        shrinker: impl Into<String>,
708        preserved_divergence_class: bool,
709        preserved_policy_class: bool,
710    ) -> Self {
711        self.minimized_seed = Some(seed);
712        self.shrinker = Some(shrinker.into());
713        self.shrink_status = if preserved_divergence_class && preserved_policy_class {
714            DivergenceShrinkStatus::PreservedSemanticClass
715        } else {
716            DivergenceShrinkStatus::Rejected
717        };
718        self.preserved_divergence_class = preserved_divergence_class;
719        self.preserved_policy_class = preserved_policy_class;
720        self
721    }
722}
723
724/// One retained divergence entry in the differential corpus.
725#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
726pub struct DivergenceCorpusEntry {
727    /// Stable schema discriminator.
728    pub schema_version: String,
729    /// Stable entry identifier used for registry upsert.
730    pub entry_id: String,
731    /// Scenario id from the differential run.
732    pub scenario_id: String,
733    /// Surface id from the differential run.
734    pub surface_id: String,
735    /// Surface contract version from the differential run.
736    pub surface_contract_version: String,
737    /// Diagnostic divergence class for this entry.
738    pub divergence_class: String,
739    /// Final differential policy class for this entry.
740    pub policy_class: DifferentialPolicyClass,
741    /// First-seen execution context.
742    pub first_seen: DivergenceFirstSeenContext,
743    /// Full seed lineage from the originating run.
744    pub seed_lineage: crate::lab::dual_run::SeedLineageRecord,
745    /// Stable mismatch field names for semantic preservation.
746    #[serde(default, skip_serializing_if = "Vec::is_empty")]
747    pub mismatch_fields: Vec<String>,
748    /// Stable retained bundle layout.
749    pub artifact_bundle: DivergenceArtifactBundle,
750    /// Shrinker/minimization lineage.
751    pub minimization_lineage: DivergenceMinimizationLineage,
752    /// Current promotion state for this entry.
753    pub regression_promotion_state: RegressionPromotionState,
754    /// Stable retention metadata.
755    pub retention: DivergenceRetentionMetadata,
756    /// Additional machine-readable annotations.
757    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
758    pub metadata: BTreeMap<String, String>,
759}
760
761impl DivergenceCorpusEntry {
762    /// Create a registry entry from a differential result and retained bundle root.
763    #[must_use]
764    pub fn from_dual_run_result(
765        result: &crate::lab::dual_run::DualRunResult,
766        runner_profile: impl Into<String>,
767        divergence_class: impl Into<String>,
768        policy_class: DifferentialPolicyClass,
769        bundle_root: impl Into<String>,
770    ) -> Self {
771        let seed_lineage = result.seed_lineage.clone();
772        let entry_id = Self::entry_id_for(
773            &result.verdict.scenario_id,
774            &seed_lineage.seed_lineage_id,
775            policy_class,
776        );
777        let mut mismatch_fields: Vec<String> = result
778            .verdict
779            .mismatches
780            .iter()
781            .map(|mismatch| mismatch.field.clone())
782            .collect();
783        mismatch_fields.sort_unstable();
784        mismatch_fields.dedup();
785
786        let mut metadata = BTreeMap::new();
787        if let Some(path) = result.lab.provenance.artifact_path.as_deref() {
788            metadata.insert("lab_artifact_path".to_string(), path.to_string());
789        }
790        if let Some(path) = result.live.provenance.artifact_path.as_deref() {
791            metadata.insert("live_artifact_path".to_string(), path.to_string());
792        }
793        if let Some(cmd) = result.lab.provenance.repro_command.as_deref() {
794            metadata.insert("lab_repro_command".to_string(), cmd.to_string());
795        }
796        if let Some(cmd) = result.live.provenance.repro_command.as_deref() {
797            metadata.insert("live_repro_command".to_string(), cmd.to_string());
798        }
799        if !result.lab_invariant_violations.is_empty() {
800            metadata.insert(
801                "lab_invariant_violations".to_string(),
802                result.lab_invariant_violations.join(","),
803            );
804        }
805        if !result.live_invariant_violations.is_empty() {
806            metadata.insert(
807                "live_invariant_violations".to_string(),
808                result.live_invariant_violations.join(","),
809            );
810        }
811
812        Self {
813            schema_version: DIVERGENCE_CORPUS_SCHEMA_VERSION.to_string(),
814            entry_id,
815            scenario_id: result.verdict.scenario_id.clone(),
816            surface_id: result.verdict.surface_id.clone(),
817            surface_contract_version: result.lab.surface_contract_version.clone(),
818            divergence_class: divergence_class.into(),
819            policy_class,
820            first_seen: DivergenceFirstSeenContext {
821                runner_profile: runner_profile.into(),
822                attempt_index: 0,
823                rerun_count: 0,
824            },
825            seed_lineage: seed_lineage.clone(),
826            mismatch_fields,
827            artifact_bundle: DivergenceArtifactBundle::under(bundle_root),
828            minimization_lineage: DivergenceMinimizationLineage::from_seed_lineage(&seed_lineage),
829            regression_promotion_state: RegressionPromotionState::Investigating,
830            retention: DivergenceRetentionMetadata::for_policy_class(policy_class),
831            metadata,
832        }
833    }
834
835    /// Stable entry id from the scenario, seed lineage, and final policy class.
836    #[must_use]
837    pub fn entry_id_for(
838        scenario_id: &str,
839        seed_lineage_id: &str,
840        policy_class: DifferentialPolicyClass,
841    ) -> String {
842        format!(
843            "{}.{}.{}",
844            sanitize_registry_component(scenario_id),
845            sanitize_registry_component(seed_lineage_id),
846            policy_class.as_str()
847        )
848    }
849
850    /// Default bundle root for this entry under `artifacts/differential/`.
851    #[must_use]
852    pub fn default_bundle_root(&self) -> String {
853        format!(
854            "artifacts/differential/{}/{}/{}",
855            sanitize_registry_component(&self.surface_id),
856            sanitize_registry_component(&self.scenario_id),
857            sanitize_registry_component(&self.seed_lineage.seed_lineage_id)
858        )
859    }
860
861    /// Update first-seen attempt/rerun counters.
862    #[must_use]
863    pub fn with_first_seen_attempt(mut self, attempt_index: u32, rerun_count: u32) -> Self {
864        self.first_seen.attempt_index = attempt_index;
865        self.first_seen.rerun_count = rerun_count;
866        self
867    }
868
869    /// Update the minimization lineage.
870    #[must_use]
871    pub fn with_minimization_lineage(mut self, lineage: DivergenceMinimizationLineage) -> Self {
872        self.minimization_lineage = lineage;
873        self.regression_promotion_state = if self.minimization_lineage.minimized_seed.is_some() {
874            RegressionPromotionState::Minimized
875        } else {
876            self.regression_promotion_state
877        };
878        self
879    }
880
881    /// Promote the entry into a durable regression artifact.
882    #[must_use]
883    pub fn promote_to_regression(mut self, promoted_scenario_id: impl Into<String>) -> Self {
884        self.regression_promotion_state = RegressionPromotionState::PromotedRegression;
885        self.metadata.insert(
886            "promoted_scenario_id".to_string(),
887            promoted_scenario_id.into(),
888        );
889        self
890    }
891}
892
893/// Deterministic registry of retained divergences.
894#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
895pub struct DivergenceCorpusRegistry {
896    /// Stable schema discriminator.
897    pub schema_version: String,
898    /// Entries sorted by stable entry id.
899    pub entries: Vec<DivergenceCorpusEntry>,
900}
901
902impl DivergenceCorpusRegistry {
903    /// Create an empty divergence corpus registry.
904    #[must_use]
905    pub fn new() -> Self {
906        Self {
907            schema_version: DIVERGENCE_CORPUS_SCHEMA_VERSION.to_string(),
908            entries: Vec::new(),
909        }
910    }
911
912    /// Insert or replace an entry by stable id, preserving deterministic order.
913    pub fn upsert(&mut self, entry: DivergenceCorpusEntry) {
914        if let Some(existing) = self
915            .entries
916            .iter_mut()
917            .find(|existing| existing.entry_id == entry.entry_id)
918        {
919            *existing = entry;
920        } else {
921            self.entries.push(entry);
922            self.entries
923                .sort_by(|left, right| left.entry_id.cmp(&right.entry_id));
924        }
925    }
926}
927
928impl Default for DivergenceCorpusRegistry {
929    fn default() -> Self {
930        Self::new()
931    }
932}
933
934/// Schema version for the retained divergence summary payload.
935pub const DIFFERENTIAL_SUMMARY_SCHEMA_VERSION: &str = "lab-live-differential-summary-v1";
936/// Schema version for runtime/failure artifact linkage.
937pub const DIFFERENTIAL_FAILURES_SCHEMA_VERSION: &str = "lab-live-differential-failures-v1";
938/// Schema version for mismatch/deviation details.
939pub const DIFFERENTIAL_DEVIATIONS_SCHEMA_VERSION: &str = "lab-live-differential-deviations-v1";
940/// Schema version for the replay/minimization repro manifest.
941pub const DIFFERENTIAL_REPRO_MANIFEST_SCHEMA_VERSION: &str =
942    "lab-live-differential-repro-manifest-v1";
943
944/// Serializable crashpack linkage metadata for retained divergence bundles.
945#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
946pub struct DifferentialCrashpackReference {
947    /// Crashpack artifact path.
948    pub path: String,
949    /// Stable crashpack identifier.
950    pub id: String,
951    /// Canonical trace fingerprint associated with the crashpack.
952    pub fingerprint: u64,
953    /// One-line replay command for the crashpack.
954    pub replay_command: String,
955}
956
957impl DifferentialCrashpackReference {
958    /// Convert an existing runtime crashpack link into the retained schema.
959    #[must_use]
960    pub fn from_runtime_link(link: &CrashpackLink) -> Self {
961        Self {
962            path: link.path.clone(),
963            id: link.id.clone(),
964            fingerprint: link.fingerprint,
965            replay_command: link.replay.command_line.clone(),
966        }
967    }
968
969    /// Infer crashpack linkage from normalized provenance when the artifact path
970    /// already points at a crashpack-like artifact.
971    #[must_use]
972    pub fn from_provenance(provenance: &crate::lab::dual_run::ReplayMetadata) -> Option<Self> {
973        let path = provenance.artifact_path.as_ref()?;
974        let file_name = path.rsplit('/').next().unwrap_or(path);
975        if !file_name.contains("crashpack") {
976            return None;
977        }
978        let fingerprint = provenance.trace_fingerprint?;
979        Some(Self {
980            path: path.clone(),
981            id: format!(
982                "crashpack-{:016x}-{:016x}",
983                provenance.effective_seed, fingerprint
984            ),
985            fingerprint,
986            replay_command: provenance
987                .repro_command
988                .clone()
989                .unwrap_or_else(|| provenance.default_repro_command()),
990        })
991    }
992}
993
994/// One runtime-side artifact record inside `differential_failures.json`.
995#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
996pub struct DifferentialFailureArtifact {
997    /// Runtime side that produced the artifact.
998    pub runtime_kind: String,
999    /// Canonical normalized-record path inside the retained bundle.
1000    pub normalized_record_path: String,
1001    /// Optional source artifact path from the original execution.
1002    #[serde(skip_serializing_if = "Option::is_none")]
1003    pub artifact_path: Option<String>,
1004    /// Replay command for rerunning this side.
1005    pub repro_command: String,
1006    /// Crashpack metadata when the source artifact is a crashpack.
1007    #[serde(skip_serializing_if = "Option::is_none")]
1008    pub crashpack_link: Option<DifferentialCrashpackReference>,
1009    /// Side-specific invariant violations.
1010    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1011    pub invariant_violations: Vec<String>,
1012}
1013
1014impl DifferentialFailureArtifact {
1015    #[must_use]
1016    fn from_observable(
1017        observable: &crate::lab::dual_run::NormalizedObservable,
1018        normalized_record_path: impl Into<String>,
1019        invariant_violations: &[String],
1020    ) -> Self {
1021        let repro_command = observable
1022            .provenance
1023            .repro_command
1024            .clone()
1025            .unwrap_or_else(|| observable.provenance.default_repro_command());
1026
1027        Self {
1028            runtime_kind: observable.runtime_kind.to_string(),
1029            normalized_record_path: normalized_record_path.into(),
1030            artifact_path: observable.provenance.artifact_path.clone(),
1031            repro_command,
1032            crashpack_link: DifferentialCrashpackReference::from_provenance(&observable.provenance),
1033            invariant_violations: invariant_violations.to_vec(),
1034        }
1035    }
1036}
1037
1038/// Stable contents for `differential_summary.json`.
1039#[derive(Debug, Clone, Serialize, Deserialize)]
1040pub struct DifferentialSummaryDocument {
1041    /// Stable schema discriminator.
1042    pub schema_version: String,
1043    /// Stable divergence entry identifier.
1044    pub entry_id: String,
1045    /// Scenario identifier.
1046    pub scenario_id: String,
1047    /// Surface identifier.
1048    pub surface_id: String,
1049    /// Surface contract version.
1050    pub surface_contract_version: String,
1051    /// Human-readable verdict summary.
1052    pub verdict_summary: String,
1053    /// Policy-layer summary.
1054    pub policy_summary: String,
1055    /// Divergence class retained for the bundle.
1056    pub divergence_class: String,
1057    /// Final policy class retained for the bundle.
1058    pub policy_class: DifferentialPolicyClass,
1059    /// Current promotion state.
1060    pub regression_promotion_state: RegressionPromotionState,
1061    /// Stable mismatch field names.
1062    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1063    pub mismatch_fields: Vec<String>,
1064    /// Number of mismatch fields retained in the summary.
1065    pub mismatch_count: usize,
1066    /// Whether the underlying run semantically passed.
1067    pub passed: bool,
1068    /// Number of lab-side invariant violations.
1069    pub lab_invariant_violation_count: usize,
1070    /// Number of live-side invariant violations.
1071    pub live_invariant_violation_count: usize,
1072    /// Retained bundle strength.
1073    pub bundle_level: DivergenceBundleLevel,
1074    /// Stable retained bundle root.
1075    pub bundle_root: String,
1076}
1077
1078/// Stable contents for `differential_failures.json`.
1079#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1080pub struct DifferentialFailuresDocument {
1081    /// Stable schema discriminator.
1082    pub schema_version: String,
1083    /// Stable divergence entry identifier.
1084    pub entry_id: String,
1085    /// Scenario identifier.
1086    pub scenario_id: String,
1087    /// Surface identifier.
1088    pub surface_id: String,
1089    /// Runtime-side artifact linkage records.
1090    pub failure_artifacts: Vec<DifferentialFailureArtifact>,
1091}
1092
1093/// Stable contents for `differential_deviations.json`.
1094#[derive(Debug, Clone, Serialize, Deserialize)]
1095pub struct DifferentialDeviationsDocument {
1096    /// Stable schema discriminator.
1097    pub schema_version: String,
1098    /// Stable divergence entry identifier.
1099    pub entry_id: String,
1100    /// Scenario identifier.
1101    pub scenario_id: String,
1102    /// Surface identifier.
1103    pub surface_id: String,
1104    /// Policy-layer summary for the mismatch.
1105    pub policy_summary: String,
1106    /// Provisional divergence class.
1107    pub provisional_class: String,
1108    /// Suggested final divergence class when already known.
1109    #[serde(skip_serializing_if = "Option::is_none")]
1110    pub suggested_final_class: Option<String>,
1111    /// Human-readable explanation for downstream reports.
1112    pub explanation: String,
1113    /// Stable semantic mismatches in field order.
1114    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1115    pub mismatches: Vec<crate::lab::dual_run::SemanticMismatch>,
1116    /// Lab-side invariant violations.
1117    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1118    pub lab_invariant_violations: Vec<String>,
1119    /// Live-side invariant violations.
1120    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1121    pub live_invariant_violations: Vec<String>,
1122}
1123
1124/// Stable contents for `differential_repro_manifest.json`.
1125#[derive(Debug, Clone, Serialize, Deserialize)]
1126pub struct DifferentialReproManifest {
1127    /// Stable schema discriminator.
1128    pub schema_version: String,
1129    /// Stable divergence entry identifier.
1130    pub entry_id: String,
1131    /// Scenario identifier.
1132    pub scenario_id: String,
1133    /// Surface identifier.
1134    pub surface_id: String,
1135    /// Surface contract version.
1136    pub surface_contract_version: String,
1137    /// Divergence class retained for the bundle.
1138    pub divergence_class: String,
1139    /// Final policy class retained for the bundle.
1140    pub policy_class: DifferentialPolicyClass,
1141    /// Current promotion state.
1142    pub regression_promotion_state: RegressionPromotionState,
1143    /// Automatic rerun decision from the policy layer.
1144    pub rerun_decision: crate::lab::dual_run::RerunDecision,
1145    /// Original first-seen run context.
1146    pub first_seen: DivergenceFirstSeenContext,
1147    /// Seed lineage for replay/reproduction.
1148    pub seed_lineage: crate::lab::dual_run::SeedLineageRecord,
1149    /// Shrinker/minimization lineage.
1150    pub minimization_lineage: DivergenceMinimizationLineage,
1151    /// Stable retained bundle root.
1152    pub bundle_root: String,
1153    /// Stable retained summary path.
1154    pub summary_path: String,
1155    /// Stable retained deviations path.
1156    pub deviations_path: String,
1157    /// Stable retained failures path.
1158    pub failure_artifacts_path: String,
1159    /// Stable retained lab normalized observable path.
1160    pub lab_normalized_path: String,
1161    /// Stable retained live normalized observable path.
1162    pub live_normalized_path: String,
1163    /// Stable reproduction commands across both sides.
1164    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1165    pub repro_commands: Vec<String>,
1166    /// Promoted regression scenario identifier when available.
1167    #[serde(skip_serializing_if = "Option::is_none")]
1168    pub promoted_scenario_id: Option<String>,
1169}
1170
1171/// Complete in-memory payload set for a retained divergence bundle.
1172#[derive(Debug, Clone, Serialize, Deserialize)]
1173pub struct DifferentialBundleArtifacts {
1174    /// Summary payload for `differential_summary.json`.
1175    pub summary: DifferentialSummaryDocument,
1176    /// Artifact linkage payload for `differential_failures.json`.
1177    pub failures: DifferentialFailuresDocument,
1178    /// Mismatch/deviation payload for `differential_deviations.json`.
1179    pub deviations: DifferentialDeviationsDocument,
1180    /// Replay/minimization manifest for `differential_repro_manifest.json`.
1181    pub repro_manifest: DifferentialReproManifest,
1182    /// Canonical lab-side normalized observable for `lab_normalized.json`.
1183    pub lab_normalized: crate::lab::dual_run::NormalizedObservable,
1184    /// Canonical live-side normalized observable for `live_normalized.json`.
1185    pub live_normalized: crate::lab::dual_run::NormalizedObservable,
1186}
1187
1188impl DifferentialBundleArtifacts {
1189    /// Build the full retained bundle payload set from a divergence entry and
1190    /// the originating differential result.
1191    #[must_use]
1192    pub fn from_dual_run_result(
1193        entry: &DivergenceCorpusEntry,
1194        result: &crate::lab::dual_run::DualRunResult,
1195    ) -> Self {
1196        let failure_artifacts = vec![
1197            DifferentialFailureArtifact::from_observable(
1198                &result.lab,
1199                entry.artifact_bundle.lab_normalized_path.clone(),
1200                &result.lab_invariant_violations,
1201            ),
1202            DifferentialFailureArtifact::from_observable(
1203                &result.live,
1204                entry.artifact_bundle.live_normalized_path.clone(),
1205                &result.live_invariant_violations,
1206            ),
1207        ];
1208        let mut repro_commands: Vec<String> = failure_artifacts
1209            .iter()
1210            .map(|artifact| artifact.repro_command.clone())
1211            .collect();
1212        repro_commands.sort_unstable();
1213        repro_commands.dedup();
1214
1215        let summary = DifferentialSummaryDocument {
1216            schema_version: DIFFERENTIAL_SUMMARY_SCHEMA_VERSION.to_string(),
1217            entry_id: entry.entry_id.clone(),
1218            scenario_id: entry.scenario_id.clone(),
1219            surface_id: entry.surface_id.clone(),
1220            surface_contract_version: entry.surface_contract_version.clone(),
1221            verdict_summary: result.verdict.summary(),
1222            policy_summary: result.policy.summary(),
1223            divergence_class: entry.divergence_class.clone(),
1224            policy_class: entry.policy_class,
1225            regression_promotion_state: entry.regression_promotion_state,
1226            mismatch_fields: entry.mismatch_fields.clone(),
1227            mismatch_count: entry.mismatch_fields.len(),
1228            passed: result.passed(),
1229            lab_invariant_violation_count: result.lab_invariant_violations.len(),
1230            live_invariant_violation_count: result.live_invariant_violations.len(),
1231            bundle_level: entry.retention.bundle_level,
1232            bundle_root: entry.artifact_bundle.bundle_root.clone(),
1233        };
1234
1235        let failures = DifferentialFailuresDocument {
1236            schema_version: DIFFERENTIAL_FAILURES_SCHEMA_VERSION.to_string(),
1237            entry_id: entry.entry_id.clone(),
1238            scenario_id: entry.scenario_id.clone(),
1239            surface_id: entry.surface_id.clone(),
1240            failure_artifacts,
1241        };
1242
1243        let deviations = DifferentialDeviationsDocument {
1244            schema_version: DIFFERENTIAL_DEVIATIONS_SCHEMA_VERSION.to_string(),
1245            entry_id: entry.entry_id.clone(),
1246            scenario_id: entry.scenario_id.clone(),
1247            surface_id: entry.surface_id.clone(),
1248            policy_summary: result.policy.summary(),
1249            provisional_class: result.policy.provisional_class.to_string(),
1250            suggested_final_class: result
1251                .policy
1252                .suggested_final_class
1253                .map(|class| class.to_string()),
1254            explanation: result.policy.explanation.clone(),
1255            mismatches: result.verdict.mismatches.clone(),
1256            lab_invariant_violations: result.lab_invariant_violations.clone(),
1257            live_invariant_violations: result.live_invariant_violations.clone(),
1258        };
1259
1260        let repro_manifest = DifferentialReproManifest {
1261            schema_version: DIFFERENTIAL_REPRO_MANIFEST_SCHEMA_VERSION.to_string(),
1262            entry_id: entry.entry_id.clone(),
1263            scenario_id: entry.scenario_id.clone(),
1264            surface_id: entry.surface_id.clone(),
1265            surface_contract_version: entry.surface_contract_version.clone(),
1266            divergence_class: entry.divergence_class.clone(),
1267            policy_class: entry.policy_class,
1268            regression_promotion_state: entry.regression_promotion_state,
1269            rerun_decision: result.policy.rerun_decision,
1270            first_seen: entry.first_seen.clone(),
1271            seed_lineage: entry.seed_lineage.clone(),
1272            minimization_lineage: entry.minimization_lineage.clone(),
1273            bundle_root: entry.artifact_bundle.bundle_root.clone(),
1274            summary_path: entry.artifact_bundle.differential_summary_path.clone(),
1275            deviations_path: entry.artifact_bundle.differential_deviations_path.clone(),
1276            failure_artifacts_path: entry.artifact_bundle.differential_failures_path.clone(),
1277            lab_normalized_path: entry.artifact_bundle.lab_normalized_path.clone(),
1278            live_normalized_path: entry.artifact_bundle.live_normalized_path.clone(),
1279            repro_commands,
1280            promoted_scenario_id: entry.metadata.get("promoted_scenario_id").cloned(),
1281        };
1282
1283        Self {
1284            summary,
1285            failures,
1286            deviations,
1287            repro_manifest,
1288            lab_normalized: result.lab.clone(),
1289            live_normalized: result.live.clone(),
1290        }
1291    }
1292}
1293
1294fn sanitize_registry_component(input: &str) -> String {
1295    let mut out = String::with_capacity(input.len());
1296    for ch in input.chars() {
1297        if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' {
1298            out.push(ch);
1299        } else {
1300            out.push('_');
1301        }
1302    }
1303    out.trim_matches('_').to_string()
1304}
1305
1306// ============================================================================
1307// Trace Normalization for Canonical Replay
1308// ============================================================================
1309
1310/// Result of trace normalization.
1311#[derive(Debug, Clone)]
1312pub struct NormalizationResult {
1313    /// The normalized (reordered) trace events.
1314    pub normalized: Vec<TraceEvent>,
1315    /// Number of owner switches in the original trace.
1316    pub original_switches: usize,
1317    /// Number of owner switches after normalization.
1318    pub normalized_switches: usize,
1319    /// The algorithm used for normalization.
1320    pub algorithm: String,
1321}
1322
1323impl NormalizationResult {
1324    /// Returns the reduction in switch count.
1325    #[must_use]
1326    pub fn switch_reduction(&self) -> usize {
1327        self.original_switches
1328            .saturating_sub(self.normalized_switches)
1329    }
1330
1331    /// Returns the switch reduction as a percentage.
1332    #[must_use]
1333    #[allow(clippy::cast_precision_loss)]
1334    pub fn switch_reduction_pct(&self) -> f64 {
1335        if self.original_switches == 0 {
1336            0.0
1337        } else {
1338            (self.switch_reduction() as f64 / self.original_switches as f64) * 100.0
1339        }
1340    }
1341}
1342
1343impl std::fmt::Display for NormalizationResult {
1344    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1345        write!(
1346            f,
1347            "Normalized {} events: {} → {} switches ({:.1}% reduction, {})",
1348            self.normalized.len(),
1349            self.original_switches,
1350            self.normalized_switches,
1351            self.switch_reduction_pct(),
1352            self.algorithm
1353        )
1354    }
1355}
1356
1357/// Normalize a trace for canonical replay ordering.
1358///
1359/// This reorders trace events to minimize context switches while preserving
1360/// all happens-before relationships. The result is a canonical form suitable
1361/// for:
1362/// - Deterministic replay comparison
1363/// - Debugging (reduced noise from interleaving)
1364/// - Trace minimization
1365///
1366/// # Example
1367///
1368/// ```ignore
1369/// use asupersync::lab::replay::normalize_for_replay;
1370///
1371/// let events: Vec<TraceEvent> = /* captured trace */;
1372/// let result = normalize_for_replay(&events);
1373/// println!("{}", result); // Shows switch reduction
1374/// ```
1375#[must_use]
1376pub fn normalize_for_replay(events: &[TraceEvent]) -> NormalizationResult {
1377    normalize_for_replay_with_config(events, &crate::trace::GeodesicConfig::default())
1378}
1379
1380/// Normalize a trace with custom configuration.
1381///
1382/// See [`GeodesicConfig`] for available options:
1383/// - `beam_threshold`: Trace size above which beam search is used
1384/// - `beam_width`: Width of beam search
1385/// - `step_budget`: Maximum search steps
1386#[must_use]
1387pub fn normalize_for_replay_with_config(
1388    events: &[TraceEvent],
1389    config: &crate::trace::GeodesicConfig,
1390) -> NormalizationResult {
1391    let original_switches = crate::trace::trace_switch_cost(events);
1392    let (normalized, geodesic_result) = crate::trace::normalize_trace(events, config);
1393
1394    NormalizationResult {
1395        normalized,
1396        original_switches,
1397        normalized_switches: geodesic_result.switch_count,
1398        algorithm: format!("{:?}", geodesic_result.algorithm),
1399    }
1400}
1401
1402/// Compare two traces for equivalence after normalization.
1403///
1404/// Two traces are considered equivalent if their normalized forms produce
1405/// the same sequence of events (respecting happens-before ordering).
1406///
1407/// Returns `None` if the traces are equivalent, or `Some(divergence)` if
1408/// they differ.
1409#[must_use]
1410pub fn compare_normalized(a: &[TraceEvent], b: &[TraceEvent]) -> Option<TraceDivergence> {
1411    let norm_a = normalize_for_replay(a);
1412    let norm_b = normalize_for_replay(b);
1413    find_divergence(&norm_a.normalized, &norm_b.normalized)
1414}
1415
1416/// Check if two traces are equivalent under normalization.
1417///
1418/// This is a convenience wrapper around [`compare_normalized`].
1419#[must_use]
1420pub fn traces_equivalent(a: &[TraceEvent], b: &[TraceEvent]) -> bool {
1421    compare_normalized(a, b).is_none()
1422}
1423
1424#[cfg(test)]
1425mod tests {
1426    use super::*;
1427    use crate::app::AppSpec;
1428    use crate::lab::SporkScenarioSpec;
1429    use crate::trace::event::{TraceData, TraceEventKind};
1430    use crate::types::Budget;
1431    use crate::types::Time;
1432
1433    fn init_test(name: &str) {
1434        crate::test_utils::init_test_logging();
1435        crate::test_phase!(name);
1436    }
1437
1438    fn trace_message_contains(event: &TraceEvent, needle: &str) -> bool {
1439        matches!(&event.data, TraceData::Message(message) if message.contains(needle))
1440    }
1441
1442    #[test]
1443    fn identical_traces_no_divergence() {
1444        init_test("identical_traces_no_divergence");
1445        let a = vec![TraceEvent::new(
1446            1,
1447            Time::ZERO,
1448            TraceEventKind::UserTrace,
1449            TraceData::None,
1450        )];
1451        let b = vec![TraceEvent::new(
1452            1,
1453            Time::ZERO,
1454            TraceEventKind::UserTrace,
1455            TraceData::None,
1456        )];
1457
1458        let div = find_divergence(&a, &b);
1459        let ok = div.is_none();
1460        crate::assert_with_log!(ok, "no divergence", true, ok);
1461        crate::test_complete!("identical_traces_no_divergence");
1462    }
1463
1464    #[test]
1465    fn trace_seq_only_difference_no_divergence() {
1466        init_test("trace_seq_only_difference_no_divergence");
1467        let a = vec![TraceEvent::new(
1468            1,
1469            Time::ZERO,
1470            TraceEventKind::UserTrace,
1471            TraceData::Message("same".to_string()),
1472        )];
1473        let b = vec![TraceEvent::new(
1474            99,
1475            Time::ZERO,
1476            TraceEventKind::UserTrace,
1477            TraceData::Message("same".to_string()),
1478        )];
1479
1480        let div = find_divergence(&a, &b);
1481        let ok = div.is_none();
1482        crate::assert_with_log!(ok, "seq-only differences ignored", true, ok);
1483        crate::test_complete!("trace_seq_only_difference_no_divergence");
1484    }
1485
1486    #[test]
1487    fn different_traces_find_divergence() {
1488        init_test("different_traces_find_divergence");
1489        let a = vec![TraceEvent::new(
1490            1,
1491            Time::ZERO,
1492            TraceEventKind::Spawn,
1493            TraceData::None,
1494        )];
1495        let b = vec![TraceEvent::new(
1496            1,
1497            Time::ZERO,
1498            TraceEventKind::Complete,
1499            TraceData::None,
1500        )];
1501
1502        let div = find_divergence(&a, &b);
1503        let some = div.is_some();
1504        crate::assert_with_log!(some, "divergence", true, some);
1505        let pos = div.expect("divergence").position;
1506        crate::assert_with_log!(pos == 0, "position", 0, pos);
1507        crate::test_complete!("different_traces_find_divergence");
1508    }
1509
1510    #[test]
1511    fn different_traces_find_divergence_data() {
1512        init_test("different_traces_find_divergence_data");
1513        let a = vec![TraceEvent::new(
1514            1,
1515            Time::ZERO,
1516            TraceEventKind::UserTrace,
1517            TraceData::Message("a".to_string()),
1518        )];
1519        let b = vec![TraceEvent::new(
1520            1,
1521            Time::ZERO,
1522            TraceEventKind::UserTrace,
1523            TraceData::Message("b".to_string()),
1524        )];
1525
1526        let div = find_divergence(&a, &b);
1527        let some = div.is_some();
1528        crate::assert_with_log!(some, "divergence", true, some);
1529        let pos = div.expect("divergence").position;
1530        crate::assert_with_log!(pos == 0, "position", 0, pos);
1531        crate::test_complete!("different_traces_find_divergence_data");
1532    }
1533
1534    // ── Replay validation tests ─────────────────────────────────────────
1535
1536    #[test]
1537    fn replay_single_task_deterministic() {
1538        use crate::types::Budget;
1539        let validation = validate_replay(42, 1, |runtime| {
1540            let region = runtime.state.create_root_region(Budget::INFINITE);
1541            let (t, _) = runtime
1542                .state
1543                .create_task(region, Budget::INFINITE, async { 1 })
1544                .expect("t");
1545            runtime.scheduler.lock().schedule(t, 0);
1546            runtime.run_until_quiescent();
1547        });
1548
1549        assert!(validation.is_valid(), "Replay failed: {validation}");
1550        assert_eq!(
1551            validation.original_certificate,
1552            validation.replay_certificate
1553        );
1554        assert_eq!(validation.original_steps, validation.replay_steps);
1555    }
1556
1557    #[test]
1558    fn replay_two_tasks_deterministic() {
1559        use crate::types::Budget;
1560        let validation = validate_replay(0, 1, |runtime| {
1561            let region = runtime.state.create_root_region(Budget::INFINITE);
1562            let (t1, _) = runtime
1563                .state
1564                .create_task(region, Budget::INFINITE, async {})
1565                .expect("t1");
1566            let (t2, _) = runtime
1567                .state
1568                .create_task(region, Budget::INFINITE, async {})
1569                .expect("t2");
1570            {
1571                let mut sched = runtime.scheduler.lock();
1572                sched.schedule(t1, 0);
1573                sched.schedule(t2, 0);
1574            }
1575            runtime.run_until_quiescent();
1576        });
1577
1578        assert!(validation.is_valid(), "Replay failed: {validation}");
1579    }
1580
1581    #[test]
1582    fn replay_multi_seeds_all_deterministic() {
1583        use crate::types::Budget;
1584        let seeds: Vec<u64> = (0..10).collect();
1585        let results = validate_replay_multi(&seeds, 1, |runtime| {
1586            let region = runtime.state.create_root_region(Budget::INFINITE);
1587            let (t, _) = runtime
1588                .state
1589                .create_task(region, Budget::INFINITE, async { 42 })
1590                .expect("t");
1591            runtime.scheduler.lock().schedule(t, 0);
1592            runtime.run_until_quiescent();
1593        });
1594
1595        for (i, v) in results.iter().enumerate() {
1596            assert!(v.is_valid(), "Seed {} replay failed: {v}", seeds[i]);
1597        }
1598    }
1599
1600    #[test]
1601    fn replay_validation_display_ok() {
1602        let v = ReplayValidation {
1603            matched: true,
1604            original_certificate: 0x1234,
1605            replay_certificate: 0x1234,
1606            divergence: None,
1607            original_steps: 5,
1608            replay_steps: 5,
1609        };
1610        let s = format!("{v}");
1611        assert!(s.contains("Replay OK"));
1612    }
1613
1614    #[test]
1615    fn replay_validation_display_diverged() {
1616        let v = ReplayValidation {
1617            matched: false,
1618            original_certificate: 0x1234,
1619            replay_certificate: 0x5678,
1620            divergence: None,
1621            original_steps: 5,
1622            replay_steps: 5,
1623        };
1624        let s = format!("{v}");
1625        assert!(s.contains("DIVERGED"));
1626        assert!(s.contains("Certificate mismatch"));
1627    }
1628
1629    // ── Normalization tests ─────────────────────────────────────────────
1630
1631    #[test]
1632    fn normalization_single_owner_no_switches() {
1633        init_test("normalization_single_owner_no_switches");
1634        // All events from owner 1 - should have 0 switches
1635        let events = vec![
1636            TraceEvent::new(
1637                1,
1638                Time::from_nanos(0),
1639                TraceEventKind::Spawn,
1640                TraceData::None,
1641            ),
1642            TraceEvent::new(
1643                2,
1644                Time::from_nanos(1),
1645                TraceEventKind::Poll,
1646                TraceData::None,
1647            ),
1648            TraceEvent::new(
1649                3,
1650                Time::from_nanos(2),
1651                TraceEventKind::Complete,
1652                TraceData::None,
1653            ),
1654        ];
1655        // All have seq numbers, but owner extraction uses seq % some_value or similar
1656        // The trace module should handle this; we're testing the wrapper
1657
1658        let result = normalize_for_replay(&events);
1659        // Single-owner trace has no switches before or after
1660        assert_eq!(result.switch_reduction(), 0);
1661        crate::test_complete!("normalization_single_owner_no_switches");
1662    }
1663
1664    #[test]
1665    fn normalization_result_display() {
1666        init_test("normalization_result_display");
1667        let result = NormalizationResult {
1668            normalized: vec![],
1669            original_switches: 10,
1670            normalized_switches: 3,
1671            algorithm: "Greedy".to_string(),
1672        };
1673
1674        let display = format!("{result}");
1675        assert!(display.contains("10 → 3 switches"));
1676        assert!(display.contains("70.0% reduction"));
1677        assert!(display.contains("Greedy"));
1678        crate::test_complete!("normalization_result_display");
1679    }
1680
1681    #[test]
1682    fn normalization_result_zero_switches() {
1683        init_test("normalization_result_zero_switches");
1684        let result = NormalizationResult {
1685            normalized: vec![],
1686            original_switches: 0,
1687            normalized_switches: 0,
1688            algorithm: "Trivial".to_string(),
1689        };
1690
1691        // Avoid division by zero
1692        let pct = result.switch_reduction_pct();
1693        assert!((pct - 0.0).abs() < f64::EPSILON);
1694        crate::test_complete!("normalization_result_zero_switches");
1695    }
1696
1697    #[test]
1698    fn traces_equivalent_identical() {
1699        init_test("traces_equivalent_identical");
1700        let events = vec![
1701            TraceEvent::new(
1702                1,
1703                Time::from_nanos(0),
1704                TraceEventKind::Spawn,
1705                TraceData::None,
1706            ),
1707            TraceEvent::new(
1708                2,
1709                Time::from_nanos(1),
1710                TraceEventKind::Complete,
1711                TraceData::None,
1712            ),
1713        ];
1714
1715        let equivalent = traces_equivalent(&events, &events);
1716        crate::assert_with_log!(equivalent, "identical traces equivalent", true, equivalent);
1717        crate::test_complete!("traces_equivalent_identical");
1718    }
1719
1720    #[test]
1721    fn traces_equivalent_ignores_sequence_numbers() {
1722        init_test("traces_equivalent_ignores_sequence_numbers");
1723        let a = vec![TraceEvent::new(
1724            1,
1725            Time::from_nanos(0),
1726            TraceEventKind::Spawn,
1727            TraceData::None,
1728        )];
1729        let b = vec![TraceEvent::new(
1730            42,
1731            Time::from_nanos(0),
1732            TraceEventKind::Spawn,
1733            TraceData::None,
1734        )];
1735
1736        let equivalent = traces_equivalent(&a, &b);
1737        crate::assert_with_log!(
1738            equivalent,
1739            "seq-only differences still equivalent",
1740            true,
1741            equivalent
1742        );
1743        crate::test_complete!("traces_equivalent_ignores_sequence_numbers");
1744    }
1745
1746    #[test]
1747    fn traces_equivalent_different_kinds() {
1748        init_test("traces_equivalent_different_kinds");
1749        let a = vec![TraceEvent::new(
1750            1,
1751            Time::from_nanos(0),
1752            TraceEventKind::Spawn,
1753            TraceData::None,
1754        )];
1755        let b = vec![TraceEvent::new(
1756            1,
1757            Time::from_nanos(0),
1758            TraceEventKind::Complete,
1759            TraceData::None,
1760        )];
1761
1762        let equivalent = traces_equivalent(&a, &b);
1763        crate::assert_with_log!(
1764            !equivalent,
1765            "different kinds not equivalent",
1766            false,
1767            equivalent
1768        );
1769        crate::test_complete!("traces_equivalent_different_kinds");
1770    }
1771
1772    #[test]
1773    fn compare_normalized_returns_divergence() {
1774        init_test("compare_normalized_returns_divergence");
1775        let a = vec![TraceEvent::new(
1776            1,
1777            Time::from_nanos(0),
1778            TraceEventKind::Spawn,
1779            TraceData::None,
1780        )];
1781        let b = vec![TraceEvent::new(
1782            1,
1783            Time::from_nanos(0),
1784            TraceEventKind::Complete,
1785            TraceData::None,
1786        )];
1787
1788        let divergence = compare_normalized(&a, &b);
1789        let has_div = divergence.is_some();
1790        crate::assert_with_log!(has_div, "divergence found", true, has_div);
1791        crate::test_complete!("compare_normalized_returns_divergence");
1792    }
1793
1794    #[test]
1795    fn normalize_with_config_custom_beam() {
1796        use crate::trace::GeodesicConfig;
1797
1798        init_test("normalize_with_config_custom_beam");
1799        let events = vec![
1800            TraceEvent::new(
1801                1,
1802                Time::from_nanos(0),
1803                TraceEventKind::Spawn,
1804                TraceData::None,
1805            ),
1806            TraceEvent::new(
1807                2,
1808                Time::from_nanos(1),
1809                TraceEventKind::Poll,
1810                TraceData::None,
1811            ),
1812        ];
1813
1814        let config = GeodesicConfig {
1815            exact_threshold: 0,
1816            beam_threshold: 1,
1817            beam_width: 4,
1818            step_budget: 100,
1819        };
1820
1821        let result = normalize_for_replay_with_config(&events, &config);
1822        // Just verify it runs without panic; algorithm choice depends on trace size
1823        assert!(!result.algorithm.is_empty());
1824        crate::test_complete!("normalize_with_config_custom_beam");
1825    }
1826
1827    #[test]
1828    fn classify_fingerprint_classes_is_deterministic() {
1829        init_test("classify_fingerprint_classes_is_deterministic");
1830
1831        let runs = vec![
1832            ExplorationRunSummary {
1833                seed: 9,
1834                schedule_hash: 0xB,
1835                trace_fingerprint: 0xAA,
1836            },
1837            ExplorationRunSummary {
1838                seed: 3,
1839                schedule_hash: 0xA,
1840                trace_fingerprint: 0xBB,
1841            },
1842            ExplorationRunSummary {
1843                seed: 7,
1844                schedule_hash: 0xC,
1845                trace_fingerprint: 0xAA,
1846            },
1847            ExplorationRunSummary {
1848                seed: 7,
1849                schedule_hash: 0xC,
1850                trace_fingerprint: 0xAA,
1851            },
1852        ];
1853
1854        let classes = classify_fingerprint_classes(&runs);
1855        assert_eq!(classes.len(), 2);
1856        assert_eq!(classes[0].trace_fingerprint, 0xAA);
1857        assert_eq!(classes[0].run_count, 3);
1858        assert_eq!(classes[0].seeds, vec![7, 9]);
1859        assert_eq!(classes[0].schedule_hashes, vec![0xB, 0xC]);
1860        assert_eq!(classes[1].trace_fingerprint, 0xBB);
1861        assert_eq!(classes[1].run_count, 1);
1862        assert_eq!(classes[1].seeds, vec![3]);
1863        assert_eq!(classes[1].schedule_hashes, vec![0xA]);
1864
1865        crate::test_complete!("classify_fingerprint_classes_is_deterministic");
1866    }
1867
1868    #[test]
1869    fn explore_seed_space_is_deterministic_for_same_inputs() {
1870        init_test("explore_seed_space_is_deterministic_for_same_inputs");
1871
1872        let seeds = [11_u64, 13_u64, 11_u64];
1873        let scenario = |runtime: &mut LabRuntime| {
1874            let region = runtime.state.create_root_region(Budget::INFINITE);
1875            let (task, _) = runtime
1876                .state
1877                .create_task(region, Budget::INFINITE, async {})
1878                .expect("task");
1879            runtime.scheduler.lock().schedule(task, 0);
1880            runtime.run_until_quiescent();
1881        };
1882
1883        let a = explore_seed_space(&seeds, 1, scenario);
1884        let b = explore_seed_space(&seeds, 1, scenario);
1885
1886        assert_eq!(a, b, "same seeds and scenario must produce same report");
1887        assert_eq!(a.runs.len(), seeds.len());
1888        assert!(a.unique_fingerprint_count() >= 1);
1889
1890        crate::test_complete!("explore_seed_space_is_deterministic_for_same_inputs");
1891    }
1892
1893    fn make_spork_report(seed: u64, failing: bool) -> SporkHarnessReport {
1894        use crate::record::ObligationKind;
1895
1896        let config = LabConfig::new(seed).panic_on_leak(false);
1897        let mut runtime = LabRuntime::new(config);
1898        let region = runtime.state.create_root_region(Budget::INFINITE);
1899        let (task, _) = runtime
1900            .state
1901            .create_task(region, Budget::INFINITE, async {})
1902            .expect("create task");
1903        runtime.scheduler.lock().schedule(task, 0);
1904        // Create the obligation while the task is still live so the holder
1905        // validation passes.  Running to quiescence afterward leaves the
1906        // obligation unresolved (intentional leak → failing report).
1907        if failing {
1908            runtime
1909                .state
1910                .create_obligation(
1911                    ObligationKind::SendPermit,
1912                    task,
1913                    region,
1914                    Some("intentional failure for exploration".to_string()),
1915                )
1916                .expect("create failing obligation");
1917        }
1918        runtime.run_until_quiescent();
1919
1920        runtime.spork_report("spork_exploration", Vec::new())
1921    }
1922
1923    #[test]
1924    fn summarize_spork_reports_links_failures_to_crashpacks() {
1925        init_test("summarize_spork_reports_links_failures_to_crashpacks");
1926
1927        let passing = make_spork_report(31, false);
1928        let failing = make_spork_report(32, true);
1929
1930        let summary = summarize_spork_reports(&[failing, passing]);
1931        assert_eq!(summary.runs.len(), 2);
1932        assert_eq!(summary.failure_count(), 1);
1933        assert!(summary.unique_fingerprint_count() >= 1);
1934        assert!(
1935            summary.all_failures_linked_to_crashpacks(),
1936            "failed runs must include crashpack linkage metadata"
1937        );
1938
1939        let failed_run = summary
1940            .runs
1941            .iter()
1942            .find(|run| !run.passed)
1943            .expect("one failing run expected");
1944        let crashpack = failed_run
1945            .crashpack_link
1946            .as_ref()
1947            .expect("failing run should have crashpack link");
1948        assert!(
1949            crashpack.path.starts_with("crashpack-"),
1950            "unexpected crashpack path: {}",
1951            crashpack.path
1952        );
1953
1954        crate::test_complete!("summarize_spork_reports_links_failures_to_crashpacks");
1955    }
1956
1957    #[test]
1958    fn explore_spork_seed_space_is_deterministic() {
1959        init_test("explore_spork_seed_space_is_deterministic");
1960
1961        let seeds = [42_u64, 41_u64, 42_u64];
1962
1963        let run_for_seed = |seed: u64| make_spork_report(seed, seed.is_multiple_of(2));
1964        let a = explore_spork_seed_space(&seeds, run_for_seed);
1965
1966        let run_for_seed = |seed: u64| make_spork_report(seed, seed.is_multiple_of(2));
1967        let b = explore_spork_seed_space(&seeds, run_for_seed);
1968
1969        assert_eq!(a, b, "same seeds must produce deterministic report");
1970        assert_eq!(a.runs.len(), seeds.len());
1971        assert_eq!(a.failure_count(), 2);
1972        assert!(a.unique_fingerprint_count() >= 1);
1973        assert!(a.all_failures_linked_to_crashpacks());
1974
1975        crate::test_complete!("explore_spork_seed_space_is_deterministic");
1976    }
1977
1978    #[test]
1979    fn scenario_runner_exploration_has_deterministic_fingerprints() {
1980        init_test("scenario_runner_exploration_has_deterministic_fingerprints");
1981
1982        let mut runner = SporkScenarioRunner::new();
1983        runner
1984            .register(
1985                SporkScenarioSpec::new("replay.scenario", |_| AppSpec::new("replay_app"))
1986                    .with_default_config(SporkScenarioConfig::default()),
1987            )
1988            .expect("register scenario");
1989
1990        let base_config = SporkScenarioConfig::default();
1991        let seeds = [12_u64, 13_u64, 12_u64];
1992
1993        let a =
1994            explore_scenario_runner_seed_space(&runner, "replay.scenario", &base_config, &seeds)
1995                .expect("exploration A");
1996        let b =
1997            explore_scenario_runner_seed_space(&runner, "replay.scenario", &base_config, &seeds)
1998                .expect("exploration B");
1999
2000        assert_eq!(a, b, "scenario exploration must be deterministic");
2001        assert_eq!(a.runs.len(), seeds.len());
2002        assert!(a.unique_fingerprint_count() >= 1);
2003
2004        // Same seed should map to the same fingerprint.
2005        let seed_12: Vec<_> = a.runs.iter().filter(|run| run.seed == 12).collect();
2006        assert_eq!(seed_12.len(), 2);
2007        assert_eq!(seed_12[0].trace_fingerprint, seed_12[1].trace_fingerprint);
2008
2009        crate::test_complete!("scenario_runner_exploration_has_deterministic_fingerprints");
2010    }
2011
2012    fn make_dual_run_divergence_result() -> crate::lab::dual_run::DualRunResult {
2013        use crate::lab::dual_run::{
2014            CancellationRecord, DualRunHarness, LoserDrainRecord, ObligationBalanceRecord,
2015            RegionCloseRecord, ResourceSurfaceRecord, TerminalOutcome,
2016        };
2017
2018        fn base_semantics() -> crate::lab::dual_run::NormalizedSemantics {
2019            crate::lab::dual_run::NormalizedSemantics {
2020                terminal_outcome: TerminalOutcome::ok(),
2021                cancellation: CancellationRecord::none(),
2022                loser_drain: LoserDrainRecord::not_applicable(),
2023                region_close: RegionCloseRecord::quiescent(),
2024                obligation_balance: ObligationBalanceRecord::zero(),
2025                resource_surface: ResourceSurfaceRecord::empty("test.surface"),
2026            }
2027        }
2028
2029        let mut result = DualRunHarness::phase1(
2030            "divergence.registry.case",
2031            "test.surface",
2032            "v1",
2033            "Divergence corpus registry coverage",
2034            0xD1,
2035        )
2036        .lab(|_config| base_semantics())
2037        .live(|_seed, _entropy| {
2038            let mut sem = base_semantics();
2039            sem.obligation_balance = ObligationBalanceRecord {
2040                reserved: 1,
2041                committed: 0,
2042                aborted: 0,
2043                leaked: 1,
2044                unresolved: 0,
2045                balanced: false,
2046            };
2047            sem
2048        })
2049        .run();
2050
2051        let mut lab_provenance = result
2052            .lab
2053            .provenance
2054            .clone()
2055            .with_artifact_path("crashpack-divergence.registry.case.json")
2056            .with_repro_command("cargo test divergence.registry.case -- --nocapture");
2057        if lab_provenance.trace_fingerprint.is_none() {
2058            lab_provenance.trace_fingerprint = Some(0xC0DE_CAFE);
2059        }
2060        result.lab.provenance = lab_provenance;
2061
2062        let mut live_provenance = result
2063            .live
2064            .provenance
2065            .clone()
2066            .with_artifact_path("artifacts/live/divergence.registry.case.json")
2067            .with_repro_command("cargo test divergence.registry.case -- --nocapture --live");
2068        if live_provenance.trace_fingerprint.is_none() {
2069            live_provenance.trace_fingerprint = Some(0xBEEF_BAAD);
2070        }
2071        result.live.provenance = live_provenance;
2072        result
2073    }
2074
2075    #[test]
2076    fn divergence_artifact_bundle_uses_stable_bundle_layout() {
2077        init_test("divergence_artifact_bundle_uses_stable_bundle_layout");
2078
2079        let bundle = DivergenceArtifactBundle::under("artifacts/differential/run-001");
2080        assert_eq!(
2081            bundle.differential_summary_path,
2082            "artifacts/differential/run-001/differential_summary.json"
2083        );
2084        assert_eq!(
2085            bundle.live_normalized_path,
2086            "artifacts/differential/run-001/live_normalized.json"
2087        );
2088
2089        crate::test_complete!("divergence_artifact_bundle_uses_stable_bundle_layout");
2090    }
2091
2092    #[test]
2093    fn divergence_retention_defaults_follow_policy_class() {
2094        init_test("divergence_retention_defaults_follow_policy_class");
2095
2096        let full = DivergenceRetentionMetadata::for_policy_class(
2097            DifferentialPolicyClass::RuntimeSemanticBug,
2098        );
2099        assert_eq!(full.bundle_level, DivergenceBundleLevel::Full);
2100        assert_eq!(full.local_retention_days, 14);
2101        assert_eq!(full.ci_retention_days, 30);
2102        assert_eq!(full.redaction_mode, "metadata_only");
2103
2104        let reduced = DivergenceRetentionMetadata::for_policy_class(
2105            DifferentialPolicyClass::UnsupportedSurface,
2106        );
2107        assert_eq!(reduced.bundle_level, DivergenceBundleLevel::Reduced);
2108
2109        crate::test_complete!("divergence_retention_defaults_follow_policy_class");
2110    }
2111
2112    #[test]
2113    fn divergence_corpus_entry_tracks_lineage_and_promotion_state() {
2114        init_test("divergence_corpus_entry_tracks_lineage_and_promotion_state");
2115
2116        let result = make_dual_run_divergence_result();
2117        assert!(!result.passed(), "test fixture must produce a divergence");
2118
2119        let entry = DivergenceCorpusEntry::from_dual_run_result(
2120            &result,
2121            "pilot_surface",
2122            "obligation_balance_mismatch",
2123            DifferentialPolicyClass::RuntimeSemanticBug,
2124            "artifacts/differential/test-run",
2125        )
2126        .with_first_seen_attempt(2, 1)
2127        .with_minimization_lineage(
2128            DivergenceMinimizationLineage::from_seed_lineage(&result.seed_lineage)
2129                .with_minimized_seed(0x2A, "prefix_shrinker", true, true),
2130        )
2131        .promote_to_regression("regression.test.surface.obligation_leak.seed_2a");
2132
2133        assert_eq!(
2134            entry.policy_class,
2135            DifferentialPolicyClass::RuntimeSemanticBug
2136        );
2137        assert_eq!(entry.first_seen.runner_profile, "pilot_surface");
2138        assert_eq!(entry.first_seen.attempt_index, 2);
2139        assert_eq!(entry.first_seen.rerun_count, 1);
2140        assert_eq!(
2141            entry.minimization_lineage.shrink_status,
2142            DivergenceShrinkStatus::PreservedSemanticClass
2143        );
2144        assert_eq!(
2145            entry.regression_promotion_state,
2146            RegressionPromotionState::PromotedRegression
2147        );
2148        assert_eq!(
2149            entry.metadata.get("promoted_scenario_id"),
2150            Some(&"regression.test.surface.obligation_leak.seed_2a".to_string())
2151        );
2152        assert!(
2153            entry
2154                .mismatch_fields
2155                .contains(&"semantics.obligation_balance.balanced".to_string()),
2156            "mismatch fields should retain the semantic mismatch path"
2157        );
2158        assert!(
2159            entry
2160                .artifact_bundle
2161                .differential_repro_manifest_path
2162                .ends_with("differential_repro_manifest.json")
2163        );
2164        assert_eq!(
2165            entry.artifact_bundle.bundle_root,
2166            "artifacts/differential/test-run"
2167        );
2168
2169        crate::test_complete!("divergence_corpus_entry_tracks_lineage_and_promotion_state");
2170    }
2171
2172    #[test]
2173    fn divergence_registry_upsert_is_deterministic() {
2174        init_test("divergence_registry_upsert_is_deterministic");
2175
2176        let result = make_dual_run_divergence_result();
2177        let entry = DivergenceCorpusEntry::from_dual_run_result(
2178            &result,
2179            "nightly",
2180            "obligation_balance_mismatch",
2181            DifferentialPolicyClass::RuntimeSemanticBug,
2182            "artifacts/differential/nightly-case",
2183        );
2184
2185        let mut registry = DivergenceCorpusRegistry::new();
2186        registry.upsert(entry.clone());
2187        registry.upsert(entry.promote_to_regression("regression.promoted"));
2188
2189        assert_eq!(registry.schema_version, DIVERGENCE_CORPUS_SCHEMA_VERSION);
2190        assert_eq!(registry.entries.len(), 1);
2191        assert_eq!(
2192            registry.entries[0].regression_promotion_state,
2193            RegressionPromotionState::PromotedRegression
2194        );
2195
2196        crate::test_complete!("divergence_registry_upsert_is_deterministic");
2197    }
2198
2199    #[test]
2200    fn differential_bundle_artifacts_capture_repro_and_minimization_lineage() {
2201        init_test("differential_bundle_artifacts_capture_repro_and_minimization_lineage");
2202
2203        let result = make_dual_run_divergence_result();
2204        let entry = DivergenceCorpusEntry::from_dual_run_result(
2205            &result,
2206            "nightly",
2207            "obligation_balance_mismatch",
2208            DifferentialPolicyClass::RuntimeSemanticBug,
2209            "artifacts/differential/nightly/divergence.registry.case",
2210        )
2211        .with_first_seen_attempt(3, 2)
2212        .with_minimization_lineage(
2213            DivergenceMinimizationLineage::from_seed_lineage(&result.seed_lineage)
2214                .with_minimized_seed(0x2A, "prefix_shrinker", true, true),
2215        )
2216        .promote_to_regression("regression.test.surface.obligation_leak.seed_2a");
2217
2218        let bundle = DifferentialBundleArtifacts::from_dual_run_result(&entry, &result);
2219        assert_eq!(
2220            bundle.summary.schema_version,
2221            DIFFERENTIAL_SUMMARY_SCHEMA_VERSION
2222        );
2223        assert_eq!(
2224            bundle.summary.bundle_root,
2225            "artifacts/differential/nightly/divergence.registry.case"
2226        );
2227        assert_eq!(bundle.failures.failure_artifacts.len(), 2);
2228        assert_eq!(
2229            bundle.failures.failure_artifacts[0].runtime_kind,
2230            "lab".to_string()
2231        );
2232        assert_eq!(
2233            bundle.failures.failure_artifacts[0]
2234                .crashpack_link
2235                .as_ref()
2236                .map(|link| link.path.as_str()),
2237            Some("crashpack-divergence.registry.case.json")
2238        );
2239        assert_eq!(
2240            bundle.repro_manifest.promoted_scenario_id.as_deref(),
2241            Some("regression.test.surface.obligation_leak.seed_2a")
2242        );
2243        assert_eq!(
2244            bundle.repro_manifest.minimization_lineage.shrink_status,
2245            DivergenceShrinkStatus::PreservedSemanticClass
2246        );
2247        assert_eq!(
2248            bundle.repro_manifest.failure_artifacts_path,
2249            "artifacts/differential/nightly/divergence.registry.case/differential_failures.json"
2250        );
2251        assert!(
2252            bundle
2253                .repro_manifest
2254                .repro_commands
2255                .contains(&"cargo test divergence.registry.case -- --nocapture".to_string())
2256        );
2257        assert!(
2258            bundle
2259                .deviations
2260                .mismatches
2261                .iter()
2262                .any(|mismatch| mismatch.field == "semantics.obligation_balance.balanced")
2263        );
2264
2265        crate::test_complete!(
2266            "differential_bundle_artifacts_capture_repro_and_minimization_lineage"
2267        );
2268    }
2269
2270    #[test]
2271    fn inferred_crashpack_reference_requires_crashpack_like_path() {
2272        init_test("inferred_crashpack_reference_requires_crashpack_like_path");
2273
2274        let result = make_dual_run_divergence_result();
2275        let lab_link = DifferentialCrashpackReference::from_provenance(&result.lab.provenance);
2276        let live_link = DifferentialCrashpackReference::from_provenance(&result.live.provenance);
2277
2278        assert!(
2279            lab_link.is_some(),
2280            "crashpack-like lab artifact should infer linkage"
2281        );
2282        assert!(
2283            live_link.is_none(),
2284            "non-crashpack live artifact should not infer crashpack linkage"
2285        );
2286
2287        crate::test_complete!("inferred_crashpack_reference_requires_crashpack_like_path");
2288    }
2289
2290    // =========================================================================
2291    // METAMORPHIC TESTING: Lab::Replay Deterministic Fork/Join
2292    // =========================================================================
2293
2294    /// Configuration for metamorphic replay testing
2295    #[derive(Debug, Clone)]
2296    struct ReplayMetamorphicConfig {
2297        /// Number of workers for parallel execution
2298        worker_count: usize,
2299        /// Number of checkpoints to test
2300        checkpoint_count: usize,
2301        /// Number of concurrent tasks to spawn
2302        task_count: usize,
2303    }
2304
2305    impl Default for ReplayMetamorphicConfig {
2306        fn default() -> Self {
2307            Self {
2308                worker_count: 4,
2309                checkpoint_count: 5,
2310                task_count: 8,
2311            }
2312        }
2313    }
2314
2315    /// Generate deterministic test scenario for fork/join patterns
2316    fn create_fork_join_test_scenario(
2317        config: &ReplayMetamorphicConfig,
2318        rng_seed: u64,
2319    ) -> impl Fn(&mut LabRuntime) + Clone {
2320        let task_count = config.task_count;
2321        move |runtime: &mut LabRuntime| {
2322            // Use the runtime's deterministic execution to create fork/join patterns
2323            use crate::util::det_rng::DetRng;
2324            let mut rng = DetRng::new(rng_seed);
2325
2326            // Create a simple fork/join pattern with multiple concurrent tasks
2327            for i in 0..task_count {
2328                let _task_seed = rng.next_u64();
2329                // This would normally spawn tasks using the runtime's spawn mechanisms
2330                // For testing, we'll create trace events that represent fork/join operations
2331                runtime.trace().record_event(|id| {
2332                    crate::trace::TraceEvent::user_trace(
2333                        id,
2334                        runtime.now(),
2335                        format!("fork_task_{}", i),
2336                    )
2337                });
2338            }
2339
2340            // Simulate join phase
2341            for i in 0..task_count {
2342                runtime.trace().record_event(|id| {
2343                    crate::trace::TraceEvent::user_trace(
2344                        id,
2345                        runtime.now(),
2346                        format!("join_task_{}", i),
2347                    )
2348                });
2349            }
2350        }
2351    }
2352
2353    // =========================================================================
2354    // MR1: Checkpoint Replay Equivalence
2355    // =========================================================================
2356
2357    #[test]
2358    fn metamorphic_checkpoint_replay_equivalence() {
2359        init_test("metamorphic_checkpoint_replay_equivalence");
2360
2361        let seed = std::time::SystemTime::now()
2362            .duration_since(std::time::UNIX_EPOCH)
2363            .unwrap()
2364            .as_nanos() as u64;
2365
2366        let config = ReplayMetamorphicConfig::default();
2367
2368        // Test scenario: original execution vs replay from various checkpoints
2369        let test_scenario = create_fork_join_test_scenario(&config, seed);
2370
2371        // Run original execution
2372        let mut original_config = LabConfig::new(seed);
2373        original_config = original_config.worker_count(config.worker_count);
2374        let mut original_runtime = LabRuntime::new(original_config);
2375        test_scenario(&mut original_runtime);
2376        let original_trace = original_runtime.trace().snapshot();
2377        let original_certificate = original_runtime.certificate().hash();
2378
2379        // MR: Replay from different checkpoints should produce equivalent results
2380        // when executed to the same point
2381        for checkpoint_idx in 0..config.checkpoint_count.min(original_trace.len()) {
2382            let mut replay_config = LabConfig::new(seed);
2383            replay_config = replay_config.worker_count(config.worker_count);
2384            let mut replay_runtime = LabRuntime::new(replay_config);
2385
2386            // Simulate replay from checkpoint by processing events up to checkpoint
2387            for event in &original_trace[..checkpoint_idx] {
2388                replay_runtime.trace().push_event(event.clone());
2389            }
2390
2391            // Continue execution from checkpoint
2392            test_scenario(&mut replay_runtime);
2393            let replay_trace = replay_runtime.trace().snapshot();
2394            let replay_certificate = replay_runtime.certificate().hash();
2395
2396            // MR: Certificate hashes should match between original and replayed execution
2397            assert_eq!(
2398                original_certificate, replay_certificate,
2399                "Checkpoint {} replay diverged in certificate hash",
2400                checkpoint_idx
2401            );
2402
2403            // MR: The portion of the trace after the checkpoint should match
2404            // when both executions reach the same logical point
2405            if replay_trace.len() >= original_trace.len() {
2406                for (i, (orig_event, replay_event)) in
2407                    original_trace.iter().zip(replay_trace.iter()).enumerate()
2408                {
2409                    if i >= checkpoint_idx {
2410                        assert!(
2411                            events_match(orig_event, replay_event),
2412                            "Event {} after checkpoint {} doesn't match: {:?} vs {:?}",
2413                            i,
2414                            checkpoint_idx,
2415                            orig_event,
2416                            replay_event
2417                        );
2418                    }
2419                }
2420            }
2421        }
2422
2423        crate::test_complete!("metamorphic_checkpoint_replay_equivalence");
2424    }
2425
2426    // =========================================================================
2427    // MR2: Parallel Scope Fork/Join Order Determinism
2428    // =========================================================================
2429
2430    #[test]
2431    fn metamorphic_parallel_scope_fork_join_determinism() {
2432        init_test("metamorphic_parallel_scope_fork_join_determinism");
2433
2434        let seed = std::time::SystemTime::now()
2435            .duration_since(std::time::UNIX_EPOCH)
2436            .unwrap()
2437            .as_nanos() as u64;
2438
2439        let config = ReplayMetamorphicConfig::default();
2440
2441        // MR: Fork/join order should be deterministic across multiple runs with same seed
2442        let test_scenario = create_fork_join_test_scenario(&config, seed);
2443
2444        let mut executions = Vec::new();
2445
2446        // Execute the same scenario multiple times with the same seed
2447        for _run_idx in 0..5 {
2448            let mut runtime_config = LabConfig::new(seed); // Same seed every time
2449            runtime_config = runtime_config.worker_count(config.worker_count);
2450            let mut runtime = LabRuntime::new(runtime_config);
2451
2452            test_scenario(&mut runtime);
2453
2454            let trace = runtime.trace().snapshot();
2455            let certificate = runtime.certificate().hash();
2456            let steps = runtime.steps();
2457
2458            executions.push((trace, certificate, steps));
2459        }
2460
2461        // MR: All executions should produce identical results
2462        for (run_idx, (trace, certificate, steps)) in executions.iter().enumerate().skip(1) {
2463            assert_eq!(
2464                executions[0].1, *certificate,
2465                "Run {} has different certificate than run 0",
2466                run_idx
2467            );
2468            assert_eq!(
2469                executions[0].2, *steps,
2470                "Run {} has different step count than run 0",
2471                run_idx
2472            );
2473
2474            // Check trace equivalence
2475            let divergence = find_divergence(&executions[0].0, trace);
2476            assert!(
2477                divergence.is_none(),
2478                "Run {} diverged from run 0: {:?}",
2479                run_idx,
2480                divergence
2481            );
2482        }
2483
2484        // MR: Fork/join ordering should be stable within each trace
2485        for (run_idx, (trace, _, _)) in executions.iter().enumerate() {
2486            let mut fork_events = Vec::new();
2487            let mut join_events = Vec::new();
2488
2489            for event in trace {
2490                if matches!(&event.data, crate::trace::event::TraceData::Message(msg) if msg.contains("fork_task_"))
2491                {
2492                    fork_events.push(event.clone());
2493                } else if matches!(&event.data, crate::trace::event::TraceData::Message(msg) if msg.contains("join_task_"))
2494                {
2495                    join_events.push(event.clone());
2496                }
2497            }
2498
2499            // Verify fork events appear before join events (proper fork/join ordering)
2500            if let (Some(last_fork), Some(first_join)) = (fork_events.last(), join_events.first()) {
2501                assert!(
2502                    last_fork.time <= first_join.time,
2503                    "Run {}: Fork events should complete before join events start",
2504                    run_idx
2505                );
2506            }
2507        }
2508
2509        crate::test_complete!("metamorphic_parallel_scope_fork_join_determinism");
2510    }
2511
2512    // =========================================================================
2513    // MR3: Panic Replay Cause Chain Consistency
2514    // =========================================================================
2515
2516    #[test]
2517    fn metamorphic_panic_replay_cause_chain_consistency() {
2518        init_test("metamorphic_panic_replay_cause_chain_consistency");
2519
2520        let seed = std::time::SystemTime::now()
2521            .duration_since(std::time::UNIX_EPOCH)
2522            .unwrap()
2523            .as_nanos() as u64;
2524
2525        let config = ReplayMetamorphicConfig::default();
2526
2527        // Test scenario that includes panic conditions
2528        let panic_scenario = move |runtime: &mut LabRuntime| {
2529            use crate::util::det_rng::DetRng;
2530            let mut rng = DetRng::new(seed);
2531
2532            // Create events including simulated panic conditions
2533            for i in 0..config.task_count {
2534                if rng.next_u64() % 4 == 0 {
2535                    // 25% chance of panic
2536                    runtime.trace().record_event(|id| {
2537                        crate::trace::TraceEvent::user_trace(
2538                            id,
2539                            runtime.now(),
2540                            format!("panic_task_{}", i),
2541                        )
2542                    });
2543                } else {
2544                    runtime.trace().record_event(|id| {
2545                        crate::trace::TraceEvent::user_trace(
2546                            id,
2547                            runtime.now(),
2548                            format!("normal_task_{}", i),
2549                        )
2550                    });
2551                }
2552            }
2553        };
2554
2555        // Run original execution
2556        let mut original_config = LabConfig::new(seed);
2557        original_config = original_config.worker_count(config.worker_count);
2558        let mut original_runtime = LabRuntime::new(original_config);
2559        panic_scenario(&mut original_runtime);
2560        let original_trace = original_runtime.trace().snapshot();
2561
2562        // Run replay
2563        let mut replay_config = LabConfig::new(seed);
2564        replay_config = replay_config.worker_count(config.worker_count);
2565        let mut replay_runtime = LabRuntime::new(replay_config);
2566        panic_scenario(&mut replay_runtime);
2567        let replay_trace = replay_runtime.trace().snapshot();
2568
2569        // MR: Panic cause chains should be identical between original and replay
2570        let original_panics: Vec<_> = original_trace
2571            .iter()
2572            .filter(|event| trace_message_contains(event, "panic_"))
2573            .collect();
2574        let replay_panics: Vec<_> = replay_trace
2575            .iter()
2576            .filter(|event| trace_message_contains(event, "panic_"))
2577            .collect();
2578
2579        assert_eq!(
2580            original_panics.len(),
2581            replay_panics.len(),
2582            "Panic count should match between original and replay"
2583        );
2584
2585        for (original_panic, replay_panic) in original_panics.iter().zip(replay_panics.iter()) {
2586            assert!(
2587                events_match(original_panic, replay_panic),
2588                "Panic events should match: {:?} vs {:?}",
2589                original_panic,
2590                replay_panic
2591            );
2592        }
2593
2594        // MR: Overall trace should be identical (no divergence)
2595        let divergence = find_divergence(&original_trace, &replay_trace);
2596        assert!(
2597            divergence.is_none(),
2598            "Panic replay diverged: {:?}",
2599            divergence
2600        );
2601
2602        crate::test_complete!("metamorphic_panic_replay_cause_chain_consistency");
2603    }
2604
2605    // =========================================================================
2606    // MR4: Cross-Region Trace Ordering Preservation
2607    // =========================================================================
2608
2609    #[test]
2610    fn metamorphic_cross_region_trace_ordering_preservation() {
2611        init_test("metamorphic_cross_region_trace_ordering_preservation");
2612
2613        let seed = std::time::SystemTime::now()
2614            .duration_since(std::time::UNIX_EPOCH)
2615            .unwrap()
2616            .as_nanos() as u64;
2617
2618        let config = ReplayMetamorphicConfig::default();
2619
2620        // Test scenario with multiple regions
2621        let multi_region_scenario = move |runtime: &mut LabRuntime| {
2622            use crate::util::det_rng::DetRng;
2623            let _rng = DetRng::new(seed);
2624
2625            let region_count = 3;
2626
2627            // Create events across multiple regions
2628            for region_id in 0..region_count {
2629                for task_id in 0..config.task_count / region_count {
2630                    let now = runtime.now();
2631                    runtime.trace().record_event(|id| {
2632                        crate::trace::TraceEvent::user_trace(
2633                            id,
2634                            now,
2635                            format!("region_{}_task_{}", region_id, task_id),
2636                        )
2637                    });
2638                }
2639            }
2640        };
2641
2642        // Test ordering preservation across different execution contexts
2643        let execution_contexts = [
2644            ("single_worker", 1),
2645            ("dual_worker", 2),
2646            ("multi_worker", 4),
2647        ];
2648
2649        let mut context_traces = Vec::new();
2650
2651        for (context_name, worker_count) in &execution_contexts {
2652            let mut runtime_config = LabConfig::new(seed);
2653            runtime_config = runtime_config.worker_count(*worker_count);
2654            let mut runtime = LabRuntime::new(runtime_config);
2655
2656            multi_region_scenario(&mut runtime);
2657            let trace = runtime.trace().snapshot();
2658            context_traces.push((context_name, trace));
2659        }
2660
2661        // MR: Cross-region ordering should be preserved regardless of worker count
2662        for (context_name, trace) in &context_traces {
2663            let mut region_events: std::collections::BTreeMap<u32, Vec<&crate::trace::TraceEvent>> =
2664                std::collections::BTreeMap::new();
2665
2666            for event in trace {
2667                if let crate::trace::event::TraceData::Message(ref data_str) = event.data {
2668                    if data_str.contains("region_") {
2669                        if let Some(region_start) = data_str.find("region_") {
2670                            if let Some(region_end) = data_str[region_start + 7..].find('_') {
2671                                if let Ok(region_id) = data_str
2672                                    [region_start + 7..region_start + 7 + region_end]
2673                                    .parse::<u32>()
2674                                {
2675                                    region_events.entry(region_id).or_default().push(event);
2676                                }
2677                            }
2678                        }
2679                    }
2680                }
2681            }
2682
2683            // Verify each region has events
2684            assert!(
2685                !region_events.is_empty(),
2686                "Context {} should have region events",
2687                context_name
2688            );
2689
2690            // MR: Within each region, event ordering should be deterministic
2691            for (region_id, events) in &region_events {
2692                for window in events.windows(2) {
2693                    assert!(
2694                        window[0].time <= window[1].time,
2695                        "Context {}: Region {} events not in time order",
2696                        context_name,
2697                        region_id
2698                    );
2699                }
2700            }
2701        }
2702
2703        // MR: Different worker counts should produce equivalent logical ordering
2704        // (may have different physical timing but same logical causality)
2705        for i in 1..context_traces.len() {
2706            let (name1, trace1) = &context_traces[0];
2707            let (name2, trace2) = &context_traces[i];
2708
2709            // Extract logical ordering (ignoring precise timing)
2710            let logical_order1: Vec<_> = trace1
2711                .iter()
2712                .filter(|e| trace_message_contains(e, "region_"))
2713                .map(|e| &e.data)
2714                .collect();
2715            let logical_order2: Vec<_> = trace2
2716                .iter()
2717                .filter(|e| trace_message_contains(e, "region_"))
2718                .map(|e| &e.data)
2719                .collect();
2720
2721            assert_eq!(
2722                logical_order1, logical_order2,
2723                "Logical ordering differs between {} and {}",
2724                name1, name2
2725            );
2726        }
2727
2728        crate::test_complete!("metamorphic_cross_region_trace_ordering_preservation");
2729    }
2730
2731    // =========================================================================
2732    // MR5: LabRuntime Seed Determinism
2733    // =========================================================================
2734
2735    #[test]
2736    fn metamorphic_lab_runtime_seed_determinism() {
2737        init_test("metamorphic_lab_runtime_seed_determinism");
2738
2739        // MR: Same seed should produce identical execution across multiple runs
2740        const SEED: u64 = 0x1234_5678_9ABC_DEF0;
2741
2742        let config = ReplayMetamorphicConfig::default();
2743
2744        let deterministic_scenario = |runtime: &mut LabRuntime| {
2745            use crate::util::det_rng::DetRng;
2746            let mut rng = DetRng::new(SEED); // Use the same fixed seed
2747
2748            // Create deterministic sequence of events
2749            for i in 0..config.task_count {
2750                let choice = rng.next_u64() % 3;
2751                let event_type = match choice {
2752                    0 => "fork",
2753                    1 => "work",
2754                    _ => "join",
2755                };
2756
2757                runtime.trace().record_event(|id| {
2758                    crate::trace::TraceEvent::user_trace(
2759                        id,
2760                        runtime.now(),
2761                        format!("{}_{}", event_type, i),
2762                    )
2763                });
2764            }
2765        };
2766
2767        // Run multiple times with same seed
2768        let mut run_results = Vec::new();
2769
2770        for run_idx in 0..5 {
2771            let mut runtime_config = LabConfig::new(SEED);
2772            runtime_config = runtime_config.worker_count(config.worker_count);
2773            let mut runtime = LabRuntime::new(runtime_config);
2774
2775            deterministic_scenario(&mut runtime);
2776
2777            let trace = runtime.trace().snapshot();
2778            let certificate = runtime.certificate().hash();
2779            let steps = runtime.steps();
2780
2781            run_results.push((run_idx, trace, certificate, steps));
2782        }
2783
2784        // MR: All runs should produce identical results
2785        for (run_idx, trace, certificate, steps) in &run_results[1..] {
2786            assert_eq!(
2787                run_results[0].2, *certificate,
2788                "Run {} certificate differs from run 0",
2789                run_idx
2790            );
2791            assert_eq!(
2792                run_results[0].3, *steps,
2793                "Run {} step count differs from run 0",
2794                run_idx
2795            );
2796
2797            let divergence = find_divergence(&run_results[0].1, trace);
2798            assert!(
2799                divergence.is_none(),
2800                "Run {} trace diverged from run 0: {:?}",
2801                run_idx,
2802                divergence
2803            );
2804        }
2805
2806        // MR: Test seed independence - different seeds should produce different results
2807        let mut different_seed_config = LabConfig::new(SEED + 1);
2808        different_seed_config = different_seed_config.worker_count(config.worker_count);
2809        let mut different_seed_runtime = LabRuntime::new(different_seed_config);
2810
2811        deterministic_scenario(&mut different_seed_runtime);
2812        let _different_trace = different_seed_runtime.trace().snapshot();
2813        let different_certificate = different_seed_runtime.certificate().hash();
2814
2815        assert_ne!(
2816            run_results[0].2, different_certificate,
2817            "Different seed should produce different certificate"
2818        );
2819
2820        // Traces may or may not be different, but certificates should differ
2821        // showing that the seed is actually affecting the execution
2822
2823        crate::test_complete!("metamorphic_lab_runtime_seed_determinism");
2824    }
2825
2826    // =========================================================================
2827    // MR6: Composite Replay Invariants
2828    // =========================================================================
2829
2830    #[test]
2831    fn metamorphic_composite_replay_invariants() {
2832        init_test("metamorphic_composite_replay_invariants");
2833
2834        let seed = std::time::SystemTime::now()
2835            .duration_since(std::time::UNIX_EPOCH)
2836            .unwrap()
2837            .as_nanos() as u64;
2838
2839        let config = ReplayMetamorphicConfig::default();
2840
2841        // MR: Combination of multiple replay properties should all hold simultaneously
2842        let composite_scenario = |runtime: &mut LabRuntime| {
2843            use crate::util::det_rng::DetRng;
2844            let mut rng = DetRng::new(seed);
2845
2846            // Create a complex scenario combining:
2847            // 1. Fork/join patterns
2848            // 2. Cross-region operations
2849            // 3. Potential panic conditions
2850            // 4. Checkpoint-worthy state changes
2851
2852            let regions = 2;
2853            let tasks_per_region = config.task_count / regions;
2854
2855            for region_id in 0..regions {
2856                // Fork phase
2857                for task_id in 0..tasks_per_region {
2858                    let now = runtime.now();
2859                    runtime.trace().record_event(|id| {
2860                        crate::trace::TraceEvent::user_trace(
2861                            id,
2862                            now,
2863                            format!("fork_region_{}_task_{}", region_id, task_id),
2864                        )
2865                    });
2866                }
2867
2868                // Work phase (with occasional panics)
2869                for task_id in 0..tasks_per_region {
2870                    let event_type = if rng.next_u64() % 10 == 0 {
2871                        "panic"
2872                    } else {
2873                        "work"
2874                    };
2875                    let now = runtime.now();
2876                    runtime.trace().record_event(|id| {
2877                        crate::trace::TraceEvent::user_trace(
2878                            id,
2879                            now,
2880                            format!("{}_region_{}_task_{}", event_type, region_id, task_id),
2881                        )
2882                    });
2883                }
2884
2885                // Join phase
2886                for task_id in 0..tasks_per_region {
2887                    runtime.trace().record_event(|id| {
2888                        crate::trace::TraceEvent::user_trace(
2889                            id,
2890                            runtime.now(),
2891                            format!("join_region_{}_task_{}", region_id, task_id),
2892                        )
2893                    });
2894                }
2895            }
2896        };
2897
2898        // Test the scenario with replay validation
2899        let replay_validation = validate_replay(seed, config.worker_count, composite_scenario);
2900
2901        assert!(
2902            replay_validation.matched,
2903            "Composite scenario replay should match original: certificates {} vs {}, steps {} vs {}",
2904            replay_validation.original_certificate,
2905            replay_validation.replay_certificate,
2906            replay_validation.original_steps,
2907            replay_validation.replay_steps
2908        );
2909
2910        assert!(
2911            replay_validation.divergence.is_none(),
2912            "Composite scenario should have no divergence: {:?}",
2913            replay_validation.divergence
2914        );
2915
2916        // Test multiple seeds for robustness
2917        let test_seeds = [seed, seed + 1, seed + 42, seed + 1337, seed + 0xDEAD];
2918
2919        for &test_seed in &test_seeds {
2920            let validation = validate_replay(test_seed, config.worker_count, |runtime| {
2921                composite_scenario(runtime);
2922            });
2923
2924            assert!(
2925                validation.matched,
2926                "Seed {} composite replay failed: {:?}",
2927                test_seed, validation.divergence
2928            );
2929        }
2930
2931        // MR: Multi-seed validation should show consistent determinism
2932        let multi_validation =
2933            validate_replay_multi(&test_seeds, config.worker_count, composite_scenario);
2934
2935        for (i, validation) in multi_validation.iter().enumerate() {
2936            assert!(validation.matched, "Multi-seed run {} failed validation", i);
2937        }
2938        crate::test_complete!("metamorphic_composite_replay_invariants");
2939    }
2940}