Skip to main content

asupersync/lab/
runtime.rs

1//! Lab runtime for deterministic execution.
2//!
3//! The lab runtime executes tasks with:
4//! - Virtual time (controlled advancement)
5//! - Deterministic scheduling (seed-driven)
6//! - Trace capture for replay
7//! - Chaos injection for stress testing
8
9use super::config::LabConfig;
10use super::oracle::OracleSuite;
11use crate::lab::chaos::{ChaosRng, ChaosStats};
12use crate::record::ObligationKind;
13use crate::record::task::TaskState;
14use crate::runtime::RuntimeState;
15use crate::runtime::config::ObligationLeakResponse;
16use crate::runtime::deadline_monitor::{
17    DeadlineMonitor, DeadlineWarning, MonitorConfig, default_warning_handler,
18};
19use crate::runtime::reactor::LabReactor;
20use crate::runtime::scheduler::{DispatchLane, ScheduleCertificate};
21use crate::time::VirtualClock;
22use crate::trace::TraceBufferHandle;
23use crate::trace::crashpack::{
24    CrashPack, CrashPackConfig, FailureInfo, FailureOutcome, ReplayCommand, artifact_filename,
25};
26use crate::trace::event::TraceEventKind;
27use crate::trace::recorder::TraceRecorder;
28use crate::trace::replay::{ReplayEvent, ReplayTrace, TraceMetadata};
29use crate::trace::scoring::seed_fingerprint;
30use crate::trace::{TraceData, TraceEvent, check_refinement_firewall};
31use crate::trace::{canonicalize::trace_fingerprint, certificate::TraceCertificate};
32use crate::types::Time;
33use crate::types::{ObligationId, RegionId, TaskId};
34use crate::util::det_hash::{DetHashMap, DetHashSet};
35use crate::util::{DetEntropy, DetRng};
36use parking_lot::Mutex;
37use std::fmt;
38use std::sync::Arc;
39use std::task::{Context, Poll, Waker};
40use std::time::Duration;
41
42/// Summary of a trace certificate built from the current trace buffer.
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub struct LabTraceCertificateSummary {
45    /// Incremental hash of witnessed events.
46    pub event_hash: u64,
47    /// Total number of events witnessed.
48    pub event_count: u64,
49    /// Hash of scheduling decisions (from [`ScheduleCertificate`]).
50    pub schedule_hash: u64,
51}
52
53/// Why a [`LabRuntime::run_with_auto_advance`] loop terminated.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55pub enum AutoAdvanceTermination {
56    /// The runtime reached quiescence: no runnable tasks, no pending timers,
57    /// and all regions are closed.
58    Quiescent,
59    /// The configured `max_steps` limit was reached before quiescence.
60    StepLimitReached,
61    /// The runtime was stuck (scheduler empty, no pending deadlines, not
62    /// quiescent) for 1 000 consecutive iterations and bailed out.
63    StuckBailout,
64}
65
66impl fmt::Display for AutoAdvanceTermination {
67    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
68        match self {
69            Self::Quiescent => f.write_str("quiescent"),
70            Self::StepLimitReached => f.write_str("step-limit-reached"),
71            Self::StuckBailout => f.write_str("stuck-bailout"),
72        }
73    }
74}
75
76/// Report from a [`LabRuntime::run_with_auto_advance`] execution.
77///
78/// Captures statistics about automatic time advancement during the run.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub struct VirtualTimeReport {
81    /// Total scheduler steps executed.
82    pub steps: u64,
83    /// Number of times virtual time was auto-advanced to the next pending
84    /// timer or lab-reactor deadline.
85    pub auto_advances: u64,
86    /// Total timer wakeups triggered by auto-advances.
87    pub total_wakeups: u64,
88    /// Virtual time at the start of the run.
89    pub time_start: Time,
90    /// Virtual time at the end of the run.
91    pub time_end: Time,
92    /// Total virtual nanoseconds elapsed during the run.
93    pub virtual_elapsed_nanos: u64,
94    /// Why the auto-advance loop terminated.
95    pub termination: AutoAdvanceTermination,
96}
97
98impl VirtualTimeReport {
99    /// Returns the virtual elapsed time in milliseconds.
100    #[must_use]
101    pub const fn virtual_elapsed_ms(&self) -> u64 {
102        self.virtual_elapsed_nanos / 1_000_000
103    }
104
105    /// Returns the virtual elapsed time in seconds.
106    #[must_use]
107    pub const fn virtual_elapsed_secs(&self) -> u64 {
108        self.virtual_elapsed_nanos / 1_000_000_000
109    }
110}
111
112/// Structured report for a single lab runtime run.
113///
114/// This is intended as a low-level building block for Spork app harnesses.
115/// It contains canonical trace fingerprints and oracle outcomes, but it does
116/// not write to stdout/stderr or persist artifacts.
117#[derive(Debug, Clone)]
118pub struct LabRunReport {
119    /// Lab seed driving scheduling determinism.
120    pub seed: u64,
121    /// Steps executed during the `run_until_quiescent()` call that produced this report.
122    pub steps_delta: u64,
123    /// Total steps executed by the runtime so far.
124    pub steps_total: u64,
125    /// Whether the runtime is quiescent at report time.
126    pub quiescent: bool,
127    /// Virtual time (nanoseconds since epoch) at report time.
128    pub now_nanos: u64,
129    /// Number of events in the current trace buffer snapshot.
130    pub trace_len: usize,
131    /// Canonical fingerprint of the trace equivalence class (Foata / Mazurkiewicz).
132    pub trace_fingerprint: u64,
133    /// Trace certificate summary (event hash/count + schedule hash).
134    pub trace_certificate: LabTraceCertificateSummary,
135    /// Unified oracle report (stable ordering, serializable).
136    pub oracle_report: crate::lab::oracle::OracleReport,
137    /// Runtime invariant violations detected by `LabRuntime::check_invariants()`.
138    ///
139    /// This is distinct from the oracle suite: it's a small set of runtime-level
140    /// checks (e.g., obligation leaks, futurelocks, quiescence violations).
141    pub invariant_violations: Vec<String>,
142    /// Temporal-oracle invariants that failed in this report.
143    pub temporal_invariant_failures: Vec<String>,
144    /// Minimized divergent-prefix length for temporal failures, when available.
145    pub temporal_counterexample_prefix_len: Option<usize>,
146    /// First failed refinement-firewall rule id, when present.
147    pub refinement_firewall_rule_id: Option<String>,
148    /// Event index where refinement-firewall first failed.
149    pub refinement_firewall_event_index: Option<usize>,
150    /// Event sequence where refinement-firewall first failed.
151    pub refinement_firewall_event_seq: Option<u64>,
152    /// Deterministic minimal counterexample prefix length for refinement failures.
153    pub refinement_counterexample_prefix_len: Option<usize>,
154    /// Whether refinement checks were skipped due to trace-buffer truncation.
155    pub refinement_firewall_skipped_due_to_trace_truncation: bool,
156}
157
158impl LabRunReport {
159    /// Convert to JSON for artifact storage.
160    #[must_use]
161    pub fn to_json(&self) -> serde_json::Value {
162        use serde_json::json;
163
164        json!({
165            "seed": self.seed,
166            "steps_delta": self.steps_delta,
167            "steps_total": self.steps_total,
168            "quiescent": self.quiescent,
169            "now_nanos": self.now_nanos,
170            "trace": {
171                "len": self.trace_len,
172                "fingerprint": self.trace_fingerprint,
173                "certificate": {
174                    "event_hash": self.trace_certificate.event_hash,
175                    "event_count": self.trace_certificate.event_count,
176                    "schedule_hash": self.trace_certificate.schedule_hash,
177                }
178            },
179            "oracles": self.oracle_report.to_json(),
180            "invariants": self.invariant_violations,
181            "temporal": {
182                "failed_invariants": self.temporal_invariant_failures,
183                "counterexample_prefix_len": self.temporal_counterexample_prefix_len,
184            },
185            "refinement_firewall": {
186                "rule_id": self.refinement_firewall_rule_id,
187                "event_index": self.refinement_firewall_event_index,
188                "event_seq": self.refinement_firewall_event_seq,
189                "counterexample_prefix_len": self.refinement_counterexample_prefix_len,
190                "skipped_due_to_trace_truncation": self.refinement_firewall_skipped_due_to_trace_truncation,
191            },
192        })
193    }
194
195    /// Export this run's trace events to a TLA+ module for model checking.
196    ///
197    /// Converts the captured trace into a TLA+ behavior (concrete state
198    /// sequence) with property templates for the 6 core invariants. The
199    /// resulting module can be fed to TLC for bounded model checking.
200    ///
201    /// Returns `None` if no trace events were captured.
202    #[must_use]
203    pub fn export_tla(
204        &self,
205        trace_events: &[crate::trace::TraceEvent],
206        module_name: &str,
207    ) -> Option<crate::trace::tla_export::TlaModule> {
208        if trace_events.is_empty() {
209            return None;
210        }
211        let exporter = crate::trace::tla_export::TlaExporter::from_trace(trace_events);
212        if exporter.snapshot_count() == 0 {
213            return None;
214        }
215        Some(exporter.export_behavior(module_name))
216    }
217}
218
219const TEMPORAL_ORACLE_INVARIANTS: &[&str] = &[
220    "task_leak",
221    "obligation_leak",
222    "quiescence",
223    "cancellation_protocol",
224    "loser_drain",
225    "region_tree",
226    "deadline_monotone",
227    #[cfg(feature = "messaging-fabric")]
228    "fabric_publish",
229    #[cfg(feature = "messaging-fabric")]
230    "fabric_reply",
231    #[cfg(feature = "messaging-fabric")]
232    "fabric_quiescence",
233    #[cfg(feature = "messaging-fabric")]
234    "fabric_redelivery",
235];
236
237// ---------------------------------------------------------------------------
238// Spork app harness report schema (bd-11dm5)
239// ---------------------------------------------------------------------------
240
241/// Snapshot of a [`LabConfig`] captured into a stable, JSON-friendly schema.
242///
243/// This is intentionally a *summary* (not the raw config), so downstream tools
244/// can depend on a stable field set without pulling in internal config types.
245#[derive(Debug, Clone, PartialEq)]
246pub struct LabConfigSummary {
247    /// Random seed for deterministic scheduling.
248    pub seed: u64,
249    /// Seed for capability entropy sources (may be decoupled from `seed`).
250    pub entropy_seed: u64,
251    /// Number of modeled workers in the deterministic multi-worker simulation.
252    pub worker_count: usize,
253    /// Whether the runtime panics on obligation leaks in lab mode.
254    pub panic_on_obligation_leak: bool,
255    /// Capacity of the trace buffer.
256    pub trace_capacity: usize,
257    /// Maximum steps a task may remain unpolled while holding obligations before futurelock triggers.
258    pub futurelock_max_idle_steps: u64,
259    /// Whether the runtime panics when a futurelock is detected.
260    pub panic_on_futurelock: bool,
261    /// Optional maximum step limit for a run.
262    pub max_steps: Option<u64>,
263    /// Chaos configuration summary, when enabled.
264    pub chaos: Option<ChaosConfigSummary>,
265    /// Whether replay recording is enabled.
266    pub replay_recording_enabled: bool,
267}
268
269impl LabConfigSummary {
270    /// Build a config summary from the full [`LabConfig`].
271    #[must_use]
272    pub fn from_config(config: &LabConfig) -> Self {
273        Self {
274            seed: config.seed,
275            entropy_seed: config.entropy_seed,
276            worker_count: config.worker_count,
277            panic_on_obligation_leak: config.panic_on_obligation_leak,
278            trace_capacity: config.trace_capacity,
279            futurelock_max_idle_steps: config.futurelock_max_idle_steps,
280            panic_on_futurelock: config.panic_on_futurelock,
281            max_steps: config.max_steps,
282            chaos: config.chaos.as_ref().map(ChaosConfigSummary::from_config),
283            replay_recording_enabled: config.replay_recording.is_some(),
284        }
285    }
286
287    /// Compute a stable hash of the configuration for quick equivalence checking.
288    ///
289    /// Two configs with the same hash produced identical lab setups. Agents can
290    /// compare config hashes across runs to confirm they used the same settings.
291    #[must_use]
292    pub fn config_hash(&self) -> u64 {
293        use std::hash::{Hash, Hasher};
294        // DefaultHasher is NOT stable across Rust versions; DetHasher uses a
295        // fixed algorithm and seed for cross-version deterministic hashing.
296        let mut h = crate::util::DetHasher::default();
297        self.seed.hash(&mut h);
298        self.entropy_seed.hash(&mut h);
299        self.worker_count.hash(&mut h);
300        self.panic_on_obligation_leak.hash(&mut h);
301        self.trace_capacity.hash(&mut h);
302        self.futurelock_max_idle_steps.hash(&mut h);
303        self.panic_on_futurelock.hash(&mut h);
304        self.max_steps.hash(&mut h);
305        self.replay_recording_enabled.hash(&mut h);
306        if let Some(ref c) = self.chaos {
307            1u8.hash(&mut h);
308            c.seed.hash(&mut h);
309            c.cancel_probability.to_bits().hash(&mut h);
310            c.delay_probability.to_bits().hash(&mut h);
311            c.io_error_probability.to_bits().hash(&mut h);
312            c.wakeup_storm_probability.to_bits().hash(&mut h);
313            c.budget_exhaust_probability.to_bits().hash(&mut h);
314        } else {
315            0u8.hash(&mut h);
316        }
317        h.finish()
318    }
319
320    /// Convert to JSON for artifact storage.
321    #[must_use]
322    pub fn to_json(&self) -> serde_json::Value {
323        use serde_json::json;
324
325        json!({
326            "seed": self.seed,
327            "entropy_seed": self.entropy_seed,
328            "worker_count": self.worker_count,
329            "panic_on_obligation_leak": self.panic_on_obligation_leak,
330            "trace_capacity": self.trace_capacity,
331            "futurelock_max_idle_steps": self.futurelock_max_idle_steps,
332            "panic_on_futurelock": self.panic_on_futurelock,
333            "max_steps": self.max_steps,
334            "chaos": self.chaos.as_ref().map(ChaosConfigSummary::to_json),
335            "replay_recording_enabled": self.replay_recording_enabled,
336        })
337    }
338}
339
340/// JSON-friendly summary of chaos settings.
341#[derive(Debug, Clone, Copy, PartialEq)]
342pub struct ChaosConfigSummary {
343    /// Seed for deterministic chaos injection.
344    pub seed: u64,
345    /// Probability of injecting cancellation at each poll point.
346    pub cancel_probability: f64,
347    /// Probability of injecting delay at each poll point.
348    pub delay_probability: f64,
349    /// Probability of injecting an I/O error.
350    pub io_error_probability: f64,
351    /// Probability of triggering a spurious wakeup storm.
352    pub wakeup_storm_probability: f64,
353    /// Probability of injecting budget exhaustion.
354    pub budget_exhaust_probability: f64,
355}
356
357impl ChaosConfigSummary {
358    /// Build a chaos summary from the full chaos configuration.
359    #[must_use]
360    pub fn from_config(config: &crate::lab::chaos::ChaosConfig) -> Self {
361        Self {
362            seed: config.seed,
363            cancel_probability: config.cancel_probability,
364            delay_probability: config.delay_probability,
365            io_error_probability: config.io_error_probability,
366            wakeup_storm_probability: config.wakeup_storm_probability,
367            budget_exhaust_probability: config.budget_exhaust_probability,
368        }
369    }
370
371    /// Convert to JSON for artifact storage.
372    #[must_use]
373    pub fn to_json(&self) -> serde_json::Value {
374        use serde_json::json;
375
376        json!({
377            "seed": self.seed,
378            "cancel_probability": self.cancel_probability,
379            "delay_probability": self.delay_probability,
380            "io_error_probability": self.io_error_probability,
381            "wakeup_storm_probability": self.wakeup_storm_probability,
382            "budget_exhaust_probability": self.budget_exhaust_probability,
383        })
384    }
385}
386
387/// Attachment kind for Spork harness reports.
388#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
389pub enum HarnessAttachmentKind {
390    /// Crash pack artifact (minimal repro pack).
391    CrashPack,
392    /// Replay trace artifact (recorded non-determinism for replay).
393    ReplayTrace,
394    /// Generic trace artifact (e.g., NDJSON/JSON trace snapshot).
395    Trace,
396    /// Other harness-defined artifact.
397    Other,
398}
399
400impl fmt::Display for HarnessAttachmentKind {
401    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
402        match self {
403            Self::CrashPack => write!(f, "crashpack"),
404            Self::ReplayTrace => write!(f, "replay_trace"),
405            Self::Trace => write!(f, "trace"),
406            Self::Other => write!(f, "other"),
407        }
408    }
409}
410
411/// Report attachment reference (path-only).
412///
413/// The lab runtime does not write artifacts; this is a schema hook that a harness
414/// can fill in when it persists crash packs, replay traces, etc.
415#[derive(Debug, Clone, PartialEq, Eq, Hash)]
416pub struct HarnessAttachmentRef {
417    /// Attachment kind (used for deterministic ordering and downstream routing).
418    pub kind: HarnessAttachmentKind,
419    /// Artifact path (relative or absolute; interpreted by the harness).
420    pub path: String,
421}
422
423impl HarnessAttachmentRef {
424    /// Convenience constructor for crash pack attachments.
425    #[must_use]
426    pub fn crashpack(path: impl Into<String>) -> Self {
427        Self {
428            kind: HarnessAttachmentKind::CrashPack,
429            path: path.into(),
430        }
431    }
432
433    /// Convenience constructor for replay trace attachments.
434    #[must_use]
435    pub fn replay_trace(path: impl Into<String>) -> Self {
436        Self {
437            kind: HarnessAttachmentKind::ReplayTrace,
438            path: path.into(),
439        }
440    }
441
442    /// Convenience constructor for generic trace attachments.
443    #[must_use]
444    pub fn trace(path: impl Into<String>) -> Self {
445        Self {
446            kind: HarnessAttachmentKind::Trace,
447            path: path.into(),
448        }
449    }
450
451    /// Convert to JSON for artifact storage.
452    #[must_use]
453    pub fn to_json(&self) -> serde_json::Value {
454        use serde_json::json;
455
456        json!({
457            "kind": self.kind.to_string(),
458            "path": self.path,
459        })
460    }
461}
462
463/// Deterministic crashpack linkage metadata exposed in harness reports.
464#[derive(Debug, Clone, PartialEq, Eq)]
465pub struct CrashpackLink {
466    /// Artifact path for the crashpack attachment.
467    pub path: String,
468    /// Stable crashpack identifier (seed + fingerprint tuple).
469    pub id: String,
470    /// Canonical trace fingerprint associated with this crashpack.
471    pub fingerprint: u64,
472    /// Reproduction command generated from the run config and crashpack path.
473    pub replay: ReplayCommand,
474}
475
476impl CrashpackLink {
477    /// Convert to JSON for artifact storage.
478    #[must_use]
479    pub fn to_json(&self) -> serde_json::Value {
480        use serde_json::json;
481
482        json!({
483            "path": self.path,
484            "id": self.id,
485            "fingerprint": self.fingerprint,
486            "replay": self.replay,
487        })
488    }
489}
490
491/// Stable, JSON-first report schema for Spork app harness runs.
492///
493/// This wraps [`LabRunReport`] and adds:
494/// - config snapshot (lab-side)
495/// - stable fingerprint extraction points
496/// - optional artifact attachment references (crash packs, replay traces, ...)
497#[derive(Debug, Clone)]
498pub struct SporkHarnessReport {
499    /// Schema version for stable downstream parsing.
500    pub schema_version: u32,
501    /// Application identifier/name for the harness run.
502    pub app: String,
503    /// Lab configuration snapshot used for the run.
504    pub config: LabConfigSummary,
505    /// Low-level lab run report (trace fingerprints + oracles + invariants).
506    pub run: LabRunReport,
507    /// Optional attachment references (crash packs, replay traces, etc.).
508    pub attachments: Vec<HarnessAttachmentRef>,
509}
510
511impl SporkHarnessReport {
512    /// Current stable schema version.
513    ///
514    /// Increment when the JSON contract changes in a backward-incompatible way.
515    /// Backward-compatible additions (new optional fields) do NOT require a bump.
516    /// Breaking changes (field renames, type changes, removals) MUST bump this
517    /// and document the migration in a comment here.
518    ///
519    /// # Version history
520    ///
521    /// - **v1**: Initial schema (bd-11dm5). Top-level keys: `schema_version`,
522    ///   `app`, `lab`, `fingerprints`, `run`, `attachments`.
523    /// - **v2**: Agent report contract (bd-f262i). Added `lab.config_hash` for
524    ///   quick equivalence checking. Added `verdict` top-level key. No
525    ///   breaking changes to existing fields.
526    /// - **v3**: Crashpack linking contract (bd-1wen4). Added top-level
527    ///   `crashpack` object with deterministic path/id/fingerprint/replay.
528    pub const SCHEMA_VERSION: u32 = 3;
529
530    /// Create a new harness report from a low-level lab run report.
531    #[must_use]
532    pub fn new(
533        app: impl Into<String>,
534        config: &LabConfig,
535        run: LabRunReport,
536        attachments: Vec<HarnessAttachmentRef>,
537    ) -> Self {
538        Self {
539            schema_version: Self::SCHEMA_VERSION,
540            app: app.into(),
541            config: LabConfigSummary::from_config(config),
542            run,
543            attachments,
544        }
545    }
546
547    // -------------------------------------------------------------------------
548    // Agent UX convenience methods (bd-f262i)
549    // -------------------------------------------------------------------------
550
551    /// Quick pass/fail verdict: all oracles passed and no invariant violations.
552    #[must_use]
553    pub fn passed(&self) -> bool {
554        self.run.oracle_report.all_passed() && self.run.invariant_violations.is_empty()
555    }
556
557    /// The canonical trace fingerprint for this run.
558    #[must_use]
559    pub fn trace_fingerprint(&self) -> u64 {
560        self.run.trace_fingerprint
561    }
562
563    /// The lab seed that drove this run.
564    #[must_use]
565    pub fn seed(&self) -> u64 {
566        self.run.seed
567    }
568
569    /// Config hash for quick equivalence checking across runs.
570    #[must_use]
571    pub fn config_hash(&self) -> u64 {
572        self.config.config_hash()
573    }
574
575    /// Returns the path to the first crashpack attachment, if any.
576    #[must_use]
577    pub fn crashpack_path(&self) -> Option<&str> {
578        self.attachments
579            .iter()
580            .find(|a| a.kind == HarnessAttachmentKind::CrashPack)
581            .map(|a| a.path.as_str())
582    }
583
584    /// Deterministic crashpack linkage metadata when a crashpack is attached.
585    #[must_use]
586    pub fn crashpack_link(&self) -> Option<CrashpackLink> {
587        let path = self.crashpack_path()?.to_string();
588        let crash_config = CrashPackConfig {
589            seed: self.seed(),
590            config_hash: self.config_hash(),
591            worker_count: self.config.worker_count,
592            max_steps: self.config.max_steps,
593            commit_hash: None,
594        };
595        let replay = ReplayCommand::from_config(&crash_config, Some(path.as_str()));
596        Some(CrashpackLink {
597            id: format!(
598                "crashpack-{seed:016x}-{fingerprint:016x}",
599                seed = self.seed(),
600                fingerprint = self.trace_fingerprint()
601            ),
602            fingerprint: self.trace_fingerprint(),
603            path,
604            replay,
605        })
606    }
607
608    /// Returns oracle failure descriptions, if any.
609    #[must_use]
610    pub fn oracle_failures(&self) -> Vec<String> {
611        self.run
612            .oracle_report
613            .failures()
614            .iter()
615            .map(|e| {
616                let desc = e
617                    .violation
618                    .as_ref()
619                    .map_or_else(String::new, |v| format!(": {v}"));
620                format!("{}{desc}", e.invariant)
621            })
622            .collect()
623    }
624
625    /// One-line human-readable summary suitable for agent log output.
626    ///
627    /// Format: `[PASS|FAIL] app="name" seed=N fingerprint=N oracles=P/T`
628    #[must_use]
629    pub fn summary_line(&self) -> String {
630        let verdict = if self.passed() { "PASS" } else { "FAIL" };
631        let oracle = &self.run.oracle_report;
632        format!(
633            "[{verdict}] app=\"{}\" seed={} fingerprint={} oracles={}/{} invariant_violations={}",
634            self.app,
635            self.run.seed,
636            self.run.trace_fingerprint,
637            oracle.passed,
638            oracle.total,
639            self.run.invariant_violations.len(),
640        )
641    }
642
643    /// Convert to JSON for artifact storage.
644    ///
645    /// # Agent Report Contract (bd-f262i)
646    ///
647    /// This is the **single stable JSON schema** that agents rely on across:
648    /// - Lab runs (`SporkAppHarness::run_to_report`)
649    /// - DPOR exploration (`ExplorationReport` wraps these)
650    /// - Conformance suites (test assertions against these fields)
651    ///
652    /// ## Top-level fields (v2)
653    ///
654    /// | Key              | Type    | Stable? | Description                              |
655    /// |------------------|---------|---------|------------------------------------------|
656    /// | `schema_version` | u32     | yes     | Schema version (bump on breaking change) |
657    /// | `verdict`        | string  | yes     | `"pass"` or `"fail"`                     |
658    /// | `app.name`       | string  | yes     | Application name from `AppSpec`           |
659    /// | `lab.config`     | object  | yes     | Full config snapshot                      |
660    /// | `lab.config_hash`| u64     | yes     | Quick config equivalence hash             |
661    /// | `fingerprints.*` | u64     | yes     | Trace/schedule fingerprints               |
662    /// | `run.*`          | object  | yes     | Full `LabRunReport` (oracles, invariants) |
663    /// | `crashpack`      | object? | yes     | Deterministic crashpack linkage metadata  |
664    /// | `attachments`    | array   | yes     | Sorted by (kind, path)                    |
665    #[must_use]
666    pub fn to_json(&self) -> serde_json::Value {
667        use serde_json::json;
668
669        // Ensure stable ordering regardless of insertion order.
670        let mut attachments = self.attachments.clone();
671        attachments.sort_by(|a, b| (a.kind, &a.path).cmp(&(b.kind, &b.path)));
672
673        json!({
674            "schema_version": self.schema_version,
675            "verdict": if self.passed() { "pass" } else { "fail" },
676            "app": { "name": self.app },
677            "lab": {
678                "config": self.config.to_json(),
679                "config_hash": self.config.config_hash(),
680            },
681            "fingerprints": {
682                "trace": self.run.trace_fingerprint,
683                "event_hash": self.run.trace_certificate.event_hash,
684                "event_count": self.run.trace_certificate.event_count,
685                "schedule_hash": self.run.trace_certificate.schedule_hash,
686            },
687            "run": self.run.to_json(),
688            "crashpack": self.crashpack_link().map(|link| link.to_json()),
689            "attachments": attachments.iter().map(HarnessAttachmentRef::to_json).collect::<Vec<_>>(),
690        })
691    }
692}
693
694/// The deterministic lab runtime.
695///
696/// This runtime is designed for testing and provides:
697/// - Virtual time instead of wall-clock time
698/// - Deterministic scheduling based on a seed
699/// - Trace capture for debugging and replay
700/// - Chaos injection for stress testing
701#[derive(Debug)]
702pub struct LabRuntime {
703    /// Runtime state (public for tests and oracle access).
704    pub state: RuntimeState,
705    /// Lab reactor for deterministic I/O simulation.
706    lab_reactor: Arc<LabReactor>,
707    /// Tokens seen for I/O submissions (for trace emission).
708    seen_io_tokens: DetHashSet<usize>,
709    /// Scheduler.
710    pub scheduler: Arc<Mutex<LabScheduler>>,
711    /// Configuration.
712    config: LabConfig,
713    /// Deterministic RNG.
714    rng: DetRng,
715    /// Current virtual time.
716    virtual_time: Time,
717    /// Virtual clock backing the timer driver.
718    virtual_clock: Arc<VirtualClock>,
719    /// Number of steps executed.
720    steps: u64,
721    /// Chaos RNG for deterministic fault injection.
722    chaos_rng: Option<ChaosRng>,
723    /// Statistics about chaos injections.
724    chaos_stats: ChaosStats,
725    /// Reactor chaos statistics already folded into `chaos_stats`.
726    seen_reactor_chaos_stats: ChaosStats,
727    /// Replay recorder for deterministic trace capture.
728    replay_recorder: TraceRecorder,
729    /// Optional deadline monitor for warning callbacks.
730    deadline_monitor: Option<DeadlineMonitor>,
731    /// Oracle suite for invariant verification.
732    pub oracles: OracleSuite,
733    /// Schedule certificate for determinism verification.
734    certificate: ScheduleCertificate,
735}
736
737impl LabRuntime {
738    /// Creates a new lab runtime with the given configuration.
739    #[must_use]
740    pub fn new(config: LabConfig) -> Self {
741        let rng = config.rng();
742        let chaos_rng = config.chaos.as_ref().map(ChaosRng::from_config);
743        let lab_reactor = config.chaos.as_ref().map_or_else(
744            || Arc::new(LabReactor::new()),
745            |chaos| Arc::new(LabReactor::with_chaos(chaos.clone())),
746        );
747        let mut state = RuntimeState::with_reactor(lab_reactor.clone());
748        state.set_logical_clock_mode(crate::trace::distributed::LogicalClockMode::Lamport);
749        state.set_obligation_leak_response(if config.panic_on_obligation_leak {
750            ObligationLeakResponse::Panic
751        } else {
752            ObligationLeakResponse::Log
753        });
754        let virtual_clock = Arc::new(VirtualClock::starting_at(Time::ZERO));
755        state.set_timer_driver(crate::time::TimerDriverHandle::with_virtual_clock(
756            virtual_clock.clone(),
757        ));
758        state.set_entropy_source(Arc::new(DetEntropy::new(config.entropy_seed)));
759        state.trace = TraceBufferHandle::new(config.trace_capacity);
760
761        // Initialize replay recorder if configured
762        let mut replay_recorder = if let Some(ref rec_config) = config.replay_recording {
763            TraceRecorder::with_config(TraceMetadata::new(config.seed), rec_config.clone())
764        } else {
765            TraceRecorder::disabled()
766        };
767
768        // Record initial RNG seed
769        replay_recorder.record_rng_seed(config.seed);
770
771        crate::tracing_compat::info!("virtual clock initialized: start_time_ms=0");
772
773        Self {
774            state,
775            lab_reactor,
776            seen_io_tokens: DetHashSet::default(),
777            scheduler: Arc::new(Mutex::new(LabScheduler::new(config.worker_count))),
778            config,
779            rng,
780            virtual_time: Time::ZERO,
781            virtual_clock,
782            steps: 0,
783            chaos_rng,
784            chaos_stats: ChaosStats::new(),
785            seen_reactor_chaos_stats: ChaosStats::new(),
786            replay_recorder,
787            deadline_monitor: None,
788            oracles: OracleSuite::new(),
789            certificate: ScheduleCertificate::new(),
790        }
791    }
792
793    /// Creates a lab runtime with the default configuration.
794    #[must_use]
795    pub fn with_seed(seed: u64) -> Self {
796        Self::new(LabConfig::new(seed))
797    }
798
799    /// Returns the current virtual time.
800    #[must_use]
801    pub const fn now(&self) -> Time {
802        self.virtual_time
803    }
804
805    /// Returns the number of steps executed.
806    #[must_use]
807    pub const fn steps(&self) -> u64 {
808        self.steps
809    }
810
811    /// Returns a reference to the configuration.
812    #[must_use]
813    pub const fn config(&self) -> &LabConfig {
814        &self.config
815    }
816
817    /// Returns a handle to the lab reactor for deterministic I/O injection.
818    #[must_use]
819    pub fn lab_reactor(&self) -> &Arc<LabReactor> {
820        &self.lab_reactor
821    }
822
823    /// Returns a reference to the trace buffer handle.
824    #[must_use]
825    pub fn trace(&self) -> &TraceBufferHandle {
826        &self.state.trace
827    }
828
829    /// Returns a race report derived from the current trace buffer.
830    #[must_use]
831    pub fn detected_races(&self) -> crate::trace::dpor::RaceReport {
832        crate::trace::dpor::detect_hb_races(&self.state.trace.snapshot())
833    }
834
835    /// Returns aggregated chaos statistics for both task-side and reactor-side injection.
836    #[must_use]
837    pub fn chaos_stats(&self) -> &ChaosStats {
838        &self.chaos_stats
839    }
840
841    /// Returns the schedule certificate for determinism verification.
842    #[must_use]
843    pub fn certificate(&self) -> &ScheduleCertificate {
844        &self.certificate
845    }
846
847    /// Returns true if replay recording is enabled.
848    #[must_use]
849    pub fn has_replay_recording(&self) -> bool {
850        self.replay_recorder.is_enabled()
851    }
852
853    /// Returns a reference to the replay recorder.
854    #[must_use]
855    pub fn replay_recorder(&self) -> &TraceRecorder {
856        &self.replay_recorder
857    }
858
859    /// Takes the replay trace, leaving an empty trace in place.
860    ///
861    /// Returns `None` if recording is disabled.
862    pub fn take_replay_trace(&mut self) -> Option<ReplayTrace> {
863        self.replay_recorder.take()
864    }
865
866    /// Finishes recording and returns the replay trace.
867    ///
868    /// This consumes the replay recorder. Returns `None` if recording is disabled.
869    pub fn finish_replay_trace(&mut self) -> Option<ReplayTrace> {
870        // Take ownership by replacing with a disabled recorder
871        let recorder = std::mem::replace(&mut self.replay_recorder, TraceRecorder::disabled());
872        recorder.finish()
873    }
874
875    /// Returns true if chaos injection is enabled.
876    #[must_use]
877    pub fn has_chaos(&self) -> bool {
878        self.chaos_rng.is_some() && self.config.has_chaos()
879    }
880
881    /// Returns true if the runtime is quiescent.
882    #[must_use]
883    pub fn is_quiescent(&self) -> bool {
884        self.state.is_quiescent()
885    }
886
887    /// Advances virtual time by the given number of nanoseconds.
888    pub fn advance_time(&mut self, nanos: u64) {
889        let from = self.virtual_time;
890        self.virtual_time = self.virtual_time.saturating_add_nanos(nanos);
891        self.state.now = self.virtual_time;
892        self.virtual_clock.advance(nanos);
893        self.lab_reactor.advance_time(Duration::from_nanos(nanos));
894        // Record time advancement
895        self.replay_recorder
896            .record_time_advanced(from, self.virtual_time);
897
898        crate::tracing_compat::debug!(
899            "virtual clock advanced: delta_ms={}, new_time_ms={}",
900            nanos / 1_000_000,
901            self.virtual_time.as_nanos() / 1_000_000
902        );
903    }
904
905    /// Advances time to the given absolute time.
906    ///
907    /// If the target time is before the current time, logs an error
908    /// and does nothing (time cannot go backward).
909    pub fn advance_time_to(&mut self, time: Time) {
910        if time > self.virtual_time {
911            let from = self.virtual_time;
912            self.virtual_time = time;
913            self.state.now = self.virtual_time;
914            self.virtual_clock.advance_to(time);
915            self.lab_reactor.advance_time_to(time);
916            // Record time advancement
917            self.replay_recorder
918                .record_time_advanced(from, self.virtual_time);
919
920            crate::tracing_compat::debug!(
921                "virtual clock advanced: delta_ms={}, new_time_ms={}",
922                (time.as_nanos() - from.as_nanos()) / 1_000_000,
923                time.as_nanos() / 1_000_000
924            );
925        } else if time < self.virtual_time {
926            crate::tracing_compat::error!(
927                "virtual clock attempt to go backward: current_ms={}, requested_ms={}",
928                self.virtual_time.as_nanos() / 1_000_000,
929                time.as_nanos() / 1_000_000
930            );
931        }
932    }
933
934    // =========================================================================
935    // Virtual time control (bd-1hu19.3)
936    // =========================================================================
937
938    /// Advances virtual time to the next timer deadline.
939    ///
940    /// If a timer is pending, advances time to its deadline, processes the
941    /// expired timer(s), and returns the number of wakeups triggered.
942    /// Returns 0 if no timers are pending.
943    pub fn advance_to_next_timer(&mut self) -> usize {
944        let next = self
945            .state
946            .timer_driver_handle()
947            .and_then(|h| h.next_deadline());
948
949        let Some(deadline) = next else {
950            return 0;
951        };
952
953        if deadline <= self.virtual_time {
954            // Timer already expired, just process it
955            return self
956                .state
957                .timer_driver_handle()
958                .map_or(0, |h| h.process_timers());
959        }
960
961        let delta_nanos = deadline.as_nanos() - self.virtual_time.as_nanos();
962        self.advance_time(delta_nanos);
963
964        let wakeups = self
965            .state
966            .timer_driver_handle()
967            .map_or(0, |h| h.process_timers());
968
969        crate::tracing_compat::debug!(
970            "virtual clock auto-advance: reason=all_tasks_blocked, \
971             next_wakeup_ms={}, delta_ms={}, wakeup_count={}",
972            deadline.as_nanos() / 1_000_000,
973            delta_nanos / 1_000_000,
974            wakeups
975        );
976
977        wakeups
978    }
979
980    /// Returns the next timer deadline, if any timers are pending.
981    #[must_use]
982    pub fn next_timer_deadline(&self) -> Option<Time> {
983        self.state
984            .timer_driver_handle()
985            .and_then(|h| h.next_deadline())
986    }
987
988    fn next_reactor_deadline(&self) -> Option<Time> {
989        self.state
990            .io_driver_handle()
991            .and_then(|_| self.lab_reactor.next_event_time())
992    }
993
994    fn next_auto_advance_deadline(&self) -> Option<Time> {
995        match (self.next_timer_deadline(), self.next_reactor_deadline()) {
996            (Some(timer), Some(reactor)) => Some(timer.min(reactor)),
997            (Some(timer), None) => Some(timer),
998            (None, Some(reactor)) => Some(reactor),
999            (None, None) => None,
1000        }
1001    }
1002
1003    fn pump_due_system_events(&mut self) -> usize {
1004        let wakeups = self
1005            .state
1006            .timer_driver_handle()
1007            .map_or(0, |h| h.process_timers());
1008        self.poll_io();
1009        self.schedule_async_finalizers();
1010        self.check_deadline_monitor();
1011        wakeups
1012    }
1013
1014    /// Returns the number of pending timers.
1015    #[must_use]
1016    pub fn pending_timer_count(&self) -> usize {
1017        self.state
1018            .timer_driver_handle()
1019            .map_or(0, |h| h.pending_count())
1020    }
1021
1022    /// Runs until quiescent, automatically advancing virtual time to pending
1023    /// timer or lab-reactor deadlines whenever all tasks are idle.
1024    ///
1025    /// This enables "instant timeout testing": a scenario that would take
1026    /// 24 hours of wall-clock time completes in <1 second because every
1027    /// `sleep`/`timeout` deadline is jumped to instantly.
1028    ///
1029    /// The loop is:
1030    /// 1. Run until idle (no runnable tasks in scheduler).
1031    /// 2. If timers or lab-reactor events are pending, advance time to the
1032    ///    next deadline → go to 1.
1033    /// 3. If no pending virtual deadlines and quiescent → done.
1034    ///
1035    /// Returns a [`VirtualTimeReport`] with execution statistics.
1036    pub fn run_with_auto_advance(&mut self) -> VirtualTimeReport {
1037        let start_steps = self.steps;
1038        let mut auto_advances: u64 = 0;
1039        let mut total_wakeups: u64 = 0;
1040        let mut stuck_counter: u32 = 0;
1041        let start_time = self.virtual_time;
1042
1043        let termination = loop {
1044            // Check step limit
1045            if let Some(max) = self.config.max_steps {
1046                if self.steps >= max {
1047                    break AutoAdvanceTermination::StepLimitReached;
1048                }
1049            }
1050
1051            // Run until the scheduler is empty
1052            let is_empty = self.scheduler.lock().is_empty();
1053            if !is_empty {
1054                stuck_counter = 0;
1055                self.step();
1056                continue;
1057            }
1058
1059            // Scheduler is empty — check if we should auto-advance
1060            if let Some(deadline) = self.next_auto_advance_deadline() {
1061                if deadline > self.virtual_time {
1062                    self.advance_time_to(deadline);
1063                    let wakeups = self
1064                        .state
1065                        .timer_driver_handle()
1066                        .map_or(0, |h| h.process_timers());
1067                    auto_advances = auto_advances.saturating_add(1);
1068                    total_wakeups = total_wakeups.saturating_add(wakeups as u64);
1069                    continue;
1070                }
1071                // A timer or reactor event is already due at the current time.
1072                total_wakeups = total_wakeups.saturating_add(self.pump_due_system_events() as u64);
1073                continue;
1074            }
1075
1076            // No runnable tasks and no pending virtual deadlines → quiescent
1077            if self.is_quiescent() {
1078                break AutoAdvanceTermination::Quiescent;
1079            }
1080
1081            // Not quiescent but nothing to advance — try one more step
1082            // (there may be I/O or finalizers to process)
1083            stuck_counter = stuck_counter.saturating_add(1);
1084            if stuck_counter > 1000 {
1085                break AutoAdvanceTermination::StuckBailout;
1086            }
1087            self.step();
1088        };
1089
1090        VirtualTimeReport {
1091            steps: self.steps - start_steps,
1092            auto_advances,
1093            total_wakeups,
1094            time_start: start_time,
1095            time_end: self.virtual_time,
1096            virtual_elapsed_nanos: self.virtual_time.as_nanos() - start_time.as_nanos(),
1097            termination,
1098        }
1099    }
1100
1101    /// Pauses the virtual clock, freezing time at the current value.
1102    ///
1103    /// While paused, `advance_time()` and timer processing still work at the
1104    /// `LabRuntime` level (they update the runtime's own `virtual_time` field),
1105    /// but the underlying `VirtualClock` visible to tasks via `Cx::now()` is
1106    /// frozen. This is useful for testing timeout detection: tasks that call
1107    /// `Cx::now()` will see time standing still while the runtime can still
1108    /// orchestrate scheduling.
1109    pub fn pause_clock(&self) {
1110        self.virtual_clock.pause();
1111        crate::tracing_compat::info!(
1112            "virtual clock paused at time_ms={}",
1113            self.virtual_time.as_nanos() / 1_000_000
1114        );
1115    }
1116
1117    /// Resumes a paused virtual clock.
1118    pub fn resume_clock(&self) {
1119        self.virtual_clock.resume();
1120        crate::tracing_compat::info!(
1121            "virtual clock resumed at time_ms={}",
1122            self.virtual_time.as_nanos() / 1_000_000
1123        );
1124    }
1125
1126    /// Returns true if the virtual clock is currently paused.
1127    #[must_use]
1128    pub fn is_clock_paused(&self) -> bool {
1129        self.virtual_clock.is_paused()
1130    }
1131
1132    /// Injects a clock skew by jumping time forward by `skew_nanos`.
1133    ///
1134    /// This simulates clock drift or NTP corrections. A warning is logged
1135    /// because large jumps may affect lease/timeout correctness.
1136    #[allow(clippy::no_effect_underscore_binding)]
1137    pub fn inject_clock_skew(&mut self, skew_nanos: u64) {
1138        // Capture old time *before* advance for accurate logging.
1139        let old_nanos = self.virtual_time.as_nanos();
1140        self.advance_time(skew_nanos);
1141
1142        crate::tracing_compat::warn!(
1143            "virtual clock jump detected: old_time_ms={}, new_time_ms={}, jump_ms={} \
1144             -- may affect lease/timeout correctness",
1145            old_nanos / 1_000_000,
1146            self.virtual_time.as_nanos() / 1_000_000,
1147            skew_nanos / 1_000_000
1148        );
1149        #[cfg(not(feature = "tracing-integration"))]
1150        let _ = old_nanos;
1151    }
1152
1153    /// Runs until quiescent or max steps reached.
1154    ///
1155    /// Returns the number of steps executed.
1156    pub fn run_until_quiescent(&mut self) -> u64 {
1157        let start_steps = self.steps;
1158
1159        while !self.is_quiescent() {
1160            if let Some(max) = self.config.max_steps {
1161                if self.steps >= max {
1162                    break;
1163                }
1164            }
1165            self.step();
1166        }
1167
1168        self.steps - start_steps
1169    }
1170
1171    /// Runs until there are no runnable tasks in the scheduler.
1172    ///
1173    /// This is intentionally weaker than [`Self::run_until_quiescent`]:
1174    /// - It does **not** require all tasks to complete.
1175    /// - It does **not** require all obligations to be resolved.
1176    ///
1177    /// Use this when a test wants to "poll once" until the system is *idle*
1178    /// (e.g. a task is blocked on a channel receive) without forcing full
1179    /// completion and drain.
1180    pub fn run_until_idle(&mut self) -> u64 {
1181        let start_steps = self.steps;
1182
1183        loop {
1184            if let Some(max) = self.config.max_steps {
1185                if self.steps >= max {
1186                    break;
1187                }
1188            }
1189
1190            let is_empty = self.scheduler.lock().is_empty();
1191            if is_empty {
1192                break;
1193            }
1194
1195            self.step();
1196        }
1197
1198        self.steps - start_steps
1199    }
1200
1201    /// Runs until quiescent (or `max_steps` is reached) and returns a structured report.
1202    #[must_use]
1203    pub fn run_until_quiescent_with_report(&mut self) -> LabRunReport {
1204        let steps_delta = self.run_until_quiescent();
1205        self.report_with_steps_delta(steps_delta)
1206    }
1207
1208    /// Build a structured report for the current runtime state.
1209    ///
1210    /// This does not advance execution.
1211    #[must_use]
1212    pub fn report(&mut self) -> LabRunReport {
1213        self.report_with_steps_delta(0)
1214    }
1215
1216    /// Runs until quiescent (or `max_steps` is reached) and returns a Spork harness report.
1217    #[must_use]
1218    pub fn run_until_quiescent_spork_report(
1219        &mut self,
1220        app: impl Into<String>,
1221        attachments: Vec<HarnessAttachmentRef>,
1222    ) -> SporkHarnessReport {
1223        let run = self.run_until_quiescent_with_report();
1224        self.build_spork_report(app.into(), run, attachments)
1225    }
1226
1227    /// Build a Spork harness report for the current runtime state.
1228    ///
1229    /// This does not advance execution.
1230    #[must_use]
1231    pub fn spork_report(
1232        &mut self,
1233        app: impl Into<String>,
1234        attachments: Vec<HarnessAttachmentRef>,
1235    ) -> SporkHarnessReport {
1236        let run = self.report();
1237        self.build_spork_report(app.into(), run, attachments)
1238    }
1239
1240    fn build_spork_report(
1241        &self,
1242        app: String,
1243        run: LabRunReport,
1244        mut attachments: Vec<HarnessAttachmentRef>,
1245    ) -> SporkHarnessReport {
1246        if let Some(auto_crashpack) = self.auto_crashpack_attachment(&run, &attachments) {
1247            attachments.push(auto_crashpack);
1248        }
1249        SporkHarnessReport::new(app, &self.config, run, attachments)
1250    }
1251
1252    fn auto_crashpack_attachment(
1253        &self,
1254        run: &LabRunReport,
1255        attachments: &[HarnessAttachmentRef],
1256    ) -> Option<HarnessAttachmentRef> {
1257        if attachments
1258            .iter()
1259            .any(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack)
1260        {
1261            return None;
1262        }
1263        let crashpack = self.build_crashpack_for_report(run)?;
1264        Some(HarnessAttachmentRef::crashpack(artifact_filename(
1265            &crashpack,
1266        )))
1267    }
1268
1269    /// Build an in-memory crashpack for a failing report.
1270    ///
1271    /// Returns `None` for passing reports.
1272    #[must_use]
1273    pub fn build_crashpack_for_report(&self, run: &LabRunReport) -> Option<CrashPack> {
1274        let has_failure = !run.oracle_report.all_passed()
1275            || !run.invariant_violations.is_empty()
1276            || run.refinement_firewall_rule_id.is_some();
1277        if !has_failure {
1278            return None;
1279        }
1280
1281        let config_summary = LabConfigSummary::from_config(&self.config);
1282        let crash_config = CrashPackConfig {
1283            seed: run.seed,
1284            config_hash: config_summary.config_hash(),
1285            worker_count: self.config.worker_count,
1286            max_steps: self.config.max_steps,
1287            commit_hash: None,
1288        };
1289
1290        let (task, region) = self
1291            .state
1292            .tasks_iter()
1293            .find(|(_, task)| !task.state.is_terminal())
1294            .map(|(_, task)| (task.id, task.owner))
1295            .or_else(|| {
1296                self.state
1297                    .obligations_iter()
1298                    .find(|(_, obligation)| obligation.is_pending())
1299                    .map(|(_, obligation)| (obligation.holder, obligation.region))
1300            })
1301            .or_else(|| {
1302                self.state
1303                    .regions_iter()
1304                    .next()
1305                    .map(|(_, region)| (TaskId::testing_default(), region.id))
1306            })
1307            .or_else(|| {
1308                self.state
1309                    .root_region
1310                    .map(|root| (TaskId::testing_default(), root))
1311            })
1312            .unwrap_or((TaskId::testing_default(), RegionId::testing_default()));
1313
1314        let mut oracle_violations = run.invariant_violations.clone();
1315        oracle_violations.extend(
1316            run.oracle_report
1317                .failures()
1318                .iter()
1319                .map(|entry| entry.invariant.clone()),
1320        );
1321        if let Some(rule_id) = &run.refinement_firewall_rule_id {
1322            oracle_violations.push(format!("refinement_firewall:{rule_id}"));
1323        }
1324        if let Some(prefix_len) = run.refinement_counterexample_prefix_len {
1325            oracle_violations.push(format!(
1326                "refinement_firewall:minimal_counterexample_prefix_len={prefix_len}"
1327            ));
1328        }
1329        oracle_violations.sort();
1330        oracle_violations.dedup();
1331
1332        let trace_events = self.trace().snapshot();
1333        let mut builder = CrashPack::builder(crash_config.clone())
1334            .failure(FailureInfo {
1335                task,
1336                region,
1337                outcome: FailureOutcome::Err,
1338                virtual_time: Time::from_nanos(run.now_nanos),
1339            })
1340            .oracle_violations(oracle_violations)
1341            .replay(ReplayCommand::from_config(&crash_config, None));
1342
1343        let divergent_prefix = self.auto_divergent_prefix();
1344        if !divergent_prefix.is_empty() {
1345            builder = builder.divergent_prefix(divergent_prefix);
1346        }
1347
1348        builder = if trace_events.is_empty() {
1349            builder
1350                .fingerprint(run.trace_fingerprint)
1351                .event_count(run.trace_certificate.event_count)
1352        } else {
1353            builder.from_trace(&trace_events)
1354        };
1355
1356        match builder.build() {
1357            Ok(pack) => Some(pack),
1358            Err(err) => {
1359                let _ = &err;
1360                crate::tracing_compat::error!("failed to build crash pack for lab report: {err}");
1361                None
1362            }
1363        }
1364    }
1365
1366    fn auto_divergent_prefix(&self) -> Vec<ReplayEvent> {
1367        let Some(replay_trace) = self.replay_recorder.snapshot() else {
1368            return Vec::new();
1369        };
1370        if replay_trace.events.is_empty() {
1371            return Vec::new();
1372        }
1373
1374        let failure_index = replay_trace
1375            .events
1376            .iter()
1377            .position(
1378                |event| matches!(event, ReplayEvent::TaskCompleted { outcome, .. } if *outcome > 0),
1379            )
1380            .unwrap_or(replay_trace.events.len().saturating_sub(1));
1381
1382        crate::trace::minimal_divergent_prefix(&replay_trace, failure_index).events
1383    }
1384
1385    fn report_with_steps_delta(&mut self, steps_delta: u64) -> LabRunReport {
1386        let seed = self.config.seed;
1387        let quiescent = self.is_quiescent();
1388        let now = self.now();
1389
1390        let trace_events = self.trace().snapshot();
1391        let trace_len = trace_events.len();
1392
1393        let trace_fingerprint = if trace_events.is_empty() {
1394            // Mirror explorer behavior: ensure the report fingerprint varies by seed
1395            // even if trace capture is effectively disabled / empty.
1396            seed_fingerprint(seed)
1397        } else {
1398            trace_fingerprint(&trace_events)
1399        };
1400
1401        let schedule_hash = self.certificate().hash();
1402        let mut certificate = TraceCertificate::new();
1403        for e in &trace_events {
1404            certificate.record_event(e);
1405        }
1406        certificate.set_schedule_hash(schedule_hash);
1407
1408        self.oracles.hydrate_temporal_from_state(&self.state, now);
1409        let oracle_report = self.oracles.report(now);
1410        let temporal_invariant_failures = oracle_report
1411            .failures()
1412            .into_iter()
1413            .filter(|entry| TEMPORAL_ORACLE_INVARIANTS.contains(&entry.invariant.as_str()))
1414            .map(|entry| entry.invariant.clone())
1415            .collect::<Vec<_>>();
1416        let temporal_counterexample_prefix_len = if temporal_invariant_failures.is_empty() {
1417            None
1418        } else {
1419            let prefix_len = self.auto_divergent_prefix().len();
1420            (prefix_len > 0).then_some(prefix_len)
1421        };
1422        // br-asupersync-9ri7x0: capture the truncation watermark BEFORE
1423        // deciding whether to run the firewall. Previously, when
1424        // total_pushed > buffer_len the refinement-firewall oracle was
1425        // silently disabled and the scenario could still report
1426        // 'passed' — adversarial event-heavy scenarios could drown out
1427        // detection by deliberately exceeding the buffer. The new
1428        // contract: a truncated trace MUST surface as an explicit
1429        // violation in invariant_violations so the scenario fails
1430        // loudly, naming the seed and the truncation watermark.
1431        let trace_total_pushed = self.trace().total_pushed();
1432        let trace_buffered_len = trace_events.len() as u64;
1433        let refinement_firewall_skipped_due_to_trace_truncation =
1434            trace_total_pushed > trace_buffered_len;
1435        let refinement_violation = if refinement_firewall_skipped_due_to_trace_truncation {
1436            None
1437        } else {
1438            check_refinement_firewall(&trace_events).first_violation
1439        };
1440        let refinement_violation = refinement_violation.as_ref();
1441        let refinement_firewall_rule_id = refinement_violation.map(|v| v.rule_id.to_owned());
1442        let refinement_firewall_event_index = refinement_violation.map(|v| v.event_index);
1443        let refinement_firewall_event_seq = refinement_violation.map(|v| v.event_seq);
1444        let refinement_counterexample_prefix_len =
1445            refinement_firewall_event_index.map(|idx| idx + 1);
1446
1447        let mut invariant_violations = self
1448            .check_invariants()
1449            .into_iter()
1450            .map(|v| v.to_string())
1451            .collect::<Vec<_>>();
1452        for invariant in &temporal_invariant_failures {
1453            invariant_violations.push(format!("temporal:{invariant}"));
1454        }
1455        if let Some(prefix_len) = temporal_counterexample_prefix_len {
1456            invariant_violations.push(format!(
1457                "temporal:minimal_divergent_prefix_len={prefix_len}"
1458            ));
1459        }
1460        if let Some(rule_id) = &refinement_firewall_rule_id {
1461            invariant_violations.push(format!("refinement_firewall:{rule_id}"));
1462        }
1463        if let Some(prefix_len) = refinement_counterexample_prefix_len {
1464            invariant_violations.push(format!(
1465                "refinement_firewall:minimal_counterexample_prefix_len={prefix_len}"
1466            ));
1467        }
1468        // br-asupersync-9ri7x0: when the in-memory event buffer was
1469        // overrun, the refinement-firewall oracle could not run on the
1470        // suffix that was dropped. Report it as a hard scenario
1471        // failure with the seed + watermark so an operator can
1472        // increase trace_capacity (or split the scenario) instead of
1473        // silently shipping a green result.
1474        if refinement_firewall_skipped_due_to_trace_truncation {
1475            invariant_violations.push(format!(
1476                "refinement_firewall:scenario_failed_due_to_trace_truncation:\
1477                 seed={seed},total_pushed={trace_total_pushed},buffered={trace_buffered_len}"
1478            ));
1479        }
1480        invariant_violations.sort();
1481        invariant_violations.dedup();
1482
1483        LabRunReport {
1484            seed,
1485            steps_delta,
1486            steps_total: self.steps(),
1487            quiescent,
1488            now_nanos: now.as_nanos(),
1489            trace_len,
1490            trace_fingerprint,
1491            trace_certificate: LabTraceCertificateSummary {
1492                event_hash: certificate.event_hash(),
1493                event_count: certificate.event_count(),
1494                schedule_hash: certificate.schedule_hash(),
1495            },
1496            oracle_report,
1497            invariant_violations,
1498            temporal_invariant_failures,
1499            temporal_counterexample_prefix_len,
1500            refinement_firewall_rule_id,
1501            refinement_firewall_event_index,
1502            refinement_firewall_event_seq,
1503            refinement_counterexample_prefix_len,
1504            refinement_firewall_skipped_due_to_trace_truncation,
1505        }
1506    }
1507
1508    /// Enable deadline monitoring with the default warning handler.
1509    pub fn enable_deadline_monitoring(&mut self, config: MonitorConfig) {
1510        self.enable_deadline_monitoring_with_handler(config, default_warning_handler);
1511    }
1512
1513    /// Enable deadline monitoring with a custom warning handler.
1514    pub fn enable_deadline_monitoring_with_handler<F>(&mut self, config: MonitorConfig, f: F)
1515    where
1516        F: Fn(DeadlineWarning) + Send + Sync + 'static,
1517    {
1518        let mut monitor = DeadlineMonitor::new(config);
1519        monitor.on_warning(f);
1520        self.deadline_monitor = Some(monitor);
1521    }
1522
1523    /// Returns a mutable reference to the deadline monitor, if enabled.
1524    pub fn deadline_monitor_mut(&mut self) -> Option<&mut DeadlineMonitor> {
1525        self.deadline_monitor.as_mut()
1526    }
1527
1528    /// Executes a single step.
1529    #[allow(clippy::too_many_lines)]
1530    fn step(&mut self) {
1531        self.steps += 1;
1532        let rng_value = self.rng.next_u64();
1533        if self.steps < 50 {
1534            crate::tracing_compat::trace!(
1535                "lab runtime rng sample: rng_value={}, worker_hint={}",
1536                rng_value,
1537                (rng_value >> 32) as usize % self.config.worker_count.max(1)
1538            );
1539        }
1540        self.replay_recorder.record_rng_value(rng_value);
1541        self.check_futurelocks();
1542        if let Some(timer) = self.state.timer_driver_handle() {
1543            let _ = timer.process_timers();
1544        }
1545        self.poll_io();
1546        self.schedule_async_finalizers();
1547
1548        // 1. Choose a worker and pop a task (deterministic multi-worker model)
1549        let worker_count = self.config.worker_count.max(1);
1550        // Use higher bits of rng_value since xorshift64 has poor low-bit entropy
1551        let worker_hint = ((rng_value >> 32) as usize) % worker_count;
1552        let now = self.now();
1553        let (task_id, dispatch_lane) = {
1554            let mut sched = self.scheduler.lock();
1555            if let Some((tid, lane)) = sched.pop_for_worker(worker_hint, rng_value, now) {
1556                (tid, lane)
1557            } else if let Some(tid) = sched.steal_for_worker(worker_hint, rng_value.rotate_left(17))
1558            {
1559                (tid, DispatchLane::Stolen)
1560            } else {
1561                drop(sched);
1562                self.check_deadline_monitor();
1563                return;
1564            }
1565        };
1566
1567        // Record task scheduling in certificate and replay recorder
1568        self.certificate.record(task_id, dispatch_lane, self.steps);
1569        self.replay_recorder
1570            .record_task_scheduled(task_id, self.steps);
1571
1572        // 2. Pre-poll chaos injection
1573        if self.inject_pre_poll_chaos(task_id) {
1574            // Chaos caused the task to be skipped (e.g., cancelled, budget exhausted)
1575            return;
1576        }
1577
1578        // 3. Prepare context and enforce budget
1579        let priority = self.state.task(task_id).map_or(0, |record| {
1580            record.cx_inner.as_ref().map_or(0, |inner| {
1581                let mut guard = inner.write();
1582
1583                // Enforce poll quota
1584                if guard.budget.consume_poll().is_none() {
1585                    guard.cancel_requested = true;
1586                    guard
1587                        .fast_cancel
1588                        .store(true, std::sync::atomic::Ordering::Release);
1589                    if let Some(existing) = &mut guard.cancel_reason {
1590                        existing.strengthen(&crate::types::CancelReason::poll_quota());
1591                    } else {
1592                        guard.cancel_reason = Some(crate::types::CancelReason::poll_quota());
1593                    }
1594                }
1595
1596                guard.budget.priority
1597            })
1598        });
1599
1600        let waker = Waker::from(Arc::new(TaskWaker {
1601            task_id,
1602            priority,
1603            scheduler: self.scheduler.clone(),
1604        }));
1605        let mut cx = Context::from_waker(&waker);
1606
1607        // Set cancel_waker so abort_with_reason can reschedule cancelled tasks.
1608        if let Some(record) = self.state.task(task_id) {
1609            if let Some(inner) = record.cx_inner.as_ref() {
1610                let cancel_waker = Waker::from(Arc::new(CancelTaskWaker {
1611                    task_id,
1612                    priority,
1613                    scheduler: self.scheduler.clone(),
1614                }));
1615                {
1616                    let mut guard = inner.write();
1617                    guard.cancel_waker = Some(cancel_waker);
1618                }
1619            }
1620        }
1621
1622        let current_cx = self
1623            .state
1624            .task(task_id)
1625            .and_then(|record| record.cx.clone());
1626        let _cx_guard = crate::cx::Cx::set_current(current_cx);
1627
1628        let started_running = self
1629            .state
1630            .update_task(task_id, |record| {
1631                let old_state = record.state.clone();
1632                if record.start_running() {
1633                    Some((old_state, record.state.clone()))
1634                } else {
1635                    None
1636                }
1637            })
1638            .flatten();
1639
1640        if let Some((from_state, to_state)) = started_running.as_ref() {
1641            if self.config.has_cancellation_oracle() {
1642                self.notify_cancellation_oracle_task_transition(task_id, from_state, to_state);
1643            }
1644        }
1645
1646        // 4. Poll the task
1647        if self.steps < 50 {
1648            crate::tracing_compat::trace!(
1649                "lab runtime executing task {:?} at step {}",
1650                task_id,
1651                self.steps
1652            );
1653        }
1654
1655        // Notify oracle of task poll
1656        if self.config.has_cancellation_oracle() {
1657            self.notify_cancellation_oracle_task_poll(task_id);
1658        }
1659
1660        let result = if let Some(stored) = self.state.get_stored_future(task_id) {
1661            stored.poll(&mut cx)
1662        } else {
1663            // Task lost (should not happen if consistent)
1664            return;
1665        };
1666
1667        // Record the poll so futurelock detection uses the correct idle step count.
1668        let _ = self.state.update_task(task_id, |record| {
1669            record.mark_polled(self.steps);
1670        });
1671
1672        let cancel_ack = self.consume_cancel_ack(task_id);
1673
1674        // Notify oracle of cancel acknowledgment
1675        if cancel_ack && self.config.has_cancellation_oracle() {
1676            self.notify_cancellation_oracle_cancel_ack(task_id);
1677        }
1678
1679        // 5. Handle result
1680        match result {
1681            Poll::Ready(outcome) => {
1682                // Task completed
1683                self.state.remove_stored_future(task_id);
1684                self.scheduler.lock().forget_task(task_id);
1685
1686                // Update state to Completed if not already terminal
1687                let mut oracle_transitions = Vec::new(); // Collect transitions for later oracle notification
1688
1689                let _ = self.state.update_task(task_id, |record| {
1690                    if !record.state.is_terminal() {
1691                        let old_state = record.state.clone();
1692                        let record_outcome = match outcome {
1693                            crate::types::Outcome::Ok(()) => crate::types::Outcome::Ok(()),
1694                            crate::types::Outcome::Err(()) => crate::types::Outcome::Err(
1695                                crate::error::Error::new(crate::error::ErrorKind::Internal),
1696                            ),
1697                            crate::types::Outcome::Cancelled(r) => {
1698                                crate::types::Outcome::Cancelled(r)
1699                            }
1700                            crate::types::Outcome::Panicked(p) => {
1701                                crate::types::Outcome::Panicked(p)
1702                            }
1703                        };
1704                        let completed_via_cancel =
1705                            if matches!(record_outcome, crate::types::Outcome::Ok(())) {
1706                                let should_cancel = matches!(
1707                                    record.state,
1708                                    TaskState::Cancelling { .. } | TaskState::Finalizing { .. }
1709                                ) || (cancel_ack
1710                                    && matches!(record.state, TaskState::CancelRequested { .. }));
1711                                if should_cancel {
1712                                    if matches!(record.state, TaskState::CancelRequested { .. }) {
1713                                        let state_before = record.state.clone();
1714                                        let _ = record.acknowledge_cancel();
1715                                        oracle_transitions
1716                                            .push((state_before, record.state.clone()));
1717                                    }
1718                                    if matches!(record.state, TaskState::Cancelling { .. }) {
1719                                        let state_before = record.state.clone();
1720                                        record.cleanup_done();
1721                                        oracle_transitions
1722                                            .push((state_before, record.state.clone()));
1723                                    }
1724                                    if matches!(record.state, TaskState::Finalizing { .. }) {
1725                                        let state_before = record.state.clone();
1726                                        record.finalize_done();
1727                                        oracle_transitions
1728                                            .push((state_before, record.state.clone()));
1729                                    }
1730                                    matches!(
1731                                        record.state,
1732                                        TaskState::Completed(crate::types::Outcome::Cancelled(_))
1733                                    )
1734                                } else {
1735                                    false
1736                                }
1737                            } else {
1738                                false
1739                            };
1740                        if !completed_via_cancel {
1741                            record.complete(record_outcome);
1742                            oracle_transitions.push((old_state, record.state.clone()));
1743                        }
1744                    }
1745                });
1746
1747                // Notify oracle of all state transitions after all mutations are complete
1748                if self.config.has_cancellation_oracle() {
1749                    for (from_state, to_state) in oracle_transitions {
1750                        self.notify_cancellation_oracle_task_transition(
1751                            task_id,
1752                            &from_state,
1753                            &to_state,
1754                        );
1755                    }
1756                }
1757
1758                // Record task completion with severity from the finalized task
1759                // record. Must happen AFTER state finalization above because
1760                // create_task wraps user futures to always return Outcome::Ok(())
1761                // — the real severity comes from the cancel protocol state machine.
1762                let final_severity =
1763                    self.state
1764                        .task(task_id)
1765                        .map_or(crate::types::Severity::Ok, |record| match &record.state {
1766                            TaskState::Completed(outcome) => outcome.severity(),
1767                            _ => crate::types::Severity::Ok,
1768                        });
1769                self.replay_recorder
1770                    .record_task_completed(task_id, final_severity);
1771
1772                if let Some(monitor) = &mut self.deadline_monitor {
1773                    if let Some(record) = self.state.task(task_id) {
1774                        let now = self.state.now;
1775                        let duration =
1776                            Duration::from_nanos(now.duration_since(record.created_at()));
1777                        let (task_type, deadline) = record
1778                            .cx_inner
1779                            .as_ref()
1780                            .map(|inner| inner.read())
1781                            .map_or_else(
1782                                || ("default".to_string(), None),
1783                                |inner| {
1784                                    (
1785                                        inner
1786                                            .task_type
1787                                            .clone()
1788                                            .unwrap_or_else(|| "default".to_string()),
1789                                        inner.budget.deadline,
1790                                    )
1791                                },
1792                            );
1793                        monitor.record_completion(task_id, &task_type, duration, deadline, now);
1794                    }
1795                }
1796
1797                // Notify waiters
1798                let waiters = self.state.task_completed(task_id);
1799
1800                // br-asupersync-iwqn3q: hoist priority lookup OUT of
1801                // the scheduler-locked scope. cx_inner is an
1802                // E(Config)-tier RwLock; the scheduler is an A(Tasks)
1803                // mutex. The project's lock ordering requires
1804                // E → D → B → A → C, so cx_inner.read() must precede
1805                // scheduler.lock(). Acquiring them in the loop
1806                // body inverted the order and could deadlock against
1807                // any thread holding cx_inner.write() while waiting
1808                // for the scheduler. Snapshot the (waiter, priority)
1809                // tuples first, THEN acquire the scheduler.
1810                //
1811                // The sibling pattern at schedule_for_cancel
1812                // (line ~2002) already gets this right; this site
1813                // was the asymmetric outlier.
1814                let scheduled: Vec<(TaskId, u8)> = waiters
1815                    .into_iter()
1816                    .map(|w| {
1817                        let prio = self
1818                            .state
1819                            .task(w)
1820                            .and_then(|t| t.cx_inner.as_ref())
1821                            .map_or(0, |inner| inner.read().budget.priority);
1822                        (w, prio)
1823                    })
1824                    .collect();
1825                let mut sched = self.scheduler.lock();
1826                for (waiter, prio) in scheduled {
1827                    sched.schedule(waiter, prio);
1828                }
1829            }
1830            Poll::Pending => {
1831                // Task yielded. Waker will reschedule it when ready.
1832                // Note: If the task yielded via `cx.waker().wake_by_ref()`, it might already be scheduled.
1833                // If it yielded for I/O or other events, it won't be scheduled until that event fires.
1834
1835                // Record task yielding
1836                self.replay_recorder.record_task_yielded(task_id);
1837
1838                // 6. Post-poll chaos injection (spurious wakeups for pending tasks)
1839                self.inject_post_poll_chaos(task_id, priority);
1840            }
1841        }
1842
1843        self.check_deadline_monitor();
1844    }
1845
1846    fn check_deadline_monitor(&mut self) {
1847        if let Some(monitor) = &mut self.deadline_monitor {
1848            let now = self.state.now;
1849            monitor.check(now, self.state.tasks_iter().map(|(_, record)| record));
1850        }
1851    }
1852
1853    fn poll_io(&mut self) {
1854        let Some(handle) = self.state.io_driver_handle() else {
1855            return;
1856        };
1857        let now = self.state.now;
1858        let (state, recorder, seen) = (
1859            &mut self.state,
1860            &mut self.replay_recorder,
1861            &mut self.seen_io_tokens,
1862        );
1863        if let Err(error) = handle.turn_with(Some(Duration::ZERO), |event, interest| {
1864            let token = event.token.0;
1865            let interest = interest.unwrap_or(event.ready);
1866            if seen.insert(token) {
1867                state.record_trace_event(|seq| {
1868                    TraceEvent::io_requested(seq, now, token as u64, interest.bits())
1869                });
1870            }
1871            state.record_trace_event(|seq| {
1872                TraceEvent::io_ready(seq, now, token as u64, event.ready.bits())
1873            });
1874            recorder.record_io_ready(
1875                token as u64,
1876                event.is_readable(),
1877                event.is_writable(),
1878                event.is_error(),
1879                event.is_hangup(),
1880            );
1881        }) {
1882            let _ = &error;
1883            crate::tracing_compat::warn!(
1884                error = ?error,
1885                "lab runtime io_driver poll failed"
1886            );
1887        }
1888        self.sync_reactor_chaos_stats();
1889    }
1890
1891    /// Injects chaos before polling a task.
1892    ///
1893    /// Returns `true` if the task should be skipped (e.g., cancelled or budget exhausted).
1894    fn inject_pre_poll_chaos(&mut self, task_id: TaskId) -> bool {
1895        let Some(chaos_config) = self.config.chaos.clone() else {
1896            return false;
1897        };
1898        let Some(chaos_rng) = &mut self.chaos_rng else {
1899            return false;
1900        };
1901
1902        let cancel = chaos_rng.should_inject_cancel(&chaos_config);
1903
1904        // Check for delay injection
1905        let delay = chaos_rng
1906            .should_inject_delay(&chaos_config)
1907            .then(|| chaos_rng.next_delay(&chaos_config));
1908
1909        // Check for budget exhaustion injection
1910        let budget_exhaust = chaos_rng.should_inject_budget_exhaust(&chaos_config);
1911        let skip_poll = cancel | budget_exhaust;
1912        self.chaos_stats
1913            .record_pre_poll_outcomes(cancel, delay, budget_exhaust);
1914
1915        // Now apply the injections (no more borrowing chaos_rng).
1916        // Cancel and budget_exhaust are independent — apply both when both fire.
1917        if cancel {
1918            self.inject_cancel(task_id);
1919        }
1920
1921        if let Some(d) = delay {
1922            self.advance_time(Self::duration_nanos_saturating(d));
1923        }
1924
1925        if budget_exhaust {
1926            self.inject_budget_exhaust(task_id);
1927        }
1928
1929        if skip_poll {
1930            self.reschedule_after_chaos_skip(task_id);
1931        }
1932
1933        skip_poll
1934    }
1935
1936    #[inline]
1937    fn duration_nanos_saturating(duration: Duration) -> u64 {
1938        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
1939    }
1940
1941    /// Injects chaos after polling a task that returned Pending.
1942    fn inject_post_poll_chaos(&mut self, task_id: TaskId, priority: u8) {
1943        let Some(chaos_config) = self.config.chaos.clone() else {
1944            return;
1945        };
1946        let Some(chaos_rng) = &mut self.chaos_rng else {
1947            return;
1948        };
1949
1950        // br-asupersync-4so3w3: gate wakeup_storm injection on at-least-
1951        // one-open-region. After every region has closed, the lab is in
1952        // a quiescence state that production reaches via region drop;
1953        // injecting a spurious wakeup at that point would synthesize a
1954        // schedule that production cannot reproduce, defeating the
1955        // whole point of chaos-driven trace minimisation.
1956        let has_open_region = self.state.live_region_count() > 0;
1957        let wakeup_count = if chaos_rng.should_inject_wakeup_storm(&chaos_config, has_open_region) {
1958            Some(chaos_rng.next_wakeup_count(&chaos_config))
1959        } else {
1960            None
1961        };
1962
1963        // br-asupersync-7uu7sa: even when SOME region is open, this
1964        // specific task may belong to a region that has already
1965        // transitioned to Closing/Draining/Finalizing/Closed during a different
1966        // chaos action in the same step. Re-polling such a task
1967        // violates the structured-concurrency contract the oracles
1968        // assume — it produces a 'cancel-aware future re-polled after
1969        // region close' code path that production never executes.
1970        // Filter post-decision so chaos budget accounting stays in
1971        // sync with the global gate decision.
1972        let target_region_open = self
1973            .state
1974            .task(task_id)
1975            .map(|t| t.owner)
1976            .and_then(|owner| self.state.region(owner))
1977            .is_some_and(|region| region.state().can_accept_work());
1978
1979        // Apply the injection (no more borrowing chaos_rng)
1980        if let Some(count) = wakeup_count
1981            && target_region_open
1982        {
1983            self.chaos_stats.record_wakeup_storm(count as u64);
1984            self.inject_spurious_wakes(task_id, priority, count);
1985        } else {
1986            self.chaos_stats.record_no_injection();
1987        }
1988    }
1989
1990    fn sync_reactor_chaos_stats(&mut self) {
1991        let current = self.lab_reactor.chaos_stats();
1992        let previous = &self.seen_reactor_chaos_stats;
1993        let delta = ChaosStats {
1994            cancellations: current.cancellations.saturating_sub(previous.cancellations),
1995            delays: current.delays.saturating_sub(previous.delays),
1996            total_delay: current.total_delay.saturating_sub(previous.total_delay),
1997            io_errors: current.io_errors.saturating_sub(previous.io_errors),
1998            wakeup_storms: current.wakeup_storms.saturating_sub(previous.wakeup_storms),
1999            spurious_wakeups: current
2000                .spurious_wakeups
2001                .saturating_sub(previous.spurious_wakeups),
2002            budget_exhaustions: current
2003                .budget_exhaustions
2004                .saturating_sub(previous.budget_exhaustions),
2005            decision_points: current
2006                .decision_points
2007                .saturating_sub(previous.decision_points),
2008        };
2009        self.chaos_stats.merge(&delta);
2010        self.seen_reactor_chaos_stats = current;
2011    }
2012
2013    fn reschedule_after_chaos_skip(&self, task_id: TaskId) {
2014        let Some(record) = self.state.task(task_id) else {
2015            return;
2016        };
2017        if record.state.is_terminal() {
2018            return;
2019        }
2020        let priority = record
2021            .cx_inner
2022            .as_ref()
2023            .map_or(0, |inner| inner.read().budget.priority);
2024        let mut sched = self.scheduler.lock();
2025        sched.schedule_cancel(task_id, priority);
2026    }
2027
2028    fn schedule_async_finalizers(&mut self) {
2029        let tasks = self.state.drain_ready_async_finalizers();
2030        if tasks.is_empty() {
2031            return;
2032        }
2033        let mut sched = self.scheduler.lock();
2034        for (task_id, priority) in tasks {
2035            sched.schedule(task_id, priority);
2036        }
2037    }
2038
2039    fn consume_cancel_ack(&mut self, task_id: TaskId) -> bool {
2040        self.state
2041            .update_task(task_id, |record| {
2042                let Some(inner) = record.cx_inner.as_ref() else {
2043                    return false;
2044                };
2045                let mut acknowledged = false;
2046                {
2047                    let mut guard = inner.write();
2048                    if guard.cancel_acknowledged {
2049                        guard.cancel_acknowledged = false;
2050                        drop(guard);
2051                        acknowledged = true;
2052                    }
2053                }
2054                if acknowledged {
2055                    let _ = record.acknowledge_cancel();
2056                }
2057                acknowledged
2058            })
2059            .unwrap_or(false)
2060    }
2061
2062    /// Injects a cancellation for a task.
2063    fn inject_cancel(&mut self, task_id: TaskId) {
2064        use crate::types::{Budget, CancelReason};
2065
2066        // Record replay event
2067        self.replay_recorder.record_cancel_injection(task_id);
2068
2069        // Record the cancel request in the oracle
2070        let reason = CancelReason::user("chaos-injected");
2071        if self.config.has_cancellation_oracle() {
2072            self.oracles.cancellation_protocol.on_cancel_request(
2073                task_id,
2074                reason.clone(),
2075                self.virtual_time,
2076            );
2077        }
2078
2079        // Mark the task as cancel-requested with chaos reason.
2080        let transition = self
2081            .state
2082            .update_task(task_id, |record| {
2083                if !record.state.is_terminal() {
2084                    let old_state = record.state.clone();
2085                    record.request_cancel_with_budget(reason, Budget::ZERO);
2086                    Some((old_state, record.state.clone()))
2087                } else {
2088                    None
2089                }
2090            })
2091            .flatten();
2092
2093        // Record the state transition in the oracle after mutation is complete.
2094        if let Some((old_state, new_state)) = transition {
2095            if self.config.has_cancellation_oracle() {
2096                self.oracles.cancellation_protocol.on_transition(
2097                    task_id,
2098                    &old_state,
2099                    &new_state,
2100                    self.virtual_time,
2101                );
2102            }
2103        }
2104
2105        // Emit trace event
2106        self.state.record_trace_event(|seq| {
2107            TraceEvent::new(
2108                seq,
2109                self.virtual_time,
2110                TraceEventKind::ChaosInjection,
2111                TraceData::Chaos {
2112                    kind: "cancel".to_string(),
2113                    task: Some(task_id),
2114                    detail: "chaos-injected cancellation".to_string(),
2115                },
2116            )
2117        });
2118    }
2119
2120    /// Notifies the cancellation protocol oracle about runtime events.
2121    pub fn notify_cancellation_oracle_task_create(&mut self, task_id: TaskId, region_id: RegionId) {
2122        self.oracles
2123            .cancellation_protocol
2124            .on_task_create(task_id, region_id);
2125    }
2126
2127    /// Notifies the cancellation protocol oracle about region creation.
2128    pub fn notify_cancellation_oracle_region_create(
2129        &mut self,
2130        region_id: RegionId,
2131        parent: Option<RegionId>,
2132    ) {
2133        self.oracles
2134            .cancellation_protocol
2135            .on_region_create(region_id, parent);
2136    }
2137
2138    /// Notifies the cancellation protocol oracle about task state transitions.
2139    pub fn notify_cancellation_oracle_task_transition(
2140        &mut self,
2141        task_id: TaskId,
2142        from: &crate::record::task::TaskState,
2143        to: &crate::record::task::TaskState,
2144    ) {
2145        self.oracles
2146            .cancellation_protocol
2147            .on_transition(task_id, from, to, self.virtual_time);
2148    }
2149
2150    /// Notifies the cancellation protocol oracle about cancel requests.
2151    pub fn notify_cancellation_oracle_cancel_request(
2152        &mut self,
2153        task_id: TaskId,
2154        reason: crate::types::CancelReason,
2155    ) {
2156        self.oracles
2157            .cancellation_protocol
2158            .on_cancel_request(task_id, reason, self.virtual_time);
2159    }
2160
2161    /// Notifies the cancellation protocol oracle about cancel acknowledgments.
2162    pub fn notify_cancellation_oracle_cancel_ack(&mut self, task_id: TaskId) {
2163        self.oracles
2164            .cancellation_protocol
2165            .on_cancel_ack(task_id, self.virtual_time);
2166    }
2167
2168    /// Notifies the cancellation protocol oracle about task polling.
2169    pub fn notify_cancellation_oracle_task_poll(&mut self, task_id: TaskId) {
2170        self.oracles.cancellation_protocol.on_task_poll(task_id);
2171    }
2172
2173    /// Notifies the cancellation protocol oracle about mask entry.
2174    pub fn notify_cancellation_oracle_mask_enter(&mut self, task_id: TaskId) {
2175        self.oracles
2176            .cancellation_protocol
2177            .on_mask_enter(task_id, self.virtual_time);
2178    }
2179
2180    /// Notifies the cancellation protocol oracle about mask exit.
2181    pub fn notify_cancellation_oracle_mask_exit(&mut self, task_id: TaskId) {
2182        self.oracles
2183            .cancellation_protocol
2184            .on_mask_exit(task_id, self.virtual_time);
2185    }
2186
2187    /// Notifies the cancellation protocol oracle about region cancellation.
2188    pub fn notify_cancellation_oracle_region_cancel(
2189        &mut self,
2190        region_id: RegionId,
2191        reason: crate::types::CancelReason,
2192    ) {
2193        self.oracles
2194            .cancellation_protocol
2195            .on_region_cancel(region_id, reason, self.virtual_time);
2196    }
2197
2198    /// Checks the cancellation protocol oracle for violations and optionally enforces them.
2199    pub fn check_cancellation_protocol(
2200        &mut self,
2201    ) -> Result<(), crate::lab::oracle::CancellationProtocolViolation> {
2202        if !self.config.has_cancellation_oracle() {
2203            return Ok(());
2204        }
2205
2206        let result = self.oracles.cancellation_protocol.check();
2207
2208        if let Err(ref violation) = result {
2209            if self.config.panic_on_cancellation_violation {
2210                // Configurable enforcement: panic in enforce mode
2211                panic!("Cancellation protocol violation detected: {violation}");
2212            } else {
2213                // Warn mode: log the violation
2214                crate::tracing_compat::warn!(
2215                    violation = %violation,
2216                    "Cancellation protocol violation detected"
2217                );
2218            }
2219        }
2220
2221        result
2222    }
2223
2224    /// Injects budget exhaustion for a task.
2225    fn inject_budget_exhaust(&mut self, task_id: TaskId) {
2226        // Record replay event
2227        self.replay_recorder
2228            .record_budget_exhaust_injection(task_id);
2229
2230        // Set the task's budget quotas to zero
2231        if let Some(record) = self.state.task(task_id) {
2232            if let Some(cx_inner) = &record.cx_inner {
2233                let mut inner = cx_inner.write();
2234                inner.budget.poll_quota = 0;
2235                inner.budget.cost_quota = Some(0);
2236            }
2237        }
2238
2239        // Emit trace event
2240        self.state.record_trace_event(|seq| {
2241            TraceEvent::new(
2242                seq,
2243                self.virtual_time,
2244                TraceEventKind::ChaosInjection,
2245                TraceData::Chaos {
2246                    kind: "budget_exhaust".to_string(),
2247                    task: Some(task_id),
2248                    detail: "chaos-injected budget exhaustion".to_string(),
2249                },
2250            )
2251        });
2252    }
2253
2254    /// Injects spurious wakeups for a task.
2255    fn inject_spurious_wakes(&mut self, task_id: TaskId, priority: u8, count: usize) {
2256        // br-asupersync-7uu7sa: defense-in-depth — even when the chaos
2257        // call site (inject_post_poll_chaos) gates correctly, future
2258        // callers may invoke this method directly. Refuse to wake a
2259        // task whose owning region is no longer accepting normal work
2260        // (Closing / Draining / Finalizing / Closed). This silently no-ops rather
2261        // than panicking so chaos campaigns with TaskId selection that
2262        // races region close don't artificially abort.
2263        let owner_open = self
2264            .state
2265            .task(task_id)
2266            .map(|t| t.owner)
2267            .and_then(|owner| self.state.region(owner))
2268            .is_some_and(|region| region.state().can_accept_work());
2269        if !owner_open {
2270            return;
2271        }
2272
2273        // Record replay event
2274        self.replay_recorder
2275            .record_wakeup_storm_injection(task_id, u32::try_from(count).unwrap_or(u32::MAX));
2276
2277        // Schedule the task multiple times (spurious wakeups)
2278        let mut sched = self.scheduler.lock();
2279        sched.inject_spurious_wakes(task_id, priority, count);
2280        drop(sched);
2281
2282        // Emit trace event
2283        self.state.record_trace_event(|seq| {
2284            TraceEvent::new(
2285                seq,
2286                self.virtual_time,
2287                TraceEventKind::ChaosInjection,
2288                TraceData::Chaos {
2289                    kind: "wakeup_storm".to_string(),
2290                    task: Some(task_id),
2291                    detail: format!("chaos-injected {count} spurious wakeups"),
2292                },
2293            )
2294        });
2295    }
2296
2297    /// Public wrapper for `step()` for use in tests.
2298    ///
2299    /// This is useful for testing determinism across multiple step executions.
2300    pub fn step_for_test(&mut self) {
2301        self.step();
2302    }
2303
2304    /// Checks invariants and returns any violations.
2305    #[must_use]
2306    pub fn check_invariants(&mut self) -> Vec<InvariantViolation> {
2307        let mut violations = Vec::new();
2308
2309        // Check cancellation protocol oracle
2310        if let Err(violation) = self.check_cancellation_protocol() {
2311            violations.push(InvariantViolation::CancellationProtocol {
2312                violation: violation.to_string(),
2313            });
2314        }
2315
2316        // Check for obligation leaks
2317        let leaks = self.obligation_leaks();
2318        if !leaks.is_empty() {
2319            for leak in &leaks {
2320                let _ = self.state.mark_obligation_leaked(leak.obligation);
2321            }
2322            violations.push(InvariantViolation::ObligationLeak { leaks });
2323        }
2324
2325        violations.extend(self.futurelock_violations());
2326        violations.extend(self.quiescence_violations());
2327
2328        // Check for task leaks (non-terminal tasks)
2329        let task_leak_count = self.task_leaks();
2330        if task_leak_count > 0 {
2331            violations.push(InvariantViolation::TaskLeak {
2332                count: task_leak_count,
2333            });
2334        }
2335
2336        violations
2337    }
2338
2339    fn obligation_leaks(&self) -> Vec<ObligationLeak> {
2340        let mut leaks = Vec::new();
2341
2342        for (_, obligation) in self.state.obligations_iter() {
2343            if !obligation.is_pending() {
2344                continue;
2345            }
2346
2347            let holder_terminal = self
2348                .state
2349                .task(obligation.holder)
2350                .is_none_or(|t| t.state.is_terminal());
2351            let region_closed = self
2352                .state
2353                .region(obligation.region)
2354                .is_none_or(|r| r.state().is_terminal());
2355
2356            if holder_terminal || region_closed {
2357                leaks.push(ObligationLeak {
2358                    obligation: obligation.id,
2359                    kind: obligation.kind,
2360                    holder: obligation.holder,
2361                    region: obligation.region,
2362                });
2363            }
2364        }
2365
2366        leaks
2367    }
2368
2369    fn task_leaks(&self) -> usize {
2370        self.state
2371            .tasks_iter()
2372            .filter(|(_, t)| !t.state.is_terminal())
2373            .count()
2374    }
2375
2376    fn quiescence_violations(&self) -> Vec<InvariantViolation> {
2377        let mut violations = Vec::new();
2378        for (_, region) in self.state.regions_iter() {
2379            if region.state().is_terminal() {
2380                // Check if any children or tasks are NOT terminal
2381                let live_tasks = region
2382                    .task_ids()
2383                    .iter()
2384                    .any(|&tid| self.state.task(tid).is_some_and(|t| !t.state.is_terminal()));
2385
2386                let live_children = region.child_ids().iter().any(|&rid| {
2387                    self.state
2388                        .region(rid)
2389                        .is_some_and(|r| !r.state().is_terminal())
2390                });
2391
2392                if live_tasks || live_children {
2393                    violations.push(InvariantViolation::QuiescenceViolation);
2394                }
2395            }
2396        }
2397        violations
2398    }
2399
2400    fn futurelock_violations(&self) -> Vec<InvariantViolation> {
2401        let threshold = self.config.futurelock_max_idle_steps;
2402        if threshold == 0 {
2403            return Vec::new();
2404        }
2405
2406        let current_step = self.steps;
2407        let mut violations = Vec::new();
2408
2409        for (_, task) in self.state.tasks_iter() {
2410            if task.state.is_terminal() {
2411                continue;
2412            }
2413
2414            let mut held = Vec::new();
2415            for (_, obligation) in self.state.obligations_iter() {
2416                if obligation.is_pending() && obligation.holder == task.id {
2417                    held.push(obligation.id);
2418                }
2419            }
2420
2421            if held.is_empty() {
2422                continue;
2423            }
2424
2425            let idle_steps = current_step.saturating_sub(task.last_polled_step);
2426            if idle_steps > threshold {
2427                violations.push(InvariantViolation::Futurelock {
2428                    task: task.id,
2429                    region: task.owner,
2430                    idle_steps,
2431                    held,
2432                });
2433            }
2434        }
2435
2436        violations
2437    }
2438
2439    fn check_futurelocks(&self) {
2440        let violations = self.futurelock_violations();
2441        if violations.is_empty() {
2442            return;
2443        }
2444
2445        for v in violations {
2446            let InvariantViolation::Futurelock {
2447                task,
2448                region,
2449                idle_steps,
2450                held,
2451            } = v
2452            else {
2453                continue;
2454            };
2455
2456            let mut held_kinds = Vec::new();
2457            for oid in &held {
2458                for (_, obligation) in self.state.obligations_iter() {
2459                    if obligation.id == *oid {
2460                        held_kinds.push((obligation.id, obligation.kind));
2461                        break;
2462                    }
2463                }
2464            }
2465
2466            self.state.record_trace_event(|seq| {
2467                TraceEvent::new(
2468                    seq,
2469                    self.virtual_time,
2470                    TraceEventKind::FuturelockDetected,
2471                    TraceData::Futurelock {
2472                        task,
2473                        region,
2474                        idle_steps,
2475                        held: held_kinds,
2476                    },
2477                )
2478            });
2479
2480            assert!(
2481                !self.config.panic_on_futurelock,
2482                "futurelock detected: {task} in {region} idle={idle_steps} held={held:?}"
2483            );
2484        }
2485    }
2486}
2487
2488const DEFAULT_LAB_CANCEL_STREAK_LIMIT: usize = 16;
2489
2490#[derive(Debug, Clone, Copy)]
2491struct PendingSpuriousWake {
2492    priority: u8,
2493    remaining: usize,
2494}
2495
2496#[derive(Debug)]
2497/// Deterministic lab scheduler with per-worker queues.
2498///
2499/// This is a single-threaded model of multi-worker scheduling used by the lab
2500/// runtime to simulate parallel execution deterministically.
2501pub struct LabScheduler {
2502    workers: Vec<crate::runtime::scheduler::PriorityScheduler>,
2503    scheduled: DetHashSet<TaskId>,
2504    pending_spurious_wakes: DetHashMap<TaskId, PendingSpuriousWake>,
2505    /// Task → worker assignment, indexed by arena slot.
2506    assignments: Vec<Option<usize>>,
2507    next_worker: usize,
2508    cancel_streak: Vec<usize>,
2509    cancel_streak_limit: usize,
2510}
2511
2512impl LabScheduler {
2513    fn new(worker_count: usize) -> Self {
2514        let count = if worker_count == 0 { 1 } else { worker_count };
2515        let cancel_streak_limit = DEFAULT_LAB_CANCEL_STREAK_LIMIT.max(1);
2516        Self {
2517            workers: (0..count)
2518                .map(|_| crate::runtime::scheduler::PriorityScheduler::new())
2519                .collect(),
2520            scheduled: DetHashSet::default(),
2521            pending_spurious_wakes: DetHashMap::default(),
2522            assignments: Vec::new(),
2523            next_worker: 0,
2524            cancel_streak: vec![0; count],
2525            cancel_streak_limit,
2526        }
2527    }
2528
2529    /// Returns true if no tasks are currently scheduled.
2530    #[must_use]
2531    pub fn is_empty(&self) -> bool {
2532        self.scheduled.is_empty()
2533    }
2534
2535    /// Returns the configured cancel streak limit for lab scheduling.
2536    #[must_use]
2537    pub fn cancel_streak_limit(&self) -> usize {
2538        self.cancel_streak_limit
2539    }
2540
2541    #[inline]
2542    fn set_assignment(&mut self, task: TaskId, worker: usize) {
2543        let slot = task.arena_index().index() as usize;
2544        if slot >= self.assignments.len() {
2545            self.assignments.resize(slot + 1, None);
2546        }
2547        self.assignments[slot] = Some(worker);
2548    }
2549
2550    fn assign_worker(&mut self, task: TaskId) -> usize {
2551        let slot = task.arena_index().index() as usize;
2552        if slot < self.assignments.len() {
2553            if let Some(worker) = self.assignments[slot] {
2554                return worker;
2555            }
2556        }
2557        let worker = self.next_worker % self.workers.len();
2558        self.next_worker = self.next_worker.wrapping_add(1);
2559        if slot >= self.assignments.len() {
2560            self.assignments.resize(slot + 1, None);
2561        }
2562        self.assignments[slot] = Some(worker);
2563        worker
2564    }
2565
2566    /// Schedules a task in the ready lane on its assigned worker.
2567    pub fn schedule(&mut self, task: TaskId, priority: u8) {
2568        if !self.scheduled.insert(task) {
2569            crate::tracing_compat::trace!("LabScheduler already scheduled {task:?}");
2570            return;
2571        }
2572        crate::tracing_compat::trace!("LabScheduler scheduling {task:?}");
2573
2574        let worker = self.assign_worker(task);
2575        self.workers[worker].schedule(task, priority);
2576    }
2577
2578    /// Injects ready-lane wakeups that should survive normal deduplication.
2579    ///
2580    /// The first wake is scheduled immediately when needed. Remaining wakeups
2581    /// are re-armed one-by-one after each dequeue so wake storms trigger
2582    /// repeated polls instead of collapsing to a single queued wake.
2583    fn inject_spurious_wakes(&mut self, task: TaskId, priority: u8, count: usize) {
2584        if count == 0 {
2585            return;
2586        }
2587
2588        let mut remaining = count;
2589        if self.scheduled.insert(task) {
2590            let worker = self.assign_worker(task);
2591            self.workers[worker].schedule(task, priority);
2592            remaining = remaining.saturating_sub(1);
2593        }
2594
2595        if remaining == 0 {
2596            return;
2597        }
2598
2599        self.pending_spurious_wakes
2600            .entry(task)
2601            .and_modify(|pending| {
2602                pending.priority = pending.priority.max(priority);
2603                pending.remaining = pending.remaining.saturating_add(remaining);
2604            })
2605            .or_insert(PendingSpuriousWake {
2606                priority,
2607                remaining,
2608            });
2609    }
2610
2611    /// Schedules or promotes a task into the cancel lane.
2612    pub fn schedule_cancel(&mut self, task: TaskId, priority: u8) {
2613        if self.scheduled.insert(task) {
2614            let worker = self.assign_worker(task);
2615            self.workers[worker].schedule_cancel(task, priority);
2616            return;
2617        }
2618
2619        let slot = task.arena_index().index() as usize;
2620        if let Some(&Some(worker)) = self.assignments.get(slot) {
2621            self.workers[worker].move_to_cancel_lane(task, priority);
2622        }
2623    }
2624
2625    /// Schedules a task in the timed lane on its assigned worker.
2626    pub fn schedule_timed(&mut self, task: TaskId, deadline: Time) {
2627        if !self.scheduled.insert(task) {
2628            return;
2629        }
2630
2631        let worker = self.assign_worker(task);
2632        self.workers[worker].schedule_timed(task, deadline);
2633    }
2634
2635    fn pop_for_worker(
2636        &mut self,
2637        worker: usize,
2638        rng_hint: u64,
2639        now: Time,
2640    ) -> Option<(TaskId, DispatchLane)> {
2641        if self.workers.is_empty() {
2642            return None;
2643        }
2644
2645        let worker = worker % self.workers.len();
2646        let cancel_streak = &mut self.cancel_streak[worker];
2647
2648        if *cancel_streak < self.cancel_streak_limit {
2649            if let Some((task, lane)) = self.workers[worker].pop_cancel_with_rng(rng_hint) {
2650                *cancel_streak += 1;
2651                self.scheduled.remove(&task);
2652                self.set_assignment(task, worker);
2653                self.rearm_spurious_wake(task);
2654                return Some((task, lane));
2655            }
2656        }
2657
2658        if let Some(task) = self.workers[worker].pop_timed_only_with_hint(rng_hint, now) {
2659            *cancel_streak = 0;
2660            self.scheduled.remove(&task);
2661            self.set_assignment(task, worker);
2662            self.rearm_spurious_wake(task);
2663            return Some((task, DispatchLane::Timed));
2664        }
2665
2666        if let Some(task) = self.workers[worker].pop_ready_only_with_hint(rng_hint) {
2667            *cancel_streak = 0;
2668            self.scheduled.remove(&task);
2669            self.set_assignment(task, worker);
2670            self.rearm_spurious_wake(task);
2671            return Some((task, DispatchLane::Ready));
2672        }
2673
2674        if let Some((task, lane)) = self.workers[worker].pop_cancel_with_rng(rng_hint) {
2675            *cancel_streak = 1;
2676            self.scheduled.remove(&task);
2677            self.set_assignment(task, worker);
2678            self.rearm_spurious_wake(task);
2679            return Some((task, lane));
2680        }
2681
2682        *cancel_streak = 0;
2683        None
2684    }
2685
2686    fn steal_for_worker(&mut self, thief: usize, rng_hint: u64) -> Option<TaskId> {
2687        let count = self.workers.len();
2688        if count <= 1 {
2689            return None;
2690        }
2691
2692        let thief = thief % count;
2693        let start = (rng_hint as usize) % count;
2694
2695        for offset in 0..count {
2696            let victim = (start + offset) % count;
2697            if victim == thief {
2698                continue;
2699            }
2700            if let Some(task) =
2701                self.workers[victim].pop_ready_only_with_hint(rng_hint.wrapping_add(offset as u64))
2702            {
2703                self.scheduled.remove(&task);
2704                self.set_assignment(task, thief);
2705                self.rearm_spurious_wake(task);
2706                return Some(task);
2707            }
2708        }
2709
2710        None
2711    }
2712
2713    fn forget_task(&mut self, task: TaskId) {
2714        self.scheduled.remove(&task);
2715        self.pending_spurious_wakes.remove(&task);
2716        let slot = task.arena_index().index() as usize;
2717        if slot < self.assignments.len() {
2718            self.assignments[slot] = None;
2719        }
2720        for worker in &mut self.workers {
2721            worker.remove(task);
2722        }
2723    }
2724
2725    fn rearm_spurious_wake(&mut self, task: TaskId) {
2726        let Some(mut pending) = self.pending_spurious_wakes.remove(&task) else {
2727            return;
2728        };
2729
2730        self.schedule(task, pending.priority);
2731        pending.remaining = pending.remaining.saturating_sub(1);
2732        if pending.remaining > 0 {
2733            self.pending_spurious_wakes.insert(task, pending);
2734        }
2735    }
2736}
2737
2738struct TaskWaker {
2739    task_id: crate::types::TaskId,
2740    priority: u8,
2741    scheduler: Arc<Mutex<LabScheduler>>,
2742}
2743
2744use std::task::Wake;
2745impl Wake for TaskWaker {
2746    fn wake(self: Arc<Self>) {
2747        self.scheduler.lock().schedule(self.task_id, self.priority);
2748    }
2749}
2750
2751/// Waker that reschedules a task into the cancel lane.
2752///
2753/// Set as `cancel_waker` on each task's `CxInner` before polling so that
2754/// `abort_with_reason` can wake cancelled tasks.
2755struct CancelTaskWaker {
2756    task_id: crate::types::TaskId,
2757    priority: u8,
2758    scheduler: Arc<Mutex<LabScheduler>>,
2759}
2760
2761impl Wake for CancelTaskWaker {
2762    fn wake(self: Arc<Self>) {
2763        self.scheduler
2764            .lock()
2765            .schedule_cancel(self.task_id, self.priority);
2766    }
2767}
2768
2769/// An invariant violation detected by the lab runtime.
2770#[derive(Debug, Clone, PartialEq, Eq)]
2771pub enum InvariantViolation {
2772    /// Obligations were not resolved.
2773    ObligationLeak {
2774        /// Leaked obligations and diagnostic metadata.
2775        leaks: Vec<ObligationLeak>,
2776    },
2777    /// Tasks were not drained.
2778    TaskLeak {
2779        /// Number of leaked tasks.
2780        count: usize,
2781    },
2782    /// Actors were not stopped before region close.
2783    ActorLeak {
2784        /// Number of leaked actors.
2785        count: usize,
2786    },
2787    /// A region closed with live children.
2788    QuiescenceViolation,
2789    /// A task held obligations but stopped being polled (futurelock).
2790    Futurelock {
2791        /// The task that futurelocked.
2792        task: crate::types::TaskId,
2793        /// The owning region.
2794        region: crate::types::RegionId,
2795        /// How many lab steps since last poll.
2796        idle_steps: u64,
2797        /// Held obligations.
2798        held: Vec<ObligationId>,
2799    },
2800    /// Cancellation protocol violation detected.
2801    CancellationProtocol {
2802        /// The violation description.
2803        violation: String,
2804    },
2805    /// br-asupersync-ipejce: a fuzz / scenario test closure panicked.
2806    /// Recorded so the campaign can keep searching instead of
2807    /// aborting on the first finding (the most interesting outcome
2808    /// of any fuzz campaign).
2809    TestPanic {
2810        /// Stringified panic payload (extracted via `Any::downcast`
2811        /// of `&str` and `String` — falls back to `<unknown panic>`).
2812        message: String,
2813    },
2814}
2815
2816/// Diagnostic details for a leaked obligation.
2817#[derive(Debug, Clone, PartialEq, Eq)]
2818pub struct ObligationLeak {
2819    /// The leaked obligation id.
2820    pub obligation: ObligationId,
2821    /// Kind of obligation (permit/ack/lease/io).
2822    pub kind: ObligationKind,
2823    /// Task that held the obligation.
2824    pub holder: crate::types::TaskId,
2825    /// Region that owned the obligation.
2826    pub region: crate::types::RegionId,
2827}
2828
2829impl std::fmt::Display for ObligationLeak {
2830    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2831        write!(
2832            f,
2833            "{:?} {:?} holder={:?} region={:?}",
2834            self.obligation, self.kind, self.holder, self.region
2835        )
2836    }
2837}
2838
2839impl std::fmt::Display for InvariantViolation {
2840    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2841        match self {
2842            Self::ObligationLeak { leaks } => {
2843                write!(f, "{} obligations leaked", leaks.len())
2844            }
2845            Self::TaskLeak { count } => write!(f, "{count} tasks leaked"),
2846            Self::ActorLeak { count } => write!(f, "{count} actors leaked"),
2847            Self::QuiescenceViolation => write!(f, "region closed without quiescence"),
2848            Self::Futurelock {
2849                task,
2850                region,
2851                idle_steps,
2852                held,
2853            } => write!(
2854                f,
2855                "futurelock: {task} in {region} idle={idle_steps} held={held:?}"
2856            ),
2857            Self::CancellationProtocol { violation } => {
2858                write!(f, "cancellation protocol violation: {violation}")
2859            }
2860            Self::TestPanic { message } => write!(f, "test panic: {message}"),
2861        }
2862    }
2863}
2864
2865/// Convenience function for running a test with the lab runtime.
2866pub fn test<F, R>(seed: u64, f: F) -> R
2867where
2868    F: FnOnce(&mut LabRuntime) -> R,
2869{
2870    let mut runtime = LabRuntime::with_seed(seed);
2871    let result = f(&mut runtime);
2872
2873    // Check invariants
2874    let violations = runtime.check_invariants();
2875    assert!(
2876        violations.is_empty(),
2877        "Lab runtime invariant violations: {violations:?}"
2878    );
2879
2880    result
2881}
2882
2883#[cfg(test)]
2884mod tests {
2885    #![allow(
2886        clippy::pedantic,
2887        clippy::nursery,
2888        clippy::expect_fun_call,
2889        clippy::map_unwrap_or,
2890        clippy::cast_possible_wrap,
2891        clippy::future_not_send
2892    )]
2893    use super::*;
2894    use crate::lab::chaos::ChaosConfig;
2895    use crate::record::TaskRecord;
2896    use crate::record::{ObligationAbortReason, ObligationKind};
2897    use crate::runtime::deadline_monitor::{AdaptiveDeadlineConfig, WarningReason};
2898    #[cfg(unix)]
2899    use crate::runtime::reactor::{Event, Interest};
2900    use crate::types::{Budget, CxInner, Outcome, TaskId};
2901    use crate::util::ArenaIndex;
2902    use parking_lot::Mutex;
2903    use parking_lot::RwLock;
2904    use std::sync::Arc;
2905    use std::task::Waker;
2906    use std::time::Duration;
2907
2908    #[cfg(unix)]
2909    struct TestFdSource;
2910    #[cfg(unix)]
2911    impl std::os::fd::AsRawFd for TestFdSource {
2912        fn as_raw_fd(&self) -> std::os::fd::RawFd {
2913            0
2914        }
2915    }
2916
2917    #[cfg(unix)]
2918    fn noop_waker() -> Waker {
2919        std::task::Waker::noop().clone()
2920    }
2921
2922    /// Waker that sets an `AtomicBool` when woken (for virtual time tests).
2923    struct FlagWaker(Arc<std::sync::atomic::AtomicBool>);
2924    impl Wake for FlagWaker {
2925        fn wake(self: Arc<Self>) {
2926            self.0.store(true, std::sync::atomic::Ordering::SeqCst);
2927        }
2928    }
2929
2930    /// Waker that increments an `AtomicU64` counter when woken.
2931    struct CountWaker(Arc<std::sync::atomic::AtomicU64>);
2932    impl Wake for CountWaker {
2933        fn wake(self: Arc<Self>) {
2934            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2935        }
2936    }
2937
2938    fn init_test(name: &str) {
2939        crate::test_utils::init_test_logging();
2940        crate::test_phase!(name);
2941    }
2942
2943    struct TimerAdvanceOutcome {
2944        advance_points: Vec<Time>,
2945        total_wakeups: u64,
2946        final_time: Time,
2947        cancelled_wakeups: u64,
2948    }
2949
2950    fn collect_timer_advances(
2951        deadlines_secs: &[u64],
2952        cancelled_indices: &[usize],
2953    ) -> TimerAdvanceOutcome {
2954        let mut runtime = LabRuntime::with_seed(42);
2955        let timer_handle = runtime.state.timer_driver_handle().expect("timer handle");
2956        let live_wakeups = Arc::new(std::sync::atomic::AtomicU64::new(0));
2957        let cancelled_wakeups = Arc::new(std::sync::atomic::AtomicU64::new(0));
2958        let mut handles = Vec::with_capacity(deadlines_secs.len());
2959
2960        for (idx, secs) in deadlines_secs.iter().copied().enumerate() {
2961            let counter = if cancelled_indices.contains(&idx) {
2962                cancelled_wakeups.clone()
2963            } else {
2964                live_wakeups.clone()
2965            };
2966            let waker = Waker::from(Arc::new(CountWaker(counter)));
2967            handles.push(timer_handle.register(Time::from_secs(secs), waker));
2968        }
2969
2970        for &idx in cancelled_indices {
2971            let cancelled = timer_handle.cancel(&handles[idx]);
2972            crate::assert_with_log!(
2973                cancelled,
2974                "cancelled timer handle remains removable before auto-advance",
2975                true,
2976                cancelled
2977            );
2978        }
2979
2980        let mut advance_points = Vec::new();
2981        while runtime.pending_timer_count() > 0 {
2982            let before = runtime.now();
2983            let next_deadline = runtime.next_timer_deadline().expect("pending deadline");
2984            let wakeups = runtime.advance_to_next_timer();
2985            let after = runtime.now();
2986
2987            crate::assert_with_log!(
2988                after >= before,
2989                "virtual time stays monotone while advancing timers",
2990                true,
2991                after >= before
2992            );
2993            crate::assert_with_log!(
2994                after >= next_deadline,
2995                "advance reaches or passes scheduled deadline",
2996                true,
2997                after >= next_deadline
2998            );
2999            crate::assert_with_log!(
3000                wakeups > 0,
3001                "each advance drains at least one live timer",
3002                true,
3003                wakeups > 0
3004            );
3005
3006            advance_points.push(after);
3007        }
3008
3009        TimerAdvanceOutcome {
3010            advance_points,
3011            total_wakeups: live_wakeups.load(std::sync::atomic::Ordering::SeqCst),
3012            final_time: runtime.now(),
3013            cancelled_wakeups: cancelled_wakeups.load(std::sync::atomic::Ordering::SeqCst),
3014        }
3015    }
3016
3017    #[test]
3018    fn empty_runtime_is_quiescent() {
3019        init_test("empty_runtime_is_quiescent");
3020        let runtime = LabRuntime::with_seed(42);
3021        let quiescent = runtime.is_quiescent();
3022        crate::assert_with_log!(quiescent, "quiescent", true, quiescent);
3023        crate::test_complete!("empty_runtime_is_quiescent");
3024    }
3025
3026    #[test]
3027    fn advance_time() {
3028        init_test("advance_time");
3029        let mut runtime = LabRuntime::with_seed(42);
3030        let now = runtime.now();
3031        crate::assert_with_log!(now == Time::ZERO, "now", Time::ZERO, now);
3032
3033        runtime.advance_time(1_000_000);
3034        let now = runtime.now();
3035        crate::assert_with_log!(
3036            now == Time::from_millis(1),
3037            "now",
3038            Time::from_millis(1),
3039            now
3040        );
3041        crate::test_complete!("advance_time");
3042    }
3043
3044    #[test]
3045    fn duration_nanos_saturating_clamps_large_duration() {
3046        init_test("duration_nanos_saturating_clamps_large_duration");
3047        let huge = Duration::from_secs(u64::MAX);
3048        let saturated = LabRuntime::duration_nanos_saturating(huge);
3049        crate::assert_with_log!(
3050            saturated == u64::MAX,
3051            "huge duration saturates",
3052            u64::MAX,
3053            saturated
3054        );
3055
3056        let small = Duration::from_nanos(123);
3057        let exact = LabRuntime::duration_nanos_saturating(small);
3058        crate::assert_with_log!(exact == 123, "small duration exact", 123u64, exact);
3059        crate::test_complete!("duration_nanos_saturating_clamps_large_duration");
3060    }
3061
3062    #[cfg(unix)]
3063    #[test]
3064    fn lab_runtime_records_io_ready_trace() {
3065        init_test("lab_runtime_records_io_ready_trace");
3066
3067        let mut runtime = LabRuntime::with_seed(42);
3068        let handle = runtime.state.io_driver_handle().expect("io driver");
3069        let waker = noop_waker();
3070        let source = TestFdSource;
3071
3072        let registration = handle
3073            .register(&source, Interest::READABLE, waker)
3074            .expect("register source");
3075        let token = registration.token();
3076
3077        runtime
3078            .lab_reactor()
3079            .inject_event(token, Event::readable(token), Duration::from_millis(1));
3080        runtime.advance_time(1_000_000);
3081        runtime.step_for_test();
3082
3083        let mut saw_requested = false;
3084        let mut saw_ready = false;
3085        for event in runtime.state.trace.snapshot() {
3086            if event.kind == TraceEventKind::IoRequested {
3087                saw_requested = true;
3088            }
3089            if event.kind == TraceEventKind::IoReady {
3090                saw_ready = true;
3091            }
3092        }
3093        crate::assert_with_log!(
3094            saw_requested,
3095            "io requested trace recorded",
3096            true,
3097            saw_requested
3098        );
3099        crate::assert_with_log!(saw_ready, "io ready trace recorded", true, saw_ready);
3100        crate::test_complete!("lab_runtime_records_io_ready_trace");
3101    }
3102
3103    #[cfg(unix)]
3104    #[test]
3105    fn lab_runtime_chaos_stats_include_reactor_io_error_injections() {
3106        init_test("lab_runtime_chaos_stats_include_reactor_io_error_injections");
3107
3108        let config = LabConfig::new(7).with_chaos(
3109            ChaosConfig::new(7)
3110                .with_io_error_probability(1.0)
3111                .with_io_error_kinds(vec![std::io::ErrorKind::TimedOut]),
3112        );
3113        let mut runtime = LabRuntime::new(config);
3114        let handle = runtime.state.io_driver_handle().expect("io driver");
3115        let waker = noop_waker();
3116        let source = TestFdSource;
3117
3118        let registration = handle
3119            .register(&source, Interest::READABLE, waker)
3120            .expect("register source");
3121        let token = registration.token();
3122
3123        runtime
3124            .lab_reactor()
3125            .inject_event(token, Event::readable(token), Duration::ZERO);
3126        runtime.step_for_test();
3127
3128        let stats = runtime.chaos_stats();
3129        crate::assert_with_log!(
3130            stats.io_errors == 1,
3131            "io errors aggregated",
3132            1u64,
3133            stats.io_errors
3134        );
3135        crate::assert_with_log!(
3136            stats.decision_points == 1,
3137            "reactor decision points aggregated",
3138            1u64,
3139            stats.decision_points
3140        );
3141        crate::assert_with_log!(
3142            runtime.lab_reactor().last_io_error_kind() == Some(std::io::ErrorKind::TimedOut),
3143            "reactor last error kind surfaced",
3144            Some(std::io::ErrorKind::TimedOut),
3145            runtime.lab_reactor().last_io_error_kind()
3146        );
3147
3148        crate::test_complete!("lab_runtime_chaos_stats_include_reactor_io_error_injections");
3149    }
3150
3151    #[test]
3152    fn pending_task_without_wakeup_storm_still_counts_chaos_decision_point() {
3153        init_test("pending_task_without_wakeup_storm_still_counts_chaos_decision_point");
3154
3155        let config =
3156            LabConfig::new(99).with_chaos(ChaosConfig::new(99).with_wakeup_storm_probability(0.0));
3157        let mut runtime = LabRuntime::new(config);
3158        let region = runtime.state.create_root_region(Budget::INFINITE);
3159        let (task_id, _handle) = runtime
3160            .state
3161            .create_task(region, Budget::INFINITE, async {
3162                std::future::pending::<()>().await;
3163            })
3164            .expect("create task");
3165        runtime.scheduler.lock().schedule(task_id, 0);
3166
3167        runtime.step_for_test();
3168
3169        let stats = runtime.chaos_stats();
3170        crate::assert_with_log!(
3171            stats.decision_points == 2,
3172            "pending-task decision point counted",
3173            2u64,
3174            stats.decision_points
3175        );
3176        crate::assert_with_log!(
3177            stats.wakeup_storms == 0,
3178            "no wakeup storm recorded",
3179            0u64,
3180            stats.wakeup_storms
3181        );
3182
3183        crate::test_complete!(
3184            "pending_task_without_wakeup_storm_still_counts_chaos_decision_point"
3185        );
3186    }
3187
3188    #[test]
3189    fn pre_poll_multi_injection_counts_one_chaos_decision_point() {
3190        init_test("pre_poll_multi_injection_counts_one_chaos_decision_point");
3191
3192        let config = LabConfig::new(123).with_chaos(
3193            ChaosConfig::new(123)
3194                .with_cancel_probability(1.0)
3195                .with_delay_probability(1.0)
3196                .with_delay_range(Duration::ZERO..Duration::from_nanos(2))
3197                .with_budget_exhaust_probability(1.0),
3198        );
3199        let mut runtime = LabRuntime::new(config);
3200        let region = runtime.state.create_root_region(Budget::INFINITE);
3201        let (task_id, _handle) = runtime
3202            .state
3203            .create_task(region, Budget::INFINITE, async {
3204                std::future::pending::<()>().await;
3205            })
3206            .expect("create task");
3207        runtime.scheduler.lock().schedule(task_id, 0);
3208
3209        runtime.step_for_test();
3210
3211        let stats = runtime.chaos_stats();
3212        crate::assert_with_log!(
3213            stats.decision_points == 1,
3214            "multi-injection pre-poll counts once",
3215            1u64,
3216            stats.decision_points
3217        );
3218        crate::assert_with_log!(
3219            stats.cancellations == 1,
3220            "cancel recorded",
3221            1u64,
3222            stats.cancellations
3223        );
3224        crate::assert_with_log!(stats.delays == 1, "delay recorded", 1u64, stats.delays);
3225        crate::assert_with_log!(
3226            stats.budget_exhaustions == 1,
3227            "budget exhaust recorded",
3228            1u64,
3229            stats.budget_exhaustions
3230        );
3231        crate::assert_with_log!(
3232            stats.total_delay == Duration::from_nanos(1),
3233            "positive delay preserved",
3234            Duration::from_nanos(1),
3235            stats.total_delay
3236        );
3237
3238        crate::test_complete!("pre_poll_multi_injection_counts_one_chaos_decision_point");
3239    }
3240
3241    #[test]
3242    fn deterministic_rng() {
3243        init_test("deterministic_rng");
3244        let mut r1 = LabRuntime::with_seed(42);
3245        let mut r2 = LabRuntime::with_seed(42);
3246
3247        let a = r1.rng.next_u64();
3248        let b = r2.rng.next_u64();
3249        crate::assert_with_log!(a == b, "rng", b, a);
3250        crate::test_complete!("deterministic_rng");
3251    }
3252
3253    #[test]
3254    fn lab_scheduler_pop_for_worker_respects_timed_deadlines() {
3255        init_test("lab_scheduler_pop_for_worker_respects_timed_deadlines");
3256        let mut scheduler = LabScheduler::new(1);
3257        let timed = TaskId::from_arena(ArenaIndex::new(1, 0));
3258        let ready = TaskId::from_arena(ArenaIndex::new(2, 0));
3259
3260        scheduler.schedule_timed(timed, Time::from_nanos(100));
3261        scheduler.schedule(ready, 10);
3262
3263        let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3264        crate::assert_with_log!(
3265            first == Some((ready, DispatchLane::Ready)),
3266            "ready task dispatches before not-due timed task",
3267            Some((ready, DispatchLane::Ready)),
3268            first
3269        );
3270
3271        let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3272        crate::assert_with_log!(
3273            second.is_none(),
3274            "future timed task stays queued before deadline",
3275            true,
3276            second.is_none()
3277        );
3278
3279        let third = scheduler.pop_for_worker(0, 2, Time::from_nanos(100));
3280        crate::assert_with_log!(
3281            third == Some((timed, DispatchLane::Timed)),
3282            "timed task dispatches at deadline",
3283            Some((timed, DispatchLane::Timed)),
3284            third
3285        );
3286
3287        crate::test_complete!("lab_scheduler_pop_for_worker_respects_timed_deadlines");
3288    }
3289
3290    #[test]
3291    fn lab_scheduler_steal_for_worker_only_steals_ready_tasks() {
3292        init_test("lab_scheduler_steal_for_worker_only_steals_ready_tasks");
3293        let mut scheduler = LabScheduler::new(2);
3294        let cancel = TaskId::from_arena(ArenaIndex::new(10, 0));
3295        let timed = TaskId::from_arena(ArenaIndex::new(11, 0));
3296        let ready = TaskId::from_arena(ArenaIndex::new(12, 0));
3297
3298        // With 2 workers, assignment is round-robin: cancel->w0, timed->w1, ready->w0.
3299        scheduler.schedule_cancel(cancel, 100);
3300        scheduler.schedule_timed(timed, Time::ZERO);
3301        scheduler.schedule(ready, 50);
3302
3303        let stolen = scheduler.steal_for_worker(1, 0);
3304        crate::assert_with_log!(
3305            stolen == Some(ready),
3306            "steal path takes only ready lane work",
3307            Some(ready),
3308            stolen
3309        );
3310
3311        crate::assert_with_log!(
3312            scheduler.workers[0].has_cancel_work(),
3313            "victim cancel lane remains intact after steal",
3314            true,
3315            scheduler.workers[0].has_cancel_work()
3316        );
3317
3318        let cancel_dispatch = scheduler.pop_for_worker(0, 0, Time::ZERO);
3319        crate::assert_with_log!(
3320            cancel_dispatch == Some((cancel, DispatchLane::Cancel)),
3321            "cancel lane still dispatches from victim worker",
3322            Some((cancel, DispatchLane::Cancel)),
3323            cancel_dispatch
3324        );
3325
3326        let timed_dispatch = scheduler.pop_for_worker(1, 0, Time::ZERO);
3327        crate::assert_with_log!(
3328            timed_dispatch == Some((timed, DispatchLane::Timed)),
3329            "timed lane remains on owning worker",
3330            Some((timed, DispatchLane::Timed)),
3331            timed_dispatch
3332        );
3333
3334        crate::test_complete!("lab_scheduler_steal_for_worker_only_steals_ready_tasks");
3335    }
3336
3337    #[test]
3338    fn lab_scheduler_spurious_wakes_do_not_collapse_duplicates() {
3339        init_test("lab_scheduler_spurious_wakes_do_not_collapse_duplicates");
3340        let mut scheduler = LabScheduler::new(1);
3341        let task = TaskId::from_arena(ArenaIndex::new(13, 0));
3342
3343        scheduler.inject_spurious_wakes(task, 42, 3);
3344
3345        let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3346        crate::assert_with_log!(
3347            first == Some((task, DispatchLane::Ready)),
3348            "first spurious wake dispatches",
3349            Some((task, DispatchLane::Ready)),
3350            first
3351        );
3352
3353        let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3354        crate::assert_with_log!(
3355            second == Some((task, DispatchLane::Ready)),
3356            "second spurious wake remains queued",
3357            Some((task, DispatchLane::Ready)),
3358            second
3359        );
3360
3361        let third = scheduler.pop_for_worker(0, 2, Time::ZERO);
3362        crate::assert_with_log!(
3363            third == Some((task, DispatchLane::Ready)),
3364            "third spurious wake remains queued",
3365            Some((task, DispatchLane::Ready)),
3366            third
3367        );
3368
3369        let fourth = scheduler.pop_for_worker(0, 3, Time::ZERO);
3370        crate::assert_with_log!(
3371            fourth.is_none(),
3372            "storm drains after requested wake count",
3373            true,
3374            fourth.is_none()
3375        );
3376        crate::assert_with_log!(
3377            scheduler.is_empty(),
3378            "scheduler empty after spurious storm drains",
3379            true,
3380            scheduler.is_empty()
3381        );
3382
3383        crate::test_complete!("lab_scheduler_spurious_wakes_do_not_collapse_duplicates");
3384    }
3385
3386    #[test]
3387    fn lab_scheduler_forget_task_clears_pending_spurious_wakes() {
3388        init_test("lab_scheduler_forget_task_clears_pending_spurious_wakes");
3389        let mut scheduler = LabScheduler::new(1);
3390        let task = TaskId::from_arena(ArenaIndex::new(14, 0));
3391
3392        scheduler.inject_spurious_wakes(task, 42, 3);
3393        let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3394        crate::assert_with_log!(
3395            first == Some((task, DispatchLane::Ready)),
3396            "first spurious wake dispatches before forget",
3397            Some((task, DispatchLane::Ready)),
3398            first
3399        );
3400
3401        scheduler.forget_task(task);
3402
3403        let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3404        crate::assert_with_log!(
3405            second.is_none(),
3406            "forget_task drains queued spurious wakes",
3407            true,
3408            second.is_none()
3409        );
3410        crate::assert_with_log!(
3411            scheduler.pending_spurious_wakes.is_empty(),
3412            "forget_task clears pending spurious wake budget",
3413            true,
3414            scheduler.pending_spurious_wakes.is_empty()
3415        );
3416        crate::assert_with_log!(
3417            scheduler.is_empty(),
3418            "scheduler empty after forget_task",
3419            true,
3420            scheduler.is_empty()
3421        );
3422
3423        crate::test_complete!("lab_scheduler_forget_task_clears_pending_spurious_wakes");
3424    }
3425
3426    #[test]
3427    fn lab_scheduler_steal_preserves_pending_spurious_wakes() {
3428        init_test("lab_scheduler_steal_preserves_pending_spurious_wakes");
3429        let mut scheduler = LabScheduler::new(2);
3430        let task = TaskId::from_arena(ArenaIndex::new(15, 0));
3431
3432        scheduler.inject_spurious_wakes(task, 42, 3);
3433
3434        let stolen = scheduler.steal_for_worker(1, 0);
3435        crate::assert_with_log!(
3436            stolen == Some(task),
3437            "steal dispatches first storm wake",
3438            Some(task),
3439            stolen
3440        );
3441
3442        let second = scheduler.pop_for_worker(1, 1, Time::ZERO);
3443        crate::assert_with_log!(
3444            second == Some((task, DispatchLane::Ready)),
3445            "steal path re-arms second storm wake on thief worker",
3446            Some((task, DispatchLane::Ready)),
3447            second
3448        );
3449
3450        let third = scheduler.pop_for_worker(1, 2, Time::ZERO);
3451        crate::assert_with_log!(
3452            third == Some((task, DispatchLane::Ready)),
3453            "steal path preserves final pending storm wake",
3454            Some((task, DispatchLane::Ready)),
3455            third
3456        );
3457
3458        let fourth = scheduler.pop_for_worker(1, 3, Time::ZERO);
3459        crate::assert_with_log!(
3460            fourth.is_none(),
3461            "all stolen storm wakes drain after requested count",
3462            true,
3463            fourth.is_none()
3464        );
3465        crate::assert_with_log!(
3466            scheduler.is_empty(),
3467            "scheduler empty after stolen storm drains",
3468            true,
3469            scheduler.is_empty()
3470        );
3471
3472        crate::test_complete!("lab_scheduler_steal_preserves_pending_spurious_wakes");
3473    }
3474
3475    #[test]
3476    fn deterministic_multiworker_schedule() {
3477        init_test("deterministic_multiworker_schedule");
3478        let config = LabConfig::new(7).worker_count(4);
3479
3480        crate::lab::assert_deterministic(config, |runtime| {
3481            let root = runtime.state.create_root_region(Budget::INFINITE);
3482            for _ in 0..4 {
3483                let (task_id, _handle) = runtime
3484                    .state
3485                    .create_task(root, Budget::INFINITE, async {
3486                        crate::runtime::yield_now::yield_now().await;
3487                    })
3488                    .expect("create task");
3489                runtime.scheduler.lock().schedule(task_id, 0);
3490            }
3491            runtime.run_until_quiescent();
3492        });
3493
3494        crate::test_complete!("deterministic_multiworker_schedule");
3495    }
3496
3497    #[test]
3498    fn run_until_quiescent_with_report_is_deterministic() {
3499        init_test("run_until_quiescent_with_report_is_deterministic");
3500
3501        let config = LabConfig::new(123).worker_count(4).max_steps(10_000);
3502        let mut r1 = LabRuntime::new(config.clone());
3503        let mut r2 = LabRuntime::new(config);
3504
3505        let setup = |runtime: &mut LabRuntime| {
3506            let root = runtime.state.create_root_region(Budget::INFINITE);
3507            for _ in 0..4 {
3508                let (task_id, _handle) = runtime
3509                    .state
3510                    .create_task(root, Budget::INFINITE, async {
3511                        crate::runtime::yield_now::yield_now().await;
3512                    })
3513                    .expect("create task");
3514                runtime.scheduler.lock().schedule(task_id, 0);
3515            }
3516        };
3517
3518        setup(&mut r1);
3519        setup(&mut r2);
3520
3521        let rep1 = r1.run_until_quiescent_with_report();
3522        let rep2 = r2.run_until_quiescent_with_report();
3523
3524        crate::assert_with_log!(rep1.quiescent, "quiescent", true, rep1.quiescent);
3525        crate::assert_with_log!(rep2.quiescent, "quiescent", true, rep2.quiescent);
3526
3527        assert_eq!(rep1.trace_fingerprint, rep2.trace_fingerprint);
3528        assert_eq!(rep1.trace_certificate, rep2.trace_certificate);
3529        assert_eq!(rep1.oracle_report.to_json(), rep2.oracle_report.to_json());
3530        assert_eq!(rep1.invariant_violations, rep2.invariant_violations);
3531
3532        crate::assert_with_log!(
3533            rep1.oracle_report.all_passed(),
3534            "oracles passed",
3535            true,
3536            rep1.oracle_report.all_passed()
3537        );
3538        crate::assert_with_log!(
3539            rep2.oracle_report.all_passed(),
3540            "oracles passed",
3541            true,
3542            rep2.oracle_report.all_passed()
3543        );
3544
3545        crate::test_complete!("run_until_quiescent_with_report_is_deterministic");
3546    }
3547
3548    #[test]
3549    fn deadline_monitor_emits_warning() {
3550        init_test("deadline_monitor_emits_warning");
3551        let mut runtime = LabRuntime::with_seed(42);
3552
3553        let warnings: Arc<Mutex<Vec<DeadlineWarning>>> = Arc::new(Mutex::new(Vec::new()));
3554        let warnings_clone = Arc::clone(&warnings);
3555
3556        let config = MonitorConfig {
3557            check_interval: Duration::from_secs(0),
3558            warning_threshold_fraction: 1.0,
3559            checkpoint_timeout: Duration::from_secs(0),
3560            adaptive: AdaptiveDeadlineConfig::default(),
3561            enabled: true,
3562        };
3563
3564        runtime.enable_deadline_monitoring_with_handler(config, move |warning| {
3565            warnings_clone.lock().push(warning);
3566        });
3567
3568        let root = runtime.state.create_root_region(Budget::INFINITE);
3569        let budget = Budget::new().with_deadline(Time::from_millis(10));
3570
3571        let task_idx = runtime.state.insert_task(TaskRecord::new_with_time(
3572            TaskId::from_arena(ArenaIndex::new(0, 0)),
3573            root,
3574            budget,
3575            runtime.state.now,
3576        ));
3577        let task_id = TaskId::from_arena(task_idx);
3578        runtime.state.task_mut(task_id).unwrap().id = task_id;
3579
3580        let mut inner = CxInner::new(root, task_id, budget);
3581        inner.checkpoint_state.last_checkpoint = None;
3582        runtime
3583            .state
3584            .task_mut(task_id)
3585            .unwrap()
3586            .set_cx_inner(Arc::new(RwLock::new(inner)));
3587
3588        runtime.step();
3589
3590        let warnings = warnings.lock();
3591        let warning = warnings.first().expect("expected warning");
3592        crate::assert_with_log!(
3593            warning.task_id == task_id,
3594            "task_id",
3595            task_id,
3596            warning.task_id
3597        );
3598        crate::assert_with_log!(
3599            warning.region_id == root,
3600            "region_id",
3601            root,
3602            warning.region_id
3603        );
3604        let ok = matches!(
3605            warning.reason,
3606            WarningReason::ApproachingDeadline | WarningReason::ApproachingDeadlineNoProgress
3607        );
3608        crate::assert_with_log!(ok, "reason", true, ok);
3609        drop(warnings);
3610        crate::test_complete!("deadline_monitor_emits_warning");
3611    }
3612
3613    #[test]
3614    fn deadline_monitor_e2e_stuck_detection() {
3615        init_test("deadline_monitor_e2e_stuck_detection");
3616        let mut runtime = LabRuntime::with_seed(42);
3617
3618        let warnings: Arc<Mutex<Vec<DeadlineWarning>>> = Arc::new(Mutex::new(Vec::new()));
3619        let warnings_clone = Arc::clone(&warnings);
3620
3621        let config = MonitorConfig {
3622            check_interval: Duration::ZERO,
3623            warning_threshold_fraction: 0.0,
3624            checkpoint_timeout: Duration::ZERO,
3625            adaptive: AdaptiveDeadlineConfig::default(),
3626            enabled: true,
3627        };
3628
3629        runtime.enable_deadline_monitoring_with_handler(config, move |warning| {
3630            warnings_clone.lock().push(warning);
3631        });
3632
3633        let root = runtime.state.create_root_region(Budget::INFINITE);
3634        let budget = Budget::new().with_deadline(Time::from_secs(10));
3635        let (task_id, _handle) = runtime
3636            .state
3637            .create_task(root, budget, async {})
3638            .expect("create task");
3639
3640        {
3641            let task = runtime.state.task_mut(task_id).unwrap();
3642            let cx = task.cx.as_ref().expect("task cx");
3643            cx.checkpoint_with("starting work").expect("checkpoint");
3644        }
3645
3646        runtime.step();
3647
3648        let warnings = warnings.lock();
3649        let warning = warnings.first().expect("expected warning");
3650        crate::assert_with_log!(
3651            warning.task_id == task_id,
3652            "task_id",
3653            task_id,
3654            warning.task_id
3655        );
3656        crate::assert_with_log!(
3657            warning.reason == WarningReason::NoProgress,
3658            "reason",
3659            WarningReason::NoProgress,
3660            warning.reason
3661        );
3662        crate::assert_with_log!(
3663            warning.last_checkpoint_message.as_deref() == Some("starting work"),
3664            "checkpoint message",
3665            Some("starting work"),
3666            warning.last_checkpoint_message.as_deref()
3667        );
3668        drop(warnings);
3669        crate::test_complete!("deadline_monitor_e2e_stuck_detection");
3670    }
3671
3672    #[test]
3673    fn futurelock_emits_trace_event() {
3674        init_test("futurelock_emits_trace_event");
3675        let config = LabConfig::new(42)
3676            .futurelock_max_idle_steps(3)
3677            .panic_on_futurelock(false);
3678        let mut runtime = LabRuntime::new(config);
3679
3680        let root = runtime.state.create_root_region(Budget::INFINITE);
3681
3682        // Create a task.
3683        let task_idx = runtime.state.insert_task(TaskRecord::new(
3684            TaskId::from_arena(ArenaIndex::new(0, 0)),
3685            root,
3686            Budget::INFINITE,
3687        ));
3688        let task_id = TaskId::from_arena(task_idx);
3689        runtime.state.task_mut(task_id).unwrap().id = task_id;
3690
3691        // Create a pending obligation held by that task.
3692        let obl_id = runtime
3693            .state
3694            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3695            .expect("create obligation");
3696
3697        for _ in 0..4 {
3698            runtime.step();
3699        }
3700
3701        let futurelock = runtime
3702            .trace()
3703            .snapshot()
3704            .into_iter()
3705            .find(|e| e.kind == TraceEventKind::FuturelockDetected)
3706            .expect("expected futurelock trace event");
3707
3708        match &futurelock.data {
3709            TraceData::Futurelock {
3710                task,
3711                region,
3712                idle_steps,
3713                held,
3714            } => {
3715                crate::assert_with_log!(*task == task_id, "task", task_id, *task);
3716                crate::assert_with_log!(*region == root, "region", root, *region);
3717                let idle_ok = *idle_steps > 3;
3718                crate::assert_with_log!(idle_ok, "idle_steps > 3", true, idle_ok);
3719                let ok = held.as_slice() == [(obl_id, ObligationKind::SendPermit)];
3720                crate::assert_with_log!(
3721                    ok,
3722                    "held",
3723                    &[(obl_id, ObligationKind::SendPermit)],
3724                    held.as_slice()
3725                );
3726            }
3727            other => panic!("unexpected trace data: {other:?}"),
3728        }
3729        crate::test_complete!("futurelock_emits_trace_event");
3730    }
3731
3732    #[test]
3733    #[should_panic(expected = "futurelock detected")]
3734    fn futurelock_can_panic() {
3735        init_test("futurelock_can_panic");
3736        let config = LabConfig::new(42).futurelock_max_idle_steps(1);
3737        let mut runtime = LabRuntime::new(config);
3738
3739        let root = runtime.state.create_root_region(Budget::INFINITE);
3740
3741        let task_idx = runtime.state.insert_task(TaskRecord::new(
3742            TaskId::from_arena(ArenaIndex::new(0, 0)),
3743            root,
3744            Budget::INFINITE,
3745        ));
3746        let task_id = TaskId::from_arena(task_idx);
3747        runtime.state.task_mut(task_id).unwrap().id = task_id;
3748
3749        let _ = runtime
3750            .state
3751            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3752            .expect("create obligation");
3753
3754        // Run enough steps to exceed threshold and trigger panic.
3755        for _ in 0..3 {
3756            runtime.step();
3757        }
3758    }
3759
3760    /// Regression test: actively polled tasks must NOT be flagged as futurelocked.
3761    ///
3762    /// Before the fix, `mark_polled()` was never called from `step()`, so
3763    /// `last_polled_step` stayed at 0. After threshold+1 steps, even a
3764    /// task polled every single step would be falsely flagged.
3765    #[test]
3766    fn polled_task_not_flagged_as_futurelocked() {
3767        init_test("polled_task_not_flagged_as_futurelocked");
3768        let config = LabConfig::new(42)
3769            .futurelock_max_idle_steps(5)
3770            .panic_on_futurelock(false);
3771        let mut runtime = LabRuntime::new(config);
3772
3773        let root = runtime.state.create_root_region(Budget::INFINITE);
3774
3775        // Create a task with a stored future that always yields (Pending).
3776        let (task_id, _handle) = runtime
3777            .state
3778            .create_task(root, Budget::INFINITE, async {
3779                loop {
3780                    crate::runtime::yield_now::yield_now().await;
3781                }
3782            })
3783            .expect("create task");
3784
3785        // Give the task a pending obligation so it's eligible for futurelock.
3786        let _obl = runtime
3787            .state
3788            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3789            .expect("create obligation");
3790
3791        // Schedule and run well past the threshold.
3792        runtime.scheduler.lock().schedule(task_id, 0);
3793        for _ in 0..20 {
3794            runtime.step();
3795        }
3796
3797        // The task was polled every step, so no futurelock should fire.
3798        let violations = runtime.futurelock_violations();
3799        crate::assert_with_log!(
3800            violations.is_empty(),
3801            "no futurelock for actively polled task",
3802            true,
3803            violations.is_empty()
3804        );
3805        crate::test_complete!("polled_task_not_flagged_as_futurelocked");
3806    }
3807
3808    #[test]
3809    fn immediate_completion_marks_running_before_completion() {
3810        init_test("immediate_completion_marks_running_before_completion");
3811        let mut runtime = LabRuntime::new(LabConfig::new(42));
3812        let root = runtime.state.create_root_region(Budget::INFINITE);
3813        let (task_id, _) = runtime
3814            .state
3815            .create_task(root, Budget::INFINITE, async {})
3816            .expect("create task");
3817        runtime.scheduler.lock().schedule(task_id, 0);
3818
3819        runtime.run_until_quiescent();
3820
3821        let protocol_ok = runtime.check_cancellation_protocol().is_ok();
3822        crate::assert_with_log!(
3823            runtime.is_quiescent(),
3824            "runtime reached quiescence after immediate completion",
3825            true,
3826            runtime.is_quiescent()
3827        );
3828        crate::assert_with_log!(
3829            protocol_ok,
3830            "cancellation oracle accepted Created -> Running -> Completed",
3831            true,
3832            protocol_ok
3833        );
3834
3835        crate::test_complete!("immediate_completion_marks_running_before_completion");
3836    }
3837
3838    #[test]
3839    fn obligation_leak_detected_when_holder_completed() {
3840        init_test("obligation_leak_detected_when_holder_completed");
3841        let mut runtime = LabRuntime::with_seed(7);
3842        let root = runtime.state.create_root_region(Budget::INFINITE);
3843
3844        let task_idx = runtime.state.insert_task(TaskRecord::new(
3845            TaskId::from_arena(ArenaIndex::new(0, 0)),
3846            root,
3847            Budget::INFINITE,
3848        ));
3849        let task_id = TaskId::from_arena(task_idx);
3850        runtime.state.task_mut(task_id).unwrap().id = task_id;
3851
3852        let obl_id = runtime
3853            .state
3854            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3855            .expect("create obligation");
3856
3857        runtime
3858            .state
3859            .update_task(task_id, |record| record.complete(Outcome::Ok(())))
3860            .unwrap();
3861
3862        let violations = runtime.check_invariants();
3863        let mut found = false;
3864        for violation in violations {
3865            if let InvariantViolation::ObligationLeak { leaks } = violation {
3866                found = true;
3867                let len = leaks.len();
3868                crate::assert_with_log!(len == 1, "leaks len", 1, len);
3869                let leak = &leaks[0];
3870                crate::assert_with_log!(
3871                    leak.obligation == obl_id,
3872                    "obligation",
3873                    obl_id,
3874                    leak.obligation
3875                );
3876                crate::assert_with_log!(
3877                    leak.kind == ObligationKind::SendPermit,
3878                    "kind",
3879                    ObligationKind::SendPermit,
3880                    leak.kind
3881                );
3882                crate::assert_with_log!(leak.holder == task_id, "holder", task_id, leak.holder);
3883                crate::assert_with_log!(leak.region == root, "region", root, leak.region);
3884            }
3885        }
3886        crate::assert_with_log!(found, "found leak", true, found);
3887        crate::test_complete!("obligation_leak_detected_when_holder_completed");
3888    }
3889
3890    #[test]
3891    fn obligation_leak_ignored_when_resolved() {
3892        init_test("obligation_leak_ignored_when_resolved");
3893        let mut runtime = LabRuntime::with_seed(11);
3894        let root = runtime.state.create_root_region(Budget::INFINITE);
3895
3896        let task_idx = runtime.state.insert_task(TaskRecord::new(
3897            TaskId::from_arena(ArenaIndex::new(0, 0)),
3898            root,
3899            Budget::INFINITE,
3900        ));
3901        let task_id = TaskId::from_arena(task_idx);
3902        runtime.state.task_mut(task_id).unwrap().id = task_id;
3903
3904        let obl_id = runtime
3905            .state
3906            .create_obligation(ObligationKind::Ack, task_id, root, None)
3907            .expect("create obligation");
3908        runtime
3909            .state
3910            .commit_obligation(obl_id)
3911            .expect("commit obligation");
3912
3913        runtime
3914            .state
3915            .update_task(task_id, |record| record.complete(Outcome::Ok(())))
3916            .unwrap();
3917
3918        let violations = runtime.check_invariants();
3919        let has_leak = violations
3920            .iter()
3921            .any(|v| matches!(v, InvariantViolation::ObligationLeak { .. }));
3922        crate::assert_with_log!(!has_leak, "no leak", false, has_leak);
3923        crate::test_complete!("obligation_leak_ignored_when_resolved");
3924    }
3925
3926    #[test]
3927    fn report_hydrates_temporal_oracles_from_state_snapshot() {
3928        init_test("report_hydrates_temporal_oracles_from_state_snapshot");
3929        let mut runtime = LabRuntime::with_seed(31);
3930        let root = runtime.state.create_root_region(Budget::INFINITE);
3931        let (_task, _handle) = runtime
3932            .state
3933            .create_task(root, Budget::INFINITE, async {})
3934            .expect("create task");
3935
3936        // Force-close the region while a task is still live to simulate a
3937        // temporal invariant break that must be surfaced by report hydration.
3938        runtime
3939            .state
3940            .region(root)
3941            .expect("region exists")
3942            .set_state(crate::record::region::RegionState::Closed);
3943
3944        let report = runtime.report();
3945        let task_leak = report
3946            .oracle_report
3947            .entry("task_leak")
3948            .expect("task_leak entry");
3949        let quiescence = report
3950            .oracle_report
3951            .entry("quiescence")
3952            .expect("quiescence entry");
3953
3954        crate::assert_with_log!(
3955            !task_leak.passed,
3956            "task_leak failed",
3957            false,
3958            task_leak.passed
3959        );
3960        crate::assert_with_log!(
3961            !quiescence.passed,
3962            "quiescence failed",
3963            false,
3964            quiescence.passed
3965        );
3966        let has_temporal_tag = report
3967            .invariant_violations
3968            .iter()
3969            .any(|v| v == "temporal:task_leak");
3970        crate::assert_with_log!(
3971            has_temporal_tag,
3972            "temporal marker present",
3973            true,
3974            has_temporal_tag
3975        );
3976        let temporal_failed = report
3977            .temporal_invariant_failures
3978            .iter()
3979            .any(|v| v == "task_leak");
3980        crate::assert_with_log!(
3981            temporal_failed,
3982            "temporal failure surfaced",
3983            true,
3984            temporal_failed
3985        );
3986        crate::test_complete!("report_hydrates_temporal_oracles_from_state_snapshot");
3987    }
3988
3989    #[test]
3990    fn report_hydrates_quiescence_from_finalizers_and_obligations() {
3991        init_test("report_hydrates_quiescence_from_finalizers_and_obligations");
3992        let mut runtime = LabRuntime::with_seed(32);
3993        let root = runtime.state.create_root_region(Budget::INFINITE);
3994        let (task_id, _handle) = runtime
3995            .state
3996            .create_task(root, Budget::INFINITE, async {})
3997            .expect("create task");
3998
3999        runtime.state.now = Time::from_nanos(10);
4000        let registered = runtime.state.register_sync_finalizer(root, || {});
4001        crate::assert_with_log!(registered, "registered finalizer", true, registered);
4002
4003        runtime.state.now = Time::from_nanos(20);
4004        let _obligation = runtime
4005            .state
4006            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
4007            .expect("create obligation");
4008
4009        runtime.state.now = Time::from_nanos(30);
4010        runtime
4011            .state
4012            .update_task(task_id, |record| record.complete(Outcome::Ok(())))
4013            .expect("complete task without auto-resolving obligation");
4014
4015        runtime.state.now = Time::from_nanos(40);
4016        runtime.state.record_finalizer_close_for_test(root);
4017        runtime
4018            .state
4019            .region(root)
4020            .expect("region exists")
4021            .set_state(crate::record::region::RegionState::Closed);
4022
4023        let report = runtime.report();
4024        let quiescence = report
4025            .oracle_report
4026            .entry("quiescence")
4027            .expect("quiescence entry");
4028        let finalizer = report
4029            .oracle_report
4030            .entry("finalizer")
4031            .expect("finalizer entry");
4032        let obligation_leak = report
4033            .oracle_report
4034            .entry("obligation_leak")
4035            .expect("obligation entry");
4036
4037        crate::assert_with_log!(
4038            !quiescence.passed,
4039            "quiescence failed",
4040            false,
4041            quiescence.passed
4042        );
4043        let quiescence_text = quiescence
4044            .violation
4045            .as_deref()
4046            .expect("quiescence violation text");
4047        crate::assert_with_log!(
4048            quiescence_text.contains("1 unrun finalizers"),
4049            "quiescence mentions finalizers",
4050            true,
4051            quiescence_text.contains("1 unrun finalizers")
4052        );
4053        crate::assert_with_log!(
4054            quiescence_text.contains("1 leaked obligations"),
4055            "quiescence mentions obligations",
4056            true,
4057            quiescence_text.contains("1 leaked obligations")
4058        );
4059        crate::assert_with_log!(
4060            !finalizer.passed,
4061            "finalizer failed",
4062            false,
4063            finalizer.passed
4064        );
4065        crate::assert_with_log!(
4066            !obligation_leak.passed,
4067            "obligation_leak failed",
4068            false,
4069            obligation_leak.passed
4070        );
4071        crate::test_complete!("report_hydrates_quiescence_from_finalizers_and_obligations");
4072    }
4073
4074    #[test]
4075    fn report_hydrates_cancellation_propagation_from_state_snapshot() {
4076        init_test("report_hydrates_cancellation_propagation_from_state_snapshot");
4077        // This test intentionally drives a cancellation-propagation violation
4078        // to verify it surfaces in the temporal report. Default configs panic
4079        // on such violations during `report()`, so the oracle must be allowed
4080        // to merely *record* the violation instead of aborting the test.
4081        let config = LabConfig::new(32).panic_on_cancellation_violation(false);
4082        let mut runtime = LabRuntime::new(config);
4083        let root = runtime.state.create_root_region(Budget::INFINITE);
4084        let _child = runtime
4085            .state
4086            .create_child_region(root, Budget::INFINITE)
4087            .expect("create child");
4088
4089        runtime
4090            .state
4091            .region(root)
4092            .expect("root exists")
4093            .cancel_request(crate::types::CancelReason::shutdown());
4094
4095        let report = runtime.report();
4096        let cancellation = report
4097            .oracle_report
4098            .entry("cancellation_protocol")
4099            .expect("cancellation_protocol entry");
4100        crate::assert_with_log!(
4101            !cancellation.passed,
4102            "cancellation_protocol failed",
4103            false,
4104            cancellation.passed
4105        );
4106        let has_temporal_tag = report
4107            .invariant_violations
4108            .iter()
4109            .any(|v| v == "temporal:cancellation_protocol");
4110        crate::assert_with_log!(
4111            has_temporal_tag,
4112            "temporal cancellation marker present",
4113            true,
4114            has_temporal_tag
4115        );
4116        crate::test_complete!("report_hydrates_cancellation_propagation_from_state_snapshot");
4117    }
4118
4119    #[test]
4120    fn report_surfaces_refinement_firewall_violation_from_trace_snapshot() {
4121        init_test("report_surfaces_refinement_firewall_violation_from_trace_snapshot");
4122        let mut runtime = LabRuntime::with_seed(33);
4123        let region = RegionId::new_for_test(41, 0);
4124        let task = TaskId::new_for_test(7, 0);
4125
4126        runtime
4127            .state
4128            .trace
4129            .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4130        runtime
4131            .state
4132            .trace
4133            .push_event(TraceEvent::spawn(2, Time::ZERO, task, region));
4134
4135        let report = runtime.report();
4136        crate::assert_with_log!(
4137            report.refinement_firewall_rule_id.as_deref() == Some("RFW-SPAWN-001"),
4138            "refinement rule id surfaced",
4139            Some("RFW-SPAWN-001"),
4140            report.refinement_firewall_rule_id.as_deref()
4141        );
4142        crate::assert_with_log!(
4143            report.refinement_firewall_event_index == Some(1),
4144            "refinement event index surfaced",
4145            Some(1usize),
4146            report.refinement_firewall_event_index
4147        );
4148        crate::assert_with_log!(
4149            report.refinement_counterexample_prefix_len == Some(2),
4150            "refinement prefix len surfaced",
4151            Some(2usize),
4152            report.refinement_counterexample_prefix_len
4153        );
4154        let has_marker = report
4155            .invariant_violations
4156            .iter()
4157            .any(|v| v == "refinement_firewall:RFW-SPAWN-001");
4158        crate::assert_with_log!(
4159            has_marker,
4160            "refinement invariant marker present",
4161            true,
4162            has_marker
4163        );
4164        let json = report.to_json();
4165        crate::assert_with_log!(
4166            json["refinement_firewall"]["rule_id"] == "RFW-SPAWN-001",
4167            "refinement json rule id",
4168            "RFW-SPAWN-001",
4169            json["refinement_firewall"]["rule_id"]
4170        );
4171        crate::assert_with_log!(
4172            json["refinement_firewall"]["counterexample_prefix_len"] == 2,
4173            "refinement json prefix len",
4174            2,
4175            json["refinement_firewall"]["counterexample_prefix_len"]
4176        );
4177        crate::assert_with_log!(
4178            json["refinement_firewall"]["skipped_due_to_trace_truncation"] == false,
4179            "refinement json not skipped",
4180            false,
4181            json["refinement_firewall"]["skipped_due_to_trace_truncation"]
4182        );
4183        crate::test_complete!("report_surfaces_refinement_firewall_violation_from_trace_snapshot");
4184    }
4185
4186    // br-asupersync-9ri7x0: trace truncation no longer silently
4187    // disables the refinement-firewall oracle. The flag remains set
4188    // (it is part of the report contract) and the rule_id is None
4189    // because the firewall did not run, but the scenario must now
4190    // surface a hard 'scenario_failed_due_to_trace_truncation'
4191    // invariant_violations entry so any caller that gates on
4192    // invariant_violations.is_empty() will fail loudly. The seed and
4193    // truncation watermark are embedded in the message so an
4194    // operator can immediately bump trace_capacity or split the
4195    // scenario.
4196    #[test]
4197    fn report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0() {
4198        init_test("report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0");
4199        let seed = 35;
4200        let config = LabConfig::new(seed).trace_capacity(1);
4201        let mut runtime = LabRuntime::new(config);
4202        let region = RegionId::new_for_test(43, 0);
4203        let task = TaskId::new_for_test(9, 0);
4204
4205        runtime
4206            .state
4207            .trace
4208            .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4209        runtime
4210            .state
4211            .trace
4212            .push_event(TraceEvent::complete(2, Time::ZERO, task, region));
4213
4214        let report = runtime.report();
4215        crate::assert_with_log!(
4216            report.refinement_firewall_skipped_due_to_trace_truncation,
4217            "refinement_firewall_skipped_due_to_trace_truncation",
4218            true,
4219            report.refinement_firewall_skipped_due_to_trace_truncation
4220        );
4221        crate::assert_with_log!(
4222            report.refinement_firewall_rule_id.is_none(),
4223            "no real rule_id when firewall could not run",
4224            true,
4225            report.refinement_firewall_rule_id.is_none()
4226        );
4227
4228        // The new contract: invariant_violations MUST contain a
4229        // hard-fail marker that names the truncation cause + seed +
4230        // watermark. The substring is checked rather than equality
4231        // so future format adjustments stay backward-compatible.
4232        let truncation_marker = report
4233            .invariant_violations
4234            .iter()
4235            .find(|v| v.starts_with("refinement_firewall:scenario_failed_due_to_trace_truncation"));
4236        let marker = truncation_marker.expect(
4237            "truncation must surface as a hard refinement_firewall:scenario_failed_due_to_trace_truncation marker",
4238        );
4239        assert!(
4240            marker.contains(&format!("seed={seed}")),
4241            "truncation marker must embed the seed, got: {marker}"
4242        );
4243        assert!(
4244            marker.contains("total_pushed=") && marker.contains("buffered="),
4245            "truncation marker must embed total_pushed + buffered watermark, got: {marker}"
4246        );
4247        // And of course the scenario must NOT report 'passed':
4248        // invariant_violations.is_empty() is the standard pass gate.
4249        crate::assert_with_log!(
4250            !report.invariant_violations.is_empty(),
4251            "scenario must fail loudly when trace buffer truncates",
4252            true,
4253            !report.invariant_violations.is_empty()
4254        );
4255
4256        let json = report.to_json();
4257        crate::assert_with_log!(
4258            json["refinement_firewall"]["skipped_due_to_trace_truncation"] == true,
4259            "skipped flag still serialized for downstream tooling",
4260            true,
4261            json["refinement_firewall"]["skipped_due_to_trace_truncation"]
4262        );
4263        crate::test_complete!("report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0");
4264    }
4265
4266    // br-asupersync-7uu7sa: chaos-injected wakeup_storm must be
4267    // suppressed when the targeted task's owning region has already
4268    // transitioned to Closing/Draining/Closed. Drives the inner
4269    // method directly, then drives it again after flipping the
4270    // owning region's state to Closing — the second call must NOT
4271    // schedule the task.
4272    #[test]
4273    fn inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa() {
4274        init_test("inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa");
4275
4276        let mut runtime = LabRuntime::with_seed(42);
4277        let region = runtime.state.create_root_region(Budget::INFINITE);
4278        let task_idx = runtime.state.insert_task(TaskRecord::new_with_time(
4279            TaskId::from_arena(ArenaIndex::new(0, 0)),
4280            region,
4281            Budget::INFINITE,
4282            runtime.state.now,
4283        ));
4284        let task_id = TaskId::from_arena(task_idx);
4285        runtime.state.task_mut(task_id).unwrap().id = task_id;
4286
4287        // Region is Open: the inner method must schedule the task.
4288        runtime.inject_spurious_wakes(task_id, 0, 2);
4289        let scheduled_open = {
4290            let mut sched = runtime.scheduler.lock();
4291            let mut count = 0u32;
4292            while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4293                count += 1;
4294            }
4295            count
4296        };
4297        crate::assert_with_log!(
4298            scheduled_open == 2,
4299            "while region is Open, 2 spurious wakes are scheduled",
4300            2u32,
4301            scheduled_open
4302        );
4303
4304        // Transition the region into a non-accepting state.
4305        runtime
4306            .state
4307            .region(region)
4308            .expect("region exists")
4309            .set_state(crate::record::region::RegionState::Closing);
4310
4311        // Now the call must be silently suppressed — no work on the queue.
4312        runtime.inject_spurious_wakes(task_id, 0, 5);
4313        let scheduled_closing = {
4314            let mut sched = runtime.scheduler.lock();
4315            let mut count = 0u32;
4316            while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4317                count += 1;
4318            }
4319            count
4320        };
4321        crate::assert_with_log!(
4322            scheduled_closing == 0,
4323            "after region.set_state(Closing), spurious wakes are suppressed",
4324            0u32,
4325            scheduled_closing
4326        );
4327
4328        // Defense-in-depth: walk every other terminal-ish state.
4329        for state in [
4330            crate::record::region::RegionState::Draining,
4331            crate::record::region::RegionState::Closed,
4332        ] {
4333            runtime
4334                .state
4335                .region(region)
4336                .expect("region exists")
4337                .set_state(state);
4338            runtime.inject_spurious_wakes(task_id, 0, 3);
4339            let scheduled = {
4340                let mut sched = runtime.scheduler.lock();
4341                let mut count = 0u32;
4342                while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4343                    count += 1;
4344                }
4345                count
4346            };
4347            assert_eq!(
4348                scheduled, 0,
4349                "spurious wakes must be suppressed in region state {state:?}"
4350            );
4351        }
4352
4353        crate::test_complete!(
4354            "inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa"
4355        );
4356    }
4357
4358    #[test]
4359    fn crashpack_includes_refinement_firewall_markers() {
4360        init_test("crashpack_includes_refinement_firewall_markers");
4361        let mut runtime = LabRuntime::with_seed(34);
4362        let region = RegionId::new_for_test(42, 0);
4363        let task = TaskId::new_for_test(8, 0);
4364
4365        runtime
4366            .state
4367            .trace
4368            .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4369        runtime
4370            .state
4371            .trace
4372            .push_event(TraceEvent::spawn(2, Time::ZERO, task, region));
4373
4374        let run = runtime.report();
4375        let crashpack = runtime
4376            .build_crashpack_for_report(&run)
4377            .expect("refinement-firewall failure should build crashpack");
4378        let has_rule_marker = crashpack
4379            .oracle_violations
4380            .iter()
4381            .any(|entry| entry == "refinement_firewall:RFW-SPAWN-001");
4382        crate::assert_with_log!(
4383            has_rule_marker,
4384            "crashpack includes refinement rule marker",
4385            true,
4386            has_rule_marker
4387        );
4388        let has_prefix_marker = crashpack
4389            .oracle_violations
4390            .iter()
4391            .any(|entry| entry == "refinement_firewall:minimal_counterexample_prefix_len=2");
4392        crate::assert_with_log!(
4393            has_prefix_marker,
4394            "crashpack includes refinement prefix marker",
4395            true,
4396            has_prefix_marker
4397        );
4398        crate::test_complete!("crashpack_includes_refinement_firewall_markers");
4399    }
4400
4401    #[test]
4402    #[allow(clippy::too_many_lines)]
4403    fn obligation_trace_events_emitted() {
4404        init_test("obligation_trace_events_emitted");
4405        let mut runtime = LabRuntime::with_seed(21);
4406        let root = runtime.state.create_root_region(Budget::INFINITE);
4407
4408        let task_idx = runtime.state.insert_task(TaskRecord::new(
4409            TaskId::from_arena(ArenaIndex::new(0, 0)),
4410            root,
4411            Budget::INFINITE,
4412        ));
4413        let task_id = TaskId::from_arena(task_idx);
4414        runtime.state.task_mut(task_id).unwrap().id = task_id;
4415
4416        runtime.advance_time_to(Time::from_nanos(10));
4417        let ob1 = runtime
4418            .state
4419            .create_obligation(ObligationKind::SendPermit, task_id, root, None)
4420            .unwrap();
4421
4422        runtime.advance_time_to(Time::from_nanos(25));
4423        runtime.state.commit_obligation(ob1).unwrap();
4424
4425        runtime.advance_time_to(Time::from_nanos(30));
4426        let ob2 = runtime
4427            .state
4428            .create_obligation(ObligationKind::Ack, task_id, root, None)
4429            .unwrap();
4430
4431        runtime.advance_time_to(Time::from_nanos(50));
4432        runtime
4433            .state
4434            .abort_obligation(ob2, ObligationAbortReason::Cancel)
4435            .unwrap();
4436
4437        let commit_event = runtime
4438            .trace()
4439            .snapshot()
4440            .into_iter()
4441            .find(|e| e.kind == TraceEventKind::ObligationCommit)
4442            .expect("commit event");
4443        match &commit_event.data {
4444            TraceData::Obligation {
4445                obligation,
4446                task,
4447                region,
4448                kind,
4449                state,
4450                duration_ns,
4451                abort_reason,
4452            } => {
4453                crate::assert_with_log!(*obligation == ob1, "obligation", ob1, *obligation);
4454                crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4455                crate::assert_with_log!(*region == root, "region", root, *region);
4456                crate::assert_with_log!(
4457                    *kind == ObligationKind::SendPermit,
4458                    "kind",
4459                    ObligationKind::SendPermit,
4460                    *kind
4461                );
4462                crate::assert_with_log!(
4463                    *state == crate::record::ObligationState::Committed,
4464                    "state",
4465                    crate::record::ObligationState::Committed,
4466                    *state
4467                );
4468                crate::assert_with_log!(
4469                    duration_ns == &Some(15),
4470                    "duration",
4471                    &Some(15),
4472                    duration_ns
4473                );
4474                crate::assert_with_log!(
4475                    abort_reason.is_none(),
4476                    "abort_reason",
4477                    &None::<crate::record::ObligationAbortReason>,
4478                    abort_reason
4479                );
4480            }
4481            other => panic!("unexpected commit data: {other:?}"),
4482        }
4483
4484        let abort_event = runtime
4485            .trace()
4486            .snapshot()
4487            .into_iter()
4488            .find(|e| e.kind == TraceEventKind::ObligationAbort)
4489            .expect("abort event");
4490        match &abort_event.data {
4491            TraceData::Obligation {
4492                obligation,
4493                task,
4494                region,
4495                kind,
4496                state,
4497                duration_ns,
4498                abort_reason,
4499            } => {
4500                crate::assert_with_log!(*obligation == ob2, "obligation", ob2, *obligation);
4501                crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4502                crate::assert_with_log!(*region == root, "region", root, *region);
4503                crate::assert_with_log!(
4504                    *kind == ObligationKind::Ack,
4505                    "kind",
4506                    ObligationKind::Ack,
4507                    *kind
4508                );
4509                crate::assert_with_log!(
4510                    *state == crate::record::ObligationState::Aborted,
4511                    "state",
4512                    crate::record::ObligationState::Aborted,
4513                    *state
4514                );
4515                crate::assert_with_log!(
4516                    duration_ns == &Some(20),
4517                    "duration",
4518                    &Some(20),
4519                    duration_ns
4520                );
4521                crate::assert_with_log!(
4522                    abort_reason == &Some(ObligationAbortReason::Cancel),
4523                    "abort_reason",
4524                    &Some(ObligationAbortReason::Cancel),
4525                    abort_reason
4526                );
4527            }
4528            other => panic!("unexpected abort data: {other:?}"),
4529        }
4530        crate::test_complete!("obligation_trace_events_emitted");
4531    }
4532
4533    #[test]
4534    fn obligation_leak_emits_trace_event() {
4535        init_test("obligation_leak_emits_trace_event");
4536        let mut runtime = LabRuntime::with_seed(22);
4537        let root = runtime.state.create_root_region(Budget::INFINITE);
4538
4539        let task_idx = runtime.state.insert_task(TaskRecord::new(
4540            TaskId::from_arena(ArenaIndex::new(0, 0)),
4541            root,
4542            Budget::INFINITE,
4543        ));
4544        let task_id = TaskId::from_arena(task_idx);
4545        runtime.state.task_mut(task_id).unwrap().id = task_id;
4546
4547        runtime.advance_time_to(Time::from_nanos(100));
4548        let obligation = runtime
4549            .state
4550            .create_obligation(ObligationKind::Lease, task_id, root, None)
4551            .unwrap();
4552
4553        runtime.advance_time_to(Time::from_nanos(140));
4554        runtime
4555            .state
4556            .update_task(task_id, |record| record.complete(Outcome::Ok(())))
4557            .unwrap();
4558
4559        let violations = runtime.check_invariants();
4560        let has_leak = violations
4561            .iter()
4562            .any(|v| matches!(v, InvariantViolation::ObligationLeak { .. }));
4563        crate::assert_with_log!(has_leak, "has leak", true, has_leak);
4564
4565        let leak_event = runtime
4566            .trace()
4567            .snapshot()
4568            .into_iter()
4569            .find(|e| e.kind == TraceEventKind::ObligationLeak)
4570            .expect("leak event");
4571        match &leak_event.data {
4572            TraceData::Obligation {
4573                obligation: leaked,
4574                task,
4575                region,
4576                kind,
4577                state,
4578                duration_ns,
4579                abort_reason,
4580            } => {
4581                crate::assert_with_log!(*leaked == obligation, "obligation", obligation, *leaked);
4582                crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4583                crate::assert_with_log!(*region == root, "region", root, *region);
4584                crate::assert_with_log!(
4585                    *kind == ObligationKind::Lease,
4586                    "kind",
4587                    ObligationKind::Lease,
4588                    *kind
4589                );
4590                crate::assert_with_log!(
4591                    *state == crate::record::ObligationState::Leaked,
4592                    "state",
4593                    crate::record::ObligationState::Leaked,
4594                    *state
4595                );
4596                crate::assert_with_log!(
4597                    duration_ns == &Some(40),
4598                    "duration",
4599                    &Some(40),
4600                    duration_ns
4601                );
4602                crate::assert_with_log!(
4603                    abort_reason.is_none(),
4604                    "abort_reason",
4605                    &None::<crate::record::ObligationAbortReason>,
4606                    abort_reason
4607                );
4608            }
4609            other => panic!("unexpected leak data: {other:?}"),
4610        }
4611        crate::test_complete!("obligation_leak_emits_trace_event");
4612    }
4613
4614    // =========================================================================
4615    // Agent Report Contract tests (bd-f262i)
4616    // =========================================================================
4617
4618    /// The JSON schema must contain all required top-level keys.
4619    #[test]
4620    fn contract_json_has_required_top_level_keys() {
4621        crate::test_utils::init_test_logging();
4622        crate::test_phase!("contract_json_has_required_top_level_keys");
4623
4624        let app = crate::app::AppSpec::new("contract_test");
4625        let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4626        let report = harness.run_to_report().unwrap();
4627        let json = report.to_json();
4628
4629        // Required top-level keys per bd-f262i contract.
4630        let required_keys = [
4631            "schema_version",
4632            "verdict",
4633            "app",
4634            "lab",
4635            "fingerprints",
4636            "run",
4637            "crashpack",
4638            "attachments",
4639        ];
4640        for key in &required_keys {
4641            assert!(
4642                json.get(key).is_some(),
4643                "missing required top-level key: {key}"
4644            );
4645        }
4646
4647        // Nested required keys.
4648        assert!(json["app"]["name"].is_string(), "app.name must be a string");
4649        assert!(
4650            json["lab"]["config"].is_object(),
4651            "lab.config must be an object"
4652        );
4653        assert!(
4654            json["lab"]["config_hash"].is_u64(),
4655            "lab.config_hash must be a u64"
4656        );
4657        assert!(
4658            json["fingerprints"]["trace"].is_u64(),
4659            "fingerprints.trace must be a u64"
4660        );
4661        assert!(
4662            json["fingerprints"]["event_hash"].is_u64(),
4663            "fingerprints.event_hash must be a u64"
4664        );
4665        assert!(
4666            json["fingerprints"]["event_count"].is_u64(),
4667            "fingerprints.event_count must be a u64"
4668        );
4669        assert!(
4670            json["fingerprints"]["schedule_hash"].is_u64(),
4671            "fingerprints.schedule_hash must be a u64"
4672        );
4673        assert!(json["run"]["seed"].is_u64(), "run.seed must be a u64");
4674        assert!(
4675            json["run"]["oracles"].is_object(),
4676            "run.oracles must be an object"
4677        );
4678        assert!(
4679            json["run"]["invariants"].is_array(),
4680            "run.invariants must be an array"
4681        );
4682        assert!(
4683            json["run"]["refinement_firewall"].is_object(),
4684            "run.refinement_firewall must be an object"
4685        );
4686        assert!(
4687            json["run"]["refinement_firewall"]["skipped_due_to_trace_truncation"].is_boolean(),
4688            "run.refinement_firewall.skipped_due_to_trace_truncation must be a boolean"
4689        );
4690        assert!(
4691            json["attachments"].is_array(),
4692            "attachments must be an array"
4693        );
4694        assert!(
4695            json["crashpack"].is_null(),
4696            "passing runs should have null crashpack linkage"
4697        );
4698
4699        crate::test_complete!("contract_json_has_required_top_level_keys");
4700    }
4701
4702    /// Schema version must be the current constant.
4703    #[test]
4704    fn contract_schema_version_is_current() {
4705        crate::test_utils::init_test_logging();
4706        crate::test_phase!("contract_schema_version_is_current");
4707
4708        let app = crate::app::AppSpec::new("version_test");
4709        let harness = crate::lab::SporkAppHarness::with_seed(1, app).unwrap();
4710        let report = harness.run_to_report().unwrap();
4711
4712        assert_eq!(report.schema_version, SporkHarnessReport::SCHEMA_VERSION);
4713        assert_eq!(
4714            report.to_json()["schema_version"],
4715            SporkHarnessReport::SCHEMA_VERSION
4716        );
4717
4718        crate::test_complete!("contract_schema_version_is_current");
4719    }
4720
4721    /// Config hash is deterministic: same config -> same hash.
4722    #[test]
4723    fn contract_config_hash_deterministic() {
4724        crate::test_utils::init_test_logging();
4725        crate::test_phase!("contract_config_hash_deterministic");
4726
4727        let config = LabConfig::new(42);
4728        let summary_a = LabConfigSummary::from_config(&config);
4729        let summary_b = LabConfigSummary::from_config(&config);
4730
4731        assert_eq!(summary_a.config_hash(), summary_b.config_hash());
4732
4733        // Different seed -> different hash.
4734        let config_2 = LabConfig::new(99);
4735        let summary_c = LabConfigSummary::from_config(&config_2);
4736        assert_ne!(summary_a.config_hash(), summary_c.config_hash());
4737
4738        crate::test_complete!("contract_config_hash_deterministic");
4739    }
4740
4741    /// Verdict field correctly reflects pass/fail.
4742    #[test]
4743    fn contract_verdict_reflects_oracle_state() {
4744        crate::test_utils::init_test_logging();
4745        crate::test_phase!("contract_verdict_reflects_oracle_state");
4746
4747        let app = crate::app::AppSpec::new("verdict_test");
4748        let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4749        let report = harness.run_to_report().unwrap();
4750
4751        // Empty app should pass.
4752        assert!(report.passed());
4753        assert_eq!(report.to_json()["verdict"], "pass");
4754        assert!(report.summary_line().starts_with("[PASS]"));
4755
4756        crate::test_complete!("contract_verdict_reflects_oracle_state");
4757    }
4758
4759    /// Agent UX convenience methods return consistent values.
4760    #[test]
4761    fn contract_convenience_methods_consistent() {
4762        crate::test_utils::init_test_logging();
4763        crate::test_phase!("contract_convenience_methods_consistent");
4764
4765        let app = crate::app::AppSpec::new("ux_test");
4766        let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4767        let report = harness.run_to_report().unwrap();
4768        let json = report.to_json();
4769
4770        // trace_fingerprint() matches JSON.
4771        assert_eq!(
4772            report.trace_fingerprint(),
4773            json["fingerprints"]["trace"].as_u64().unwrap()
4774        );
4775
4776        // seed() matches JSON.
4777        assert_eq!(report.seed(), json["run"]["seed"].as_u64().unwrap());
4778
4779        // config_hash() matches JSON.
4780        assert_eq!(
4781            report.config_hash(),
4782            json["lab"]["config_hash"].as_u64().unwrap()
4783        );
4784
4785        // No crashpack by default.
4786        assert!(report.crashpack_path().is_none());
4787
4788        // Empty app -> no oracle failures.
4789        assert!(report.oracle_failures().is_empty());
4790
4791        crate::test_complete!("contract_convenience_methods_consistent");
4792    }
4793
4794    /// Failing runs auto-attach a deterministic crashpack reference.
4795    #[test]
4796    fn contract_auto_crashpack_on_failure() {
4797        crate::test_utils::init_test_logging();
4798        crate::test_phase!("contract_auto_crashpack_on_failure");
4799
4800        let config = LabConfig::new(17).panic_on_leak(false);
4801        let mut runtime = LabRuntime::new(config);
4802        let region = runtime.state.create_root_region(Budget::INFINITE);
4803        let (task, _) = runtime
4804            .state
4805            .create_task(region, Budget::INFINITE, async {})
4806            .expect("create task");
4807        runtime.scheduler.lock().schedule(task, 0);
4808        // Create the obligation while the task is still live, then run to
4809        // quiescence so the task completes without resolving it (intentional leak).
4810        runtime
4811            .state
4812            .create_obligation(
4813                ObligationKind::SendPermit,
4814                task,
4815                region,
4816                Some("intentional leak".to_string()),
4817            )
4818            .expect("create obligation");
4819        runtime.run_until_quiescent();
4820
4821        let report = runtime.spork_report("failing_app", Vec::new());
4822        assert!(!report.passed(), "failing run must not report PASS");
4823        let crashpack_path = report
4824            .crashpack_path()
4825            .expect("failing run should include crashpack attachment");
4826        assert!(
4827            crashpack_path.starts_with("crashpack-"),
4828            "unexpected crashpack path: {crashpack_path}"
4829        );
4830        assert!(
4831            report
4832                .attachments
4833                .iter()
4834                .any(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack),
4835            "crashpack attachment kind must be present"
4836        );
4837        let crashpack_link = report
4838            .crashpack_link()
4839            .expect("failing run should expose crashpack link metadata");
4840        assert_eq!(crashpack_link.path, crashpack_path);
4841        assert_eq!(crashpack_link.fingerprint, report.trace_fingerprint());
4842        assert!(
4843            crashpack_link.id.starts_with("crashpack-"),
4844            "unexpected crashpack id: {}",
4845            crashpack_link.id
4846        );
4847        assert!(
4848            crashpack_link.replay.command_line.contains(crashpack_path),
4849            "replay command should include crashpack path"
4850        );
4851
4852        crate::test_complete!("contract_auto_crashpack_on_failure");
4853    }
4854
4855    /// Failing runs with replay recording enabled include a deterministic
4856    /// divergent prefix in the auto-built crashpack.
4857    #[test]
4858    fn contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled() {
4859        crate::test_utils::init_test_logging();
4860        crate::test_phase!("contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled");
4861
4862        let config = LabConfig::new(1701)
4863            .panic_on_leak(false)
4864            .with_default_replay_recording();
4865        let mut runtime = LabRuntime::new(config);
4866        let region = runtime.state.create_root_region(Budget::INFINITE);
4867        let (task, _) = runtime
4868            .state
4869            .create_task(region, Budget::INFINITE, async {})
4870            .expect("create task");
4871        runtime.scheduler.lock().schedule(task, 0);
4872        runtime
4873            .state
4874            .create_obligation(
4875                ObligationKind::SendPermit,
4876                task,
4877                region,
4878                Some("intentional leak".to_string()),
4879            )
4880            .expect("create obligation");
4881        runtime.run_until_quiescent();
4882
4883        let run = runtime.report();
4884        let crashpack = runtime
4885            .build_crashpack_for_report(&run)
4886            .expect("failing run should build crashpack");
4887
4888        assert!(crashpack.has_divergent_prefix());
4889        assert!(
4890            crashpack
4891                .manifest
4892                .has_attachment(&crate::trace::crashpack::AttachmentKind::DivergentPrefix),
4893            "manifest must include divergent prefix attachment"
4894        );
4895        assert!(
4896            crashpack.replay.is_some(),
4897            "crashpack should carry replay command metadata"
4898        );
4899
4900        crate::test_complete!(
4901            "contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled"
4902        );
4903    }
4904
4905    /// Manual crashpack attachments are preserved without auto-duplication.
4906    #[test]
4907    fn contract_manual_crashpack_not_duplicated_on_failure() {
4908        crate::test_utils::init_test_logging();
4909        crate::test_phase!("contract_manual_crashpack_not_duplicated_on_failure");
4910
4911        let config = LabConfig::new(18).panic_on_leak(false);
4912        let mut runtime = LabRuntime::new(config);
4913        let region = runtime.state.create_root_region(Budget::INFINITE);
4914        let (task, _) = runtime
4915            .state
4916            .create_task(region, Budget::INFINITE, async {})
4917            .expect("create task");
4918        runtime.scheduler.lock().schedule(task, 0);
4919        runtime
4920            .state
4921            .create_obligation(
4922                ObligationKind::SendPermit,
4923                task,
4924                region,
4925                Some("intentional leak".to_string()),
4926            )
4927            .expect("create obligation");
4928        runtime.run_until_quiescent();
4929
4930        let report = runtime.spork_report(
4931            "failing_app_manual",
4932            vec![HarnessAttachmentRef::crashpack("manual-crashpack.json")],
4933        );
4934        let crashpack_count = report
4935            .attachments
4936            .iter()
4937            .filter(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack)
4938            .count();
4939        assert_eq!(
4940            crashpack_count, 1,
4941            "manual crashpack should not be duplicated"
4942        );
4943        assert_eq!(report.crashpack_path(), Some("manual-crashpack.json"));
4944        let crashpack_link = report
4945            .crashpack_link()
4946            .expect("manual crashpack should still produce metadata");
4947        assert_eq!(crashpack_link.path, "manual-crashpack.json");
4948        assert!(
4949            crashpack_link
4950                .replay
4951                .command_line
4952                .contains("manual-crashpack.json"),
4953            "replay command should include manual crashpack path"
4954        );
4955
4956        crate::test_complete!("contract_manual_crashpack_not_duplicated_on_failure");
4957    }
4958
4959    /// JSON output is deterministic across runs with same seed.
4960    #[test]
4961    fn contract_json_deterministic_same_seed() {
4962        crate::test_utils::init_test_logging();
4963        crate::test_phase!("contract_json_deterministic_same_seed");
4964
4965        let json_a = {
4966            let app = crate::app::AppSpec::new("det_contract");
4967            let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4968            harness.run_to_report().unwrap().to_json()
4969        };
4970
4971        let json_b = {
4972            let app = crate::app::AppSpec::new("det_contract");
4973            let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4974            harness.run_to_report().unwrap().to_json()
4975        };
4976
4977        // The canonical Foata `trace_fingerprint` is the semantic determinism
4978        // signal. The sequential `event_hash` additionally embeds per-event
4979        // data (e.g. ephemeral IDs allocated from process-global counters)
4980        // that benignly drifts across invocations in the same process even
4981        // for the same seed. Normalise it before comparing so the assertion
4982        // targets what the test actually contracts for ("same seed →
4983        // equivalent run artefact") rather than incidental monotonic counters.
4984        fn strip_event_hash(obj: &mut serde_json::Map<String, serde_json::Value>) {
4985            if obj.contains_key("event_hash") {
4986                obj.insert("event_hash".into(), serde_json::Value::Null);
4987            }
4988            for val in obj.values_mut() {
4989                if let Some(sub) = val.as_object_mut() {
4990                    strip_event_hash(sub);
4991                }
4992            }
4993        }
4994        let normalize = |mut v: serde_json::Value| -> serde_json::Value {
4995            if let Some(obj) = v.as_object_mut() {
4996                strip_event_hash(obj);
4997            }
4998            v
4999        };
5000
5001        assert_eq!(
5002            normalize(json_a),
5003            normalize(json_b),
5004            "same seed must produce identical JSON (mod sequential event_hash)",
5005        );
5006
5007        crate::test_complete!("contract_json_deterministic_same_seed");
5008    }
5009
5010    /// Attachments appear in the report, sorted by (kind, path).
5011    #[test]
5012    fn contract_attachments_sorted_in_json() {
5013        crate::test_utils::init_test_logging();
5014        crate::test_phase!("contract_attachments_sorted_in_json");
5015
5016        let app = crate::app::AppSpec::new("attach_contract");
5017        let mut harness = crate::lab::SporkAppHarness::with_seed(7, app).unwrap();
5018        // Add in reverse order to verify sorting.
5019        harness.attach(HarnessAttachmentRef::trace("z_trace.json"));
5020        harness.attach(HarnessAttachmentRef::crashpack("a_crash.tar"));
5021
5022        let report = harness.run_to_report().unwrap();
5023
5024        // crashpack_path() returns the crashpack.
5025        assert_eq!(report.crashpack_path(), Some("a_crash.tar"));
5026
5027        let json = report.to_json();
5028        let attachments = json["attachments"].as_array().unwrap();
5029        assert_eq!(attachments.len(), 2);
5030
5031        // CrashPack sorts before Trace (enum ordering).
5032        assert_eq!(attachments[0]["kind"], "crashpack");
5033        assert_eq!(attachments[1]["kind"], "trace");
5034
5035        crate::test_complete!("contract_attachments_sorted_in_json");
5036    }
5037
5038    // =========================================================================
5039    // Virtual Time Control Tests (bd-1hu19.3)
5040    // =========================================================================
5041
5042    #[test]
5043    fn advance_to_next_timer_empty() {
5044        init_test("advance_to_next_timer_empty");
5045        let mut runtime = LabRuntime::with_seed(42);
5046
5047        let wakeups = runtime.advance_to_next_timer();
5048        crate::assert_with_log!(wakeups == 0, "no timers → 0 wakeups", 0, wakeups);
5049
5050        let deadline = runtime.next_timer_deadline();
5051        crate::assert_with_log!(
5052            deadline.is_none(),
5053            "no pending deadline",
5054            true,
5055            deadline.is_none()
5056        );
5057        crate::test_complete!("advance_to_next_timer_empty");
5058    }
5059
5060    #[test]
5061    fn advance_to_next_timer_fires_timer() {
5062        init_test("advance_to_next_timer_fires_timer");
5063        let mut runtime = LabRuntime::with_seed(42);
5064
5065        // Register a timer at t=1s via the timer driver handle
5066        let timer_handle = runtime.state.timer_driver_handle().unwrap();
5067        let woken = Arc::new(std::sync::atomic::AtomicBool::new(false));
5068
5069        let waker = Waker::from(Arc::new(FlagWaker(woken.clone())));
5070        let _ = timer_handle.register(Time::from_secs(1), waker);
5071
5072        // Should have 1 pending timer
5073        let count = runtime.pending_timer_count();
5074        crate::assert_with_log!(count == 1, "1 pending timer", 1, count);
5075
5076        // Advance to next timer
5077        let wakeups = runtime.advance_to_next_timer();
5078        crate::assert_with_log!(wakeups == 1, "1 wakeup", 1, wakeups);
5079
5080        // Time should now be at 1 second
5081        let now = runtime.now();
5082        crate::assert_with_log!(
5083            now == Time::from_secs(1),
5084            "time at 1s",
5085            Time::from_secs(1),
5086            now
5087        );
5088
5089        // Waker should have been called
5090        let was_woken = woken.load(std::sync::atomic::Ordering::SeqCst);
5091        crate::assert_with_log!(was_woken, "waker fired", true, was_woken);
5092        crate::test_complete!("advance_to_next_timer_fires_timer");
5093    }
5094
5095    #[test]
5096    fn metamorphic_timer_registration_permutation_preserves_virtual_time_progression() {
5097        init_test("metamorphic_timer_registration_permutation_preserves_virtual_time_progression");
5098
5099        let baseline = collect_timer_advances(&[5, 1, 3, 1, 8], &[]);
5100        let permuted = collect_timer_advances(&[1, 8, 5, 1, 3], &[]);
5101
5102        crate::assert_with_log!(
5103            baseline.advance_points == permuted.advance_points,
5104            "deadline multiset permutation preserves advance points",
5105            &baseline.advance_points,
5106            &permuted.advance_points
5107        );
5108        crate::assert_with_log!(
5109            baseline.total_wakeups == permuted.total_wakeups,
5110            "deadline multiset permutation preserves wakeup count",
5111            baseline.total_wakeups,
5112            permuted.total_wakeups
5113        );
5114        crate::assert_with_log!(
5115            baseline.final_time == permuted.final_time,
5116            "deadline multiset permutation preserves final virtual time",
5117            baseline.final_time,
5118            permuted.final_time
5119        );
5120        crate::assert_with_log!(
5121            baseline.advance_points
5122                == vec![
5123                    Time::from_secs(1),
5124                    Time::from_secs(3),
5125                    Time::from_secs(5),
5126                    Time::from_secs(8)
5127                ],
5128            "advance points collapse duplicate deadlines without moving backward",
5129            vec![
5130                Time::from_secs(1),
5131                Time::from_secs(3),
5132                Time::from_secs(5),
5133                Time::from_secs(8)
5134            ],
5135            baseline.advance_points.clone()
5136        );
5137
5138        crate::test_complete!(
5139            "metamorphic_timer_registration_permutation_preserves_virtual_time_progression"
5140        );
5141    }
5142
5143    #[test]
5144    fn metamorphic_cancelled_timer_does_not_skew_virtual_time_progression() {
5145        init_test("metamorphic_cancelled_timer_does_not_skew_virtual_time_progression");
5146
5147        let baseline = collect_timer_advances(&[2, 4, 9], &[]);
5148        let with_cancelled_timer = collect_timer_advances(&[2, 4, 6, 9], &[2]);
5149
5150        crate::assert_with_log!(
5151            baseline.advance_points == with_cancelled_timer.advance_points,
5152            "cancelling an intermediate timer preserves surviving advance points",
5153            &baseline.advance_points,
5154            &with_cancelled_timer.advance_points
5155        );
5156        crate::assert_with_log!(
5157            baseline.total_wakeups == with_cancelled_timer.total_wakeups,
5158            "cancelling an intermediate timer preserves surviving wakeup count",
5159            baseline.total_wakeups,
5160            with_cancelled_timer.total_wakeups
5161        );
5162        crate::assert_with_log!(
5163            baseline.final_time == with_cancelled_timer.final_time,
5164            "cancelling an intermediate timer preserves final virtual time",
5165            baseline.final_time,
5166            with_cancelled_timer.final_time
5167        );
5168        crate::assert_with_log!(
5169            with_cancelled_timer.cancelled_wakeups == 0,
5170            "cancelled timer never wakes after auto-advance",
5171            0u64,
5172            with_cancelled_timer.cancelled_wakeups
5173        );
5174
5175        crate::test_complete!("metamorphic_cancelled_timer_does_not_skew_virtual_time_progression");
5176    }
5177
5178    #[cfg(unix)]
5179    #[test]
5180    fn run_with_auto_advance_delivers_delayed_reactor_events() {
5181        init_test("run_with_auto_advance_delivers_delayed_reactor_events");
5182        let config = LabConfig::new(42).with_auto_advance().max_steps(32);
5183        let mut runtime = LabRuntime::new(config);
5184        let handle = runtime.state.io_driver_handle().expect("io driver");
5185        let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5186        let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5187        let source = TestFdSource;
5188
5189        let registration = handle
5190            .register(&source, Interest::READABLE, waker)
5191            .expect("register source");
5192        let token = registration.token();
5193
5194        runtime
5195            .lab_reactor()
5196            .inject_event(token, Event::readable(token), Duration::from_secs(1));
5197
5198        let report = runtime.run_with_auto_advance();
5199
5200        crate::assert_with_log!(
5201            report.auto_advances >= 1,
5202            "auto-advance reaches delayed reactor deadline",
5203            true,
5204            report.auto_advances >= 1
5205        );
5206        crate::assert_with_log!(
5207            runtime.now() >= Time::from_secs(1),
5208            "virtual time advanced to delayed reactor event",
5209            true,
5210            runtime.now() >= Time::from_secs(1)
5211        );
5212        let wakeups = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5213        crate::assert_with_log!(
5214            wakeups == 1,
5215            "reactor event woke registration",
5216            1u64,
5217            wakeups
5218        );
5219        let saw_ready = runtime
5220            .state
5221            .trace
5222            .snapshot()
5223            .iter()
5224            .any(|event| event.kind == TraceEventKind::IoReady);
5225        crate::assert_with_log!(saw_ready, "io ready trace recorded", true, saw_ready);
5226        let next_event = runtime.lab_reactor().next_event_time();
5227        crate::assert_with_log!(
5228            next_event.is_none(),
5229            "delayed reactor event drained",
5230            true,
5231            next_event.is_none()
5232        );
5233
5234        crate::test_complete!("run_with_auto_advance_delivers_delayed_reactor_events");
5235    }
5236
5237    #[test]
5238    fn run_with_auto_advance_basic() {
5239        init_test("run_with_auto_advance_basic");
5240        let config = LabConfig::new(42).with_auto_advance();
5241        let mut runtime = LabRuntime::new(config);
5242
5243        // No tasks, no timers → immediate quiescence
5244        let report = runtime.run_with_auto_advance();
5245        crate::assert_with_log!(report.steps == 0, "0 steps", 0u64, report.steps);
5246        crate::assert_with_log!(
5247            report.auto_advances == 0,
5248            "0 auto-advances",
5249            0u64,
5250            report.auto_advances
5251        );
5252        crate::test_complete!("run_with_auto_advance_basic");
5253    }
5254
5255    #[test]
5256    fn run_with_auto_advance_jumps_past_timer_deadlines() {
5257        init_test("run_with_auto_advance_jumps_past_timer_deadlines");
5258        let config = LabConfig::new(42).with_auto_advance().max_steps(1_000);
5259        let mut runtime = LabRuntime::new(config);
5260
5261        // Register timers at 1s, 5s, and 10s via timer driver
5262        let timer_handle = runtime.state.timer_driver_handle().unwrap();
5263        let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5264
5265        for secs in [1, 5, 10] {
5266            let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5267            let _ = timer_handle.register(Time::from_secs(secs), waker);
5268        }
5269
5270        let report = runtime.run_with_auto_advance();
5271
5272        // All 3 timer deadlines should have been auto-advanced to
5273        crate::assert_with_log!(
5274            report.auto_advances >= 3,
5275            "at least 3 auto-advances",
5276            true,
5277            report.auto_advances >= 3
5278        );
5279
5280        // Virtual time should be at or past 10 seconds
5281        let now = runtime.now();
5282        crate::assert_with_log!(
5283            now >= Time::from_secs(10),
5284            "time >= 10s",
5285            true,
5286            now >= Time::from_secs(10)
5287        );
5288
5289        // All wakers should have been called
5290        let count = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5291        crate::assert_with_log!(count == 3, "3 wakeups", 3u64, count);
5292        crate::test_complete!("run_with_auto_advance_jumps_past_timer_deadlines");
5293    }
5294
5295    #[test]
5296    fn virtual_time_24_hour_instant_test() {
5297        init_test("virtual_time_24_hour_instant_test");
5298        // Acceptance criterion: 24 hours of virtual time in <1 second wall time.
5299        let config = LabConfig::new(42).with_auto_advance().max_steps(100_000);
5300        let mut runtime = LabRuntime::new(config);
5301
5302        // Register timers spread across 24 hours (every hour)
5303        let timer_handle = runtime.state.timer_driver_handle().unwrap();
5304        let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5305
5306        for hour in 1..=24 {
5307            let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5308            let _ = timer_handle.register(Time::from_secs(hour * 3600), waker);
5309        }
5310
5311        let wall_start = std::time::Instant::now();
5312        let report = runtime.run_with_auto_advance();
5313        let wall_elapsed = wall_start.elapsed();
5314
5315        // Virtual time should span 24 hours = 86400 seconds
5316        crate::assert_with_log!(
5317            report.virtual_elapsed_secs() >= 86400,
5318            "24h virtual",
5319            true,
5320            report.virtual_elapsed_secs() >= 86400
5321        );
5322
5323        // All 24 timers fired
5324        let count = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5325        crate::assert_with_log!(count == 24, "24 wakeups", 24u64, count);
5326
5327        // Wall time should be well under 1 second (typically <1ms)
5328        let wall_ms = wall_elapsed.as_millis();
5329        crate::assert_with_log!(wall_ms < 1000, "wall time < 1s", true, wall_ms < 1000);
5330        crate::test_complete!("virtual_time_24_hour_instant_test");
5331    }
5332
5333    #[test]
5334    fn clock_pause_resume() {
5335        init_test("clock_pause_resume");
5336        let runtime = LabRuntime::with_seed(42);
5337
5338        let not_paused = !runtime.is_clock_paused();
5339        crate::assert_with_log!(not_paused, "not paused initially", true, not_paused);
5340
5341        runtime.pause_clock();
5342        let paused = runtime.is_clock_paused();
5343        crate::assert_with_log!(paused, "paused", true, paused);
5344
5345        runtime.resume_clock();
5346        let resumed = !runtime.is_clock_paused();
5347        crate::assert_with_log!(resumed, "resumed", true, resumed);
5348        crate::test_complete!("clock_pause_resume");
5349    }
5350
5351    #[test]
5352    fn inject_clock_skew() {
5353        init_test("inject_clock_skew");
5354        let mut runtime = LabRuntime::with_seed(42);
5355
5356        runtime.advance_time(1_000_000_000); // 1 second
5357        let before = runtime.now();
5358
5359        // Inject 5 second skew
5360        runtime.inject_clock_skew(5_000_000_000);
5361        let after = runtime.now();
5362
5363        let delta = after.as_nanos() - before.as_nanos();
5364        crate::assert_with_log!(
5365            delta == 5_000_000_000,
5366            "5s skew applied",
5367            5_000_000_000u64,
5368            delta
5369        );
5370
5371        crate::assert_with_log!(
5372            after == Time::from_secs(6),
5373            "time at 6s",
5374            Time::from_secs(6),
5375            after
5376        );
5377        crate::test_complete!("inject_clock_skew");
5378    }
5379
5380    #[test]
5381    fn virtual_time_report_conversions() {
5382        init_test("virtual_time_report_conversions");
5383        let report = VirtualTimeReport {
5384            steps: 100,
5385            auto_advances: 5,
5386            total_wakeups: 10,
5387            time_start: Time::ZERO,
5388            time_end: Time::from_secs(3600),
5389            virtual_elapsed_nanos: 3_600_000_000_000,
5390            termination: AutoAdvanceTermination::Quiescent,
5391        };
5392
5393        let ms = report.virtual_elapsed_ms();
5394        crate::assert_with_log!(ms == 3_600_000, "3600000 ms", 3_600_000u64, ms);
5395
5396        let secs = report.virtual_elapsed_secs();
5397        crate::assert_with_log!(secs == 3600, "3600 secs", 3600u64, secs);
5398        crate::test_complete!("virtual_time_report_conversions");
5399    }
5400
5401    // =========================================================================
5402    // Replay severity correctness (bd-beuyd)
5403    // =========================================================================
5404
5405    /// Regression test: replay recorder must capture the actual completion
5406    /// severity from the finalized task record, not always `Severity::Ok`.
5407    ///
5408    /// `create_task` wraps futures to always return `Outcome::Ok(())` — the
5409    /// real severity is determined by the cancel protocol state machine. This
5410    /// test puts a task through the cancel protocol and verifies the replay
5411    /// trace records the correct `Cancelled` severity.
5412    #[test]
5413    fn replay_records_correct_severity_for_cancelled_task() {
5414        init_test("replay_records_correct_severity_for_cancelled_task");
5415
5416        let config = LabConfig::new(42)
5417            .panic_on_leak(false)
5418            .with_default_replay_recording();
5419        let mut runtime = LabRuntime::new(config);
5420        let root = runtime
5421            .state
5422            .create_root_region(crate::types::Budget::INFINITE);
5423
5424        // Create a task that yields once then completes with Ok.
5425        // The yield allows us to cancel the task before it finishes.
5426        let (task_id, _) = runtime
5427            .state
5428            .create_task(root, crate::types::Budget::INFINITE, async {
5429                crate::runtime::yield_now::yield_now().await;
5430            })
5431            .expect("create task");
5432        runtime.scheduler.lock().schedule(task_id, 0);
5433
5434        // Step once: task yields (Pending)
5435        runtime.step();
5436
5437        // Put the task through cancel protocol: CancelRequested → Cancelling
5438        runtime
5439            .state
5440            .update_task(task_id, |record| {
5441                record.request_cancel(crate::types::CancelReason::user("test-cancel"));
5442                let _ = record.acknowledge_cancel();
5443                // Task is now in Cancelling state
5444                assert!(
5445                    matches!(
5446                        record.state,
5447                        crate::record::task::TaskState::Cancelling { .. }
5448                    ),
5449                    "task should be in Cancelling state"
5450                );
5451            })
5452            .unwrap();
5453
5454        // Reschedule and run to completion: the cancel protocol will
5455        // complete it as Cancelled when the wrapped future returns Ok.
5456        runtime.scheduler.lock().schedule(task_id, 0);
5457        runtime.run_until_quiescent();
5458
5459        // Verify the task completed as Cancelled
5460        if let Some(record) = runtime.state.task(task_id) {
5461            assert!(
5462                matches!(
5463                    record.state,
5464                    crate::record::task::TaskState::Completed(crate::types::Outcome::Cancelled(_))
5465                ),
5466                "task should be Completed(Cancelled), got {:?}",
5467                record.state
5468            );
5469        }
5470
5471        // Check the replay trace for the TaskCompleted event
5472        let replay = runtime
5473            .replay_recorder()
5474            .snapshot()
5475            .expect("replay recording enabled");
5476        let completed_events: Vec<_> = replay
5477            .events
5478            .iter()
5479            .filter(|e| matches!(e, crate::trace::replay::ReplayEvent::TaskCompleted { .. }))
5480            .collect();
5481
5482        assert!(
5483            !completed_events.is_empty(),
5484            "should have at least one TaskCompleted event"
5485        );
5486
5487        // The severity should be Cancelled (2), not Ok (0)
5488        for event in &completed_events {
5489            if let crate::trace::replay::ReplayEvent::TaskCompleted { outcome, .. } = event {
5490                let expected = crate::types::Severity::Cancelled.as_u8();
5491                crate::assert_with_log!(
5492                    *outcome == expected,
5493                    "severity should be Cancelled (2)",
5494                    expected,
5495                    *outcome
5496                );
5497            }
5498        }
5499
5500        crate::test_complete!("replay_records_correct_severity_for_cancelled_task");
5501    }
5502
5503    #[test]
5504    fn replay_recording_metadata_is_stable_for_same_seed() {
5505        init_test("replay_recording_metadata_is_stable_for_same_seed");
5506
5507        let mut first = LabRuntime::new(LabConfig::new(42).with_default_replay_recording());
5508        let mut second = LabRuntime::new(LabConfig::new(42).with_default_replay_recording());
5509
5510        let first_trace = first
5511            .finish_replay_trace()
5512            .expect("first replay recording enabled");
5513        let second_trace = second
5514            .finish_replay_trace()
5515            .expect("second replay recording enabled");
5516
5517        crate::assert_with_log!(
5518            first_trace.metadata == second_trace.metadata,
5519            "replay metadata should match for identical seeds",
5520            &first_trace.metadata,
5521            &second_trace.metadata
5522        );
5523        crate::assert_with_log!(
5524            first_trace.events == second_trace.events,
5525            "replay events should match for identical seeds",
5526            &first_trace.events,
5527            &second_trace.events
5528        );
5529        crate::assert_with_log!(
5530            first_trace.metadata.recorded_at == 0,
5531            "recorded_at defaults to deterministic zero stamp",
5532            0u64,
5533            first_trace.metadata.recorded_at
5534        );
5535
5536        crate::test_complete!("replay_recording_metadata_is_stable_for_same_seed");
5537    }
5538
5539    // =========================================================================
5540    // Pure data-type tests (wave 40 – CyanBarn)
5541    // =========================================================================
5542
5543    #[test]
5544    fn lab_trace_certificate_summary_debug_clone_copy_eq() {
5545        let summary = LabTraceCertificateSummary {
5546            event_hash: 123,
5547            event_count: 456,
5548            schedule_hash: 789,
5549        };
5550        let copied = summary;
5551        let cloned = summary;
5552        assert_eq!(copied, cloned);
5553        assert_ne!(
5554            summary,
5555            LabTraceCertificateSummary {
5556                event_hash: 0,
5557                event_count: 456,
5558                schedule_hash: 789,
5559            }
5560        );
5561        let dbg = format!("{summary:?}");
5562        assert!(dbg.contains("LabTraceCertificateSummary"));
5563    }
5564
5565    #[test]
5566    fn virtual_time_report_debug_clone_copy_eq() {
5567        let report = VirtualTimeReport {
5568            steps: 100,
5569            auto_advances: 5,
5570            total_wakeups: 10,
5571            time_start: Time::ZERO,
5572            time_end: Time::from_millis(500),
5573            virtual_elapsed_nanos: 500_000_000,
5574            termination: AutoAdvanceTermination::Quiescent,
5575        };
5576        let copied = report;
5577        assert_eq!(copied, report);
5578        assert_eq!(report.virtual_elapsed_ms(), 500);
5579        assert_eq!(report.virtual_elapsed_secs(), 0);
5580        let dbg = format!("{report:?}");
5581        assert!(dbg.contains("VirtualTimeReport"));
5582    }
5583
5584    // =========================================================================
5585    // AutoAdvanceTermination tests (bead 56c785)
5586    // =========================================================================
5587
5588    #[test]
5589    fn auto_advance_quiescent_termination() {
5590        init_test("auto_advance_quiescent_termination");
5591        let mut lab = LabRuntime::new(LabConfig::new(42));
5592        // No tasks enqueued → immediately quiescent
5593        let report = lab.run_with_auto_advance();
5594        assert_eq!(
5595            report.termination,
5596            AutoAdvanceTermination::Quiescent,
5597            "empty runtime should terminate as quiescent"
5598        );
5599        crate::test_complete!("auto_advance_quiescent_termination");
5600    }
5601
5602    #[test]
5603    fn auto_advance_step_limit_termination() {
5604        init_test("auto_advance_step_limit_termination");
5605        let mut lab = LabRuntime::new(LabConfig::new(42).max_steps(0));
5606        let report = lab.run_with_auto_advance();
5607        assert_eq!(
5608            report.termination,
5609            AutoAdvanceTermination::StepLimitReached,
5610            "zero max_steps should terminate as step-limit-reached"
5611        );
5612        crate::test_complete!("auto_advance_step_limit_termination");
5613    }
5614
5615    #[test]
5616    fn auto_advance_stuck_bailout_termination() {
5617        init_test("auto_advance_stuck_bailout_termination");
5618        let config = LabConfig::new(42)
5619            .with_auto_advance()
5620            .no_step_limit()
5621            .futurelock_max_idle_steps(0);
5622        let mut lab = LabRuntime::new(config);
5623        let root = lab.state.create_root_region(Budget::INFINITE);
5624        let (task_id, _handle) = lab
5625            .state
5626            .create_task(root, Budget::INFINITE, async {
5627                std::future::pending::<()>().await;
5628            })
5629            .expect("create pending task");
5630        lab.scheduler.lock().schedule(task_id, 0);
5631
5632        let report = lab.run_with_auto_advance();
5633        assert_eq!(
5634            report.termination,
5635            AutoAdvanceTermination::StuckBailout,
5636            "pending task without deadlines should terminate via stuck bailout"
5637        );
5638        assert!(
5639            !lab.is_quiescent(),
5640            "stuck bailout should preserve non-quiescent state for diagnosis"
5641        );
5642        assert_eq!(
5643            report.auto_advances, 0,
5644            "stuck bailout path should not auto-advance virtual time without deadlines"
5645        );
5646        crate::test_complete!("auto_advance_stuck_bailout_termination");
5647    }
5648
5649    #[test]
5650    fn auto_advance_termination_display() {
5651        assert_eq!(
5652            format!("{}", AutoAdvanceTermination::Quiescent),
5653            "quiescent"
5654        );
5655        assert_eq!(
5656            format!("{}", AutoAdvanceTermination::StepLimitReached),
5657            "step-limit-reached"
5658        );
5659        assert_eq!(
5660            format!("{}", AutoAdvanceTermination::StuckBailout),
5661            "stuck-bailout"
5662        );
5663    }
5664
5665    #[test]
5666    fn auto_advance_termination_debug_clone_copy_eq_hash() {
5667        use std::collections::HashSet;
5668        let variants = [
5669            AutoAdvanceTermination::Quiescent,
5670            AutoAdvanceTermination::StepLimitReached,
5671            AutoAdvanceTermination::StuckBailout,
5672        ];
5673        // Copy + Clone + Eq
5674        for &v in &variants {
5675            let copied = v;
5676            let cloned = v;
5677            assert_eq!(copied, cloned);
5678        }
5679        // Hash uniqueness
5680        let mut set = HashSet::new();
5681        for &v in &variants {
5682            assert!(set.insert(v));
5683        }
5684        assert_eq!(set.len(), 3);
5685        // Debug contains type name
5686        let dbg = format!("{:?}", AutoAdvanceTermination::StuckBailout);
5687        assert!(dbg.contains("StuckBailout"));
5688    }
5689
5690    #[test]
5691    fn harness_attachment_kind_debug_clone_copy_eq_hash_ord_display() {
5692        use std::collections::HashSet;
5693        let kinds = [
5694            HarnessAttachmentKind::CrashPack,
5695            HarnessAttachmentKind::ReplayTrace,
5696            HarnessAttachmentKind::Trace,
5697            HarnessAttachmentKind::Other,
5698        ];
5699        // Display
5700        assert_eq!(format!("{}", kinds[0]), "crashpack");
5701        assert_eq!(format!("{}", kinds[1]), "replay_trace");
5702        assert_eq!(format!("{}", kinds[2]), "trace");
5703        assert_eq!(format!("{}", kinds[3]), "other");
5704        // Copy/Clone/Eq
5705        for &k in &kinds {
5706            let copied = k;
5707            let cloned = k;
5708            assert_eq!(copied, cloned);
5709        }
5710        // Hash
5711        let mut set = HashSet::new();
5712        for &k in &kinds {
5713            set.insert(k);
5714        }
5715        assert_eq!(set.len(), 4);
5716        // Ord (derive ordering: CrashPack < ReplayTrace < Trace < Other)
5717        assert!(HarnessAttachmentKind::CrashPack < HarnessAttachmentKind::ReplayTrace);
5718        assert!(HarnessAttachmentKind::ReplayTrace < HarnessAttachmentKind::Trace);
5719        assert!(HarnessAttachmentKind::Trace < HarnessAttachmentKind::Other);
5720        let mut sorted = [kinds[3], kinds[0], kinds[2], kinds[1]];
5721        sorted.sort();
5722        assert_eq!(sorted, kinds);
5723    }
5724
5725    #[test]
5726    fn harness_attachment_ref_debug_clone_eq_hash() {
5727        use std::collections::HashSet;
5728        let ref1 = HarnessAttachmentRef::crashpack("crash.bin");
5729        let ref2 = HarnessAttachmentRef::replay_trace("replay.bin");
5730        let ref3 = HarnessAttachmentRef::trace("trace.ndjson");
5731        assert_eq!(ref1.kind, HarnessAttachmentKind::CrashPack);
5732        assert_eq!(ref2.kind, HarnessAttachmentKind::ReplayTrace);
5733        assert_eq!(ref3.kind, HarnessAttachmentKind::Trace);
5734        let cloned = ref1.clone();
5735        assert_eq!(cloned, ref1);
5736        assert_ne!(ref1, ref2);
5737        let dbg = format!("{ref1:?}");
5738        assert!(dbg.contains("HarnessAttachmentRef"));
5739        let mut set = HashSet::new();
5740        set.insert(ref1.clone());
5741        set.insert(ref2);
5742        set.insert(ref1); // duplicate
5743        assert_eq!(set.len(), 2);
5744    }
5745
5746    #[test]
5747    fn chaos_config_summary_debug_clone_copy_partial_eq() {
5748        let summary = ChaosConfigSummary {
5749            seed: 42,
5750            cancel_probability: 0.1,
5751            delay_probability: 0.2,
5752            io_error_probability: 0.05,
5753            wakeup_storm_probability: 0.01,
5754            budget_exhaust_probability: 0.03,
5755        };
5756        let copied = summary;
5757        let cloned = summary;
5758        assert_eq!(copied, cloned);
5759        let dbg = format!("{summary:?}");
5760        assert!(dbg.contains("ChaosConfigSummary"));
5761    }
5762
5763    #[test]
5764    fn obligation_leak_debug_clone_eq_display() {
5765        let leak = ObligationLeak {
5766            obligation: ObligationId::new_for_test(1, 0),
5767            kind: ObligationKind::SendPermit,
5768            holder: TaskId::from_arena(crate::util::ArenaIndex::new(1, 0)),
5769            region: RegionId::new_for_test(0, 0),
5770        };
5771        let cloned = leak.clone();
5772        assert_eq!(cloned, leak);
5773        let dbg = format!("{leak:?}");
5774        assert!(dbg.contains("ObligationLeak"));
5775        let display = format!("{leak}");
5776        assert!(!display.is_empty());
5777    }
5778
5779    // ================================================================
5780    // CONFORMANCE TESTS: LabRuntime Deterministic Seed Reproduction
5781    // ================================================================
5782    //
5783    // Golden tests verifying the non-negotiable determinism invariants:
5784    // (1) Same seed produces byte-identical execution trace
5785    // (2) Virtual-time advances in same order across replays
5786    // (3) Scheduler lottery with same seed picks same tasks
5787    // (4) Chaos injection with same seed identical
5788    // (5) Cross-thread panic semantics preserved
5789    //
5790    // These conformance tests ensure LabRuntime provides reproducible
5791    // execution for debugging, testing, and formal verification.
5792
5793    /// CONFORMANCE: Same seed produces byte-identical execution trace.
5794    ///
5795    /// Verifies that identical configuration and program produce identical
5796    /// trace fingerprints, event counts, and schedule certificates.
5797    #[test]
5798    fn conformance_identical_seed_identical_trace() {
5799        init_test("conformance_identical_seed_identical_trace");
5800
5801        let seed = 42_u64;
5802        let config = LabConfig::new(seed).worker_count(2).max_steps(1000);
5803
5804        // Run same program with same seed twice
5805        let mut reports = Vec::new();
5806        for run_id in 0..2 {
5807            let mut runtime = LabRuntime::new(config.clone());
5808            let root = runtime.state.create_root_region(Budget::INFINITE);
5809
5810            // Create deterministic workload with multiple tasks
5811            for i in 0..5 {
5812                let (task_id, _handle) = runtime
5813                    .state
5814                    .create_task(root, Budget::INFINITE, async move {
5815                        // Simulate work with deterministic operations
5816                        for j in 0..10 {
5817                            futures_lite::future::yield_now().await;
5818                            if (i + j) % 3 == 0 {
5819                                let now =
5820                                    crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
5821                                crate::time::sleep(now, Duration::from_millis(1)).await;
5822                            }
5823                        }
5824                        i * 100 + run_id
5825                    })
5826                    .expect("create task");
5827                runtime.scheduler.lock().schedule(task_id, 0);
5828            }
5829
5830            let report = runtime.run_until_quiescent_with_report();
5831            reports.push(report);
5832        }
5833
5834        // Verify identical traces
5835        let report1 = &reports[0];
5836        let report2 = &reports[1];
5837
5838        crate::assert_with_log!(
5839            report1.seed == report2.seed,
5840            "seeds should be identical",
5841            report1.seed,
5842            report2.seed
5843        );
5844
5845        crate::assert_with_log!(
5846            report1.trace_fingerprint == report2.trace_fingerprint,
5847            "trace fingerprints should be identical",
5848            report1.trace_fingerprint,
5849            report2.trace_fingerprint
5850        );
5851
5852        crate::assert_with_log!(
5853            report1.trace_certificate.event_hash == report2.trace_certificate.event_hash,
5854            "event hashes should be identical",
5855            report1.trace_certificate.event_hash,
5856            report2.trace_certificate.event_hash
5857        );
5858
5859        crate::assert_with_log!(
5860            report1.trace_certificate.event_count == report2.trace_certificate.event_count,
5861            "event counts should be identical",
5862            report1.trace_certificate.event_count,
5863            report2.trace_certificate.event_count
5864        );
5865
5866        crate::assert_with_log!(
5867            report1.trace_certificate.schedule_hash == report2.trace_certificate.schedule_hash,
5868            "schedule hashes should be identical",
5869            report1.trace_certificate.schedule_hash,
5870            report2.trace_certificate.schedule_hash
5871        );
5872
5873        crate::test_complete!("conformance_identical_seed_identical_trace");
5874    }
5875
5876    /// CONFORMANCE: Virtual-time advances in same order across replays.
5877    ///
5878    /// Verifies that virtual time progression and auto-advancement
5879    /// behavior is deterministic across runs with the same seed.
5880    #[test]
5881    fn conformance_virtual_time_deterministic_advancement() {
5882        init_test("conformance_virtual_time_deterministic_advancement");
5883
5884        let config = LabConfig::new(123).worker_count(1);
5885
5886        crate::lab::assert_deterministic(config, |runtime| {
5887            let root = runtime.state.create_root_region(Budget::INFINITE);
5888            let initial_time = runtime.now();
5889
5890            // Create tasks that sleep for different durations to test time advancement
5891            let durations = [
5892                Duration::from_millis(10),
5893                Duration::from_millis(5),
5894                Duration::from_millis(15),
5895                Duration::from_millis(1),
5896            ];
5897
5898            for (i, duration) in durations.iter().enumerate() {
5899                let dur = *duration;
5900                let (task_id, _handle) = runtime
5901                    .state
5902                    .create_task(root, Budget::INFINITE, async move {
5903                        let now = crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
5904                        crate::time::sleep(now, dur).await;
5905                        i
5906                    })
5907                    .expect("create task");
5908                runtime.scheduler.lock().schedule(task_id, 0);
5909            }
5910
5911            // Use auto-advance to let virtual time progress deterministically
5912            let vtime_report = runtime.run_with_auto_advance();
5913
5914            // Verify time advanced
5915            crate::assert_with_log!(
5916                vtime_report.time_end > initial_time,
5917                "virtual time should have advanced",
5918                vtime_report.time_end,
5919                initial_time
5920            );
5921
5922            crate::assert_with_log!(
5923                vtime_report.auto_advances > 0,
5924                "should have auto-advanced virtual time",
5925                vtime_report.auto_advances,
5926                0
5927            );
5928
5929            crate::assert_with_log!(
5930                vtime_report.termination == AutoAdvanceTermination::Quiescent,
5931                "should reach quiescence",
5932                vtime_report.termination,
5933                AutoAdvanceTermination::Quiescent
5934            );
5935        });
5936
5937        crate::test_complete!("conformance_virtual_time_deterministic_advancement");
5938    }
5939
5940    /// CONFORMANCE: Scheduler lottery with same seed picks same tasks.
5941    ///
5942    /// Verifies that scheduler decisions (task selection, worker assignment)
5943    /// are deterministic given the same random seed.
5944    #[test]
5945    fn conformance_scheduler_deterministic_lottery() {
5946        init_test("conformance_scheduler_deterministic_lottery");
5947
5948        let config = LabConfig::new(456).worker_count(4);
5949
5950        let mut schedule_sequences = Vec::new();
5951
5952        // Run same workload multiple times to capture scheduler decisions
5953        for run in 0..2 {
5954            let mut runtime = LabRuntime::new(config.clone());
5955            let root = runtime.state.create_root_region(Budget::INFINITE);
5956            let mut task_order = Vec::new();
5957
5958            // Create many competing tasks to stress scheduler lottery
5959            for task_idx in 0..20 {
5960                let (task_id, _handle) = runtime
5961                    .state
5962                    .create_task(root, Budget::INFINITE, async move {
5963                        // Add some yield points to allow preemption
5964                        for _ in 0..3 {
5965                            futures_lite::future::yield_now().await;
5966                        }
5967                        task_idx
5968                    })
5969                    .expect("create task");
5970                runtime.scheduler.lock().schedule(task_id, 0);
5971            }
5972
5973            // Execute and capture schedule certificate
5974            runtime.run_until_quiescent();
5975            let cert = runtime.certificate();
5976            task_order.push((run, cert.decisions(), cert.hash()));
5977            schedule_sequences.push(task_order);
5978        }
5979
5980        // Verify scheduler made same decisions across runs
5981        let seq1 = &schedule_sequences[0];
5982        let seq2 = &schedule_sequences[1];
5983
5984        crate::assert_with_log!(
5985            seq1.len() == seq2.len(),
5986            "should have same number of scheduling decision points",
5987            seq1.len(),
5988            seq2.len()
5989        );
5990
5991        for (i, ((_run1, count1, hash1), (_run2, count2, hash2))) in
5992            seq1.iter().zip(seq2.iter()).enumerate()
5993        {
5994            crate::assert_with_log!(
5995                count1 == count2,
5996                &format!("decision count should be identical at point {}", i),
5997                count1,
5998                count2
5999            );
6000
6001            crate::assert_with_log!(
6002                hash1 == hash2,
6003                &format!("schedule hash should be identical at point {}", i),
6004                hash1,
6005                hash2
6006            );
6007        }
6008
6009        crate::test_complete!("conformance_scheduler_deterministic_lottery");
6010    }
6011
6012    /// CONFORMANCE: Chaos injection with same seed produces identical outcomes.
6013    ///
6014    /// Verifies that chaos injection (cancellation, delays, errors) is
6015    /// deterministically reproducible with the same chaos seed.
6016    #[test]
6017    fn conformance_chaos_injection_deterministic() {
6018        init_test("conformance_chaos_injection_deterministic");
6019
6020        let chaos_config = crate::lab::chaos::ChaosConfig::new(789)
6021            .with_cancel_probability(0.1)
6022            .with_delay_probability(0.05)
6023            .with_io_error_probability(0.02);
6024        let config = LabConfig::new(999).with_chaos(chaos_config);
6025
6026        crate::lab::assert_deterministic(config, |runtime| {
6027            let root = runtime.state.create_root_region(Budget::INFINITE);
6028
6029            // Create workload susceptible to chaos injection
6030            for i in 0..10 {
6031                let (task_id, _handle) = runtime
6032                    .state
6033                    .create_task(root, Budget::INFINITE, async move {
6034                        // Multiple poll points where chaos can be injected
6035                        for j in 0..20 {
6036                            futures_lite::future::yield_now().await;
6037                            if j % 5 == 0 {
6038                                let now =
6039                                    crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
6040                                crate::time::sleep(now, Duration::from_millis(1)).await;
6041                            }
6042                        }
6043                        i
6044                    })
6045                    .expect("create task");
6046                runtime.scheduler.lock().schedule(task_id, 0);
6047            }
6048
6049            runtime.run_until_quiescent();
6050
6051            // Verify chaos was actually applied
6052            let chaos_stats = runtime.chaos_stats();
6053            let total_decisions = chaos_stats.decision_points;
6054
6055            crate::assert_with_log!(
6056                total_decisions > 0,
6057                "chaos should have made some decisions",
6058                total_decisions,
6059                0
6060            );
6061        });
6062
6063        crate::test_complete!("conformance_chaos_injection_deterministic");
6064    }
6065
6066    /// CONFORMANCE: Cross-thread panic semantics are preserved deterministically.
6067    ///
6068    /// Verifies that panic propagation and cleanup across workers/regions
6069    /// follows the same deterministic pattern with identical seeds.
6070    #[test]
6071    fn conformance_panic_semantics_deterministic() {
6072        init_test("conformance_panic_semantics_deterministic");
6073
6074        let config = LabConfig::new(333)
6075            .worker_count(3)
6076            .panic_on_leak(false)
6077            .max_steps(10_000); // Fail diagnostically instead of hanging if panic cleanup regresses
6078
6079        crate::lab::assert_deterministic(config, |runtime| {
6080            let root = runtime.state.create_root_region(Budget::INFINITE);
6081
6082            // Create tasks where one will panic
6083            for i in 0..5 {
6084                let (task_id, _handle) = runtime
6085                    .state
6086                    .create_task(root, Budget::INFINITE, async move {
6087                        // Task 2 will deterministically panic
6088                        assert!(i != 2, "deterministic panic in task {}", i);
6089
6090                        // Other tasks continue working
6091                        for _j in 0..10 {
6092                            futures_lite::future::yield_now().await;
6093                        }
6094                        i * 10
6095                    })
6096                    .expect("create task");
6097                runtime.scheduler.lock().schedule(task_id, 0);
6098            }
6099
6100            // The lab trace no longer exposes a dedicated `TaskPanicked` data
6101            // variant. Drive the panic-bearing run to quiescence once and
6102            // inspect that same run's report rather than a second already-idle
6103            // pass.
6104            let report = runtime.run_until_quiescent_with_report();
6105            let trace_events = runtime.trace().snapshot();
6106            let complete_events = trace_events
6107                .iter()
6108                .filter(|event| event.kind == TraceEventKind::Complete)
6109                .count();
6110
6111            crate::assert_with_log!(
6112                complete_events > 0,
6113                "should have recorded task completion activity",
6114                complete_events,
6115                0
6116            );
6117
6118            // Verify deterministic cleanup occurred
6119            crate::assert_with_log!(
6120                report.quiescent,
6121                "runtime should reach quiescence despite panic",
6122                report.quiescent,
6123                false
6124            );
6125        });
6126
6127        crate::test_complete!("conformance_panic_semantics_deterministic");
6128    }
6129
6130    /// CONFORMANCE: Comprehensive multi-run determinism verification.
6131    ///
6132    /// Combines all previous conformance aspects into a stress test
6133    /// that verifies determinism across many execution runs.
6134    #[test]
6135    fn conformance_comprehensive_determinism_stress() {
6136        init_test("conformance_comprehensive_determinism_stress");
6137
6138        let chaos_config = crate::lab::chaos::ChaosConfig::new(555)
6139            .with_cancel_probability(0.05)
6140            .with_delay_probability(0.03);
6141        let config = LabConfig::new(777)
6142            .worker_count(4)
6143            .with_chaos(chaos_config)
6144            .max_steps(5000);
6145
6146        // Use assert_deterministic_multi for extra confidence
6147        crate::lab::assert_deterministic_multi(&config, 3, |runtime| {
6148            let root = runtime.state.create_root_region(Budget::INFINITE);
6149
6150            // Complex workload mixing all runtime features
6151            for i in 0..15 {
6152                let (task_id, _handle) = runtime
6153                    .state
6154                    .create_task(root, Budget::INFINITE, async move {
6155                        // Mix of operations: yields, sleeps, work
6156                        for j in 0..30 {
6157                            match (i + j) % 4 {
6158                                0 => futures_lite::future::yield_now().await,
6159                                1 => {
6160                                    let now =
6161                                        crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
6162                                    crate::time::sleep(now, Duration::from_millis(j as u64 % 5))
6163                                        .await;
6164                                }
6165                                2 => {
6166                                    // Simulate CPU work
6167                                    let mut sum = 0_u64;
6168                                    for k in 0..100 {
6169                                        sum = sum.wrapping_add(k);
6170                                    }
6171                                    let _ = sum;
6172                                }
6173                                _ => futures_lite::future::yield_now().await,
6174                            }
6175                        }
6176                        i * 1000 + 42
6177                    })
6178                    .expect("create task");
6179                runtime.scheduler.lock().schedule(task_id, 0);
6180            }
6181
6182            // Use auto-advance for time progression
6183            let vtime_report = runtime.run_with_auto_advance();
6184
6185            crate::assert_with_log!(
6186                vtime_report.termination == AutoAdvanceTermination::Quiescent,
6187                "comprehensive workload should reach quiescence",
6188                vtime_report.termination,
6189                AutoAdvanceTermination::Quiescent
6190            );
6191        });
6192
6193        crate::test_complete!("conformance_comprehensive_determinism_stress");
6194    }
6195
6196    #[test]
6197    #[allow(clippy::literal_string_with_formatting_args)]
6198    fn non_test_lab_runtime_paths_do_not_use_stray_stdout_debug_prints() {
6199        let source =
6200            std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(file!()))
6201                .expect("lab runtime source must be readable");
6202
6203        for message in [
6204            "LabScheduler already scheduled {task:?}",
6205            "LabScheduler scheduling {task:?}",
6206            "Executing {:?} at step {}",
6207            "rng_value = {}, worker_hint = {}",
6208        ] {
6209            let stdout_call = format!("print{}!(\"{message}\"", "ln");
6210            assert!(
6211                !source.contains(&stdout_call),
6212                "non-test LabRuntime debug print regressed: {message}"
6213            );
6214        }
6215    }
6216}