1use super::config::LabConfig;
10use super::oracle::OracleSuite;
11use crate::lab::chaos::{ChaosRng, ChaosStats};
12use crate::record::ObligationKind;
13use crate::record::task::TaskState;
14use crate::runtime::RuntimeState;
15use crate::runtime::config::ObligationLeakResponse;
16use crate::runtime::deadline_monitor::{
17 DeadlineMonitor, DeadlineWarning, MonitorConfig, default_warning_handler,
18};
19use crate::runtime::reactor::LabReactor;
20use crate::runtime::scheduler::{DispatchLane, ScheduleCertificate};
21use crate::time::VirtualClock;
22use crate::trace::TraceBufferHandle;
23use crate::trace::crashpack::{
24 CrashPack, CrashPackConfig, FailureInfo, FailureOutcome, ReplayCommand, artifact_filename,
25};
26use crate::trace::event::TraceEventKind;
27use crate::trace::recorder::TraceRecorder;
28use crate::trace::replay::{ReplayEvent, ReplayTrace, TraceMetadata};
29use crate::trace::scoring::seed_fingerprint;
30use crate::trace::{TraceData, TraceEvent, check_refinement_firewall};
31use crate::trace::{canonicalize::trace_fingerprint, certificate::TraceCertificate};
32use crate::types::Time;
33use crate::types::{ObligationId, RegionId, TaskId};
34use crate::util::det_hash::{DetHashMap, DetHashSet};
35use crate::util::{DetEntropy, DetRng};
36use parking_lot::Mutex;
37use std::fmt;
38use std::sync::Arc;
39use std::task::{Context, Poll, Waker};
40use std::time::Duration;
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub struct LabTraceCertificateSummary {
45 pub event_hash: u64,
47 pub event_count: u64,
49 pub schedule_hash: u64,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55pub enum AutoAdvanceTermination {
56 Quiescent,
59 StepLimitReached,
61 StuckBailout,
64}
65
66impl fmt::Display for AutoAdvanceTermination {
67 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
68 match self {
69 Self::Quiescent => f.write_str("quiescent"),
70 Self::StepLimitReached => f.write_str("step-limit-reached"),
71 Self::StuckBailout => f.write_str("stuck-bailout"),
72 }
73 }
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub struct VirtualTimeReport {
81 pub steps: u64,
83 pub auto_advances: u64,
86 pub total_wakeups: u64,
88 pub time_start: Time,
90 pub time_end: Time,
92 pub virtual_elapsed_nanos: u64,
94 pub termination: AutoAdvanceTermination,
96}
97
98impl VirtualTimeReport {
99 #[must_use]
101 pub const fn virtual_elapsed_ms(&self) -> u64 {
102 self.virtual_elapsed_nanos / 1_000_000
103 }
104
105 #[must_use]
107 pub const fn virtual_elapsed_secs(&self) -> u64 {
108 self.virtual_elapsed_nanos / 1_000_000_000
109 }
110}
111
112#[derive(Debug, Clone)]
118pub struct LabRunReport {
119 pub seed: u64,
121 pub steps_delta: u64,
123 pub steps_total: u64,
125 pub quiescent: bool,
127 pub now_nanos: u64,
129 pub trace_len: usize,
131 pub trace_fingerprint: u64,
133 pub trace_certificate: LabTraceCertificateSummary,
135 pub oracle_report: crate::lab::oracle::OracleReport,
137 pub invariant_violations: Vec<String>,
142 pub temporal_invariant_failures: Vec<String>,
144 pub temporal_counterexample_prefix_len: Option<usize>,
146 pub refinement_firewall_rule_id: Option<String>,
148 pub refinement_firewall_event_index: Option<usize>,
150 pub refinement_firewall_event_seq: Option<u64>,
152 pub refinement_counterexample_prefix_len: Option<usize>,
154 pub refinement_firewall_skipped_due_to_trace_truncation: bool,
156}
157
158impl LabRunReport {
159 #[must_use]
161 pub fn to_json(&self) -> serde_json::Value {
162 use serde_json::json;
163
164 json!({
165 "seed": self.seed,
166 "steps_delta": self.steps_delta,
167 "steps_total": self.steps_total,
168 "quiescent": self.quiescent,
169 "now_nanos": self.now_nanos,
170 "trace": {
171 "len": self.trace_len,
172 "fingerprint": self.trace_fingerprint,
173 "certificate": {
174 "event_hash": self.trace_certificate.event_hash,
175 "event_count": self.trace_certificate.event_count,
176 "schedule_hash": self.trace_certificate.schedule_hash,
177 }
178 },
179 "oracles": self.oracle_report.to_json(),
180 "invariants": self.invariant_violations,
181 "temporal": {
182 "failed_invariants": self.temporal_invariant_failures,
183 "counterexample_prefix_len": self.temporal_counterexample_prefix_len,
184 },
185 "refinement_firewall": {
186 "rule_id": self.refinement_firewall_rule_id,
187 "event_index": self.refinement_firewall_event_index,
188 "event_seq": self.refinement_firewall_event_seq,
189 "counterexample_prefix_len": self.refinement_counterexample_prefix_len,
190 "skipped_due_to_trace_truncation": self.refinement_firewall_skipped_due_to_trace_truncation,
191 },
192 })
193 }
194
195 #[must_use]
203 pub fn export_tla(
204 &self,
205 trace_events: &[crate::trace::TraceEvent],
206 module_name: &str,
207 ) -> Option<crate::trace::tla_export::TlaModule> {
208 if trace_events.is_empty() {
209 return None;
210 }
211 let exporter = crate::trace::tla_export::TlaExporter::from_trace(trace_events);
212 if exporter.snapshot_count() == 0 {
213 return None;
214 }
215 Some(exporter.export_behavior(module_name))
216 }
217}
218
219const TEMPORAL_ORACLE_INVARIANTS: &[&str] = &[
220 "task_leak",
221 "obligation_leak",
222 "quiescence",
223 "cancellation_protocol",
224 "loser_drain",
225 "region_tree",
226 "deadline_monotone",
227 #[cfg(feature = "messaging-fabric")]
228 "fabric_publish",
229 #[cfg(feature = "messaging-fabric")]
230 "fabric_reply",
231 #[cfg(feature = "messaging-fabric")]
232 "fabric_quiescence",
233 #[cfg(feature = "messaging-fabric")]
234 "fabric_redelivery",
235];
236
237#[derive(Debug, Clone, PartialEq)]
246pub struct LabConfigSummary {
247 pub seed: u64,
249 pub entropy_seed: u64,
251 pub worker_count: usize,
253 pub panic_on_obligation_leak: bool,
255 pub trace_capacity: usize,
257 pub futurelock_max_idle_steps: u64,
259 pub panic_on_futurelock: bool,
261 pub max_steps: Option<u64>,
263 pub chaos: Option<ChaosConfigSummary>,
265 pub replay_recording_enabled: bool,
267}
268
269impl LabConfigSummary {
270 #[must_use]
272 pub fn from_config(config: &LabConfig) -> Self {
273 Self {
274 seed: config.seed,
275 entropy_seed: config.entropy_seed,
276 worker_count: config.worker_count,
277 panic_on_obligation_leak: config.panic_on_obligation_leak,
278 trace_capacity: config.trace_capacity,
279 futurelock_max_idle_steps: config.futurelock_max_idle_steps,
280 panic_on_futurelock: config.panic_on_futurelock,
281 max_steps: config.max_steps,
282 chaos: config.chaos.as_ref().map(ChaosConfigSummary::from_config),
283 replay_recording_enabled: config.replay_recording.is_some(),
284 }
285 }
286
287 #[must_use]
292 pub fn config_hash(&self) -> u64 {
293 use std::hash::{Hash, Hasher};
294 let mut h = crate::util::DetHasher::default();
297 self.seed.hash(&mut h);
298 self.entropy_seed.hash(&mut h);
299 self.worker_count.hash(&mut h);
300 self.panic_on_obligation_leak.hash(&mut h);
301 self.trace_capacity.hash(&mut h);
302 self.futurelock_max_idle_steps.hash(&mut h);
303 self.panic_on_futurelock.hash(&mut h);
304 self.max_steps.hash(&mut h);
305 self.replay_recording_enabled.hash(&mut h);
306 if let Some(ref c) = self.chaos {
307 1u8.hash(&mut h);
308 c.seed.hash(&mut h);
309 c.cancel_probability.to_bits().hash(&mut h);
310 c.delay_probability.to_bits().hash(&mut h);
311 c.io_error_probability.to_bits().hash(&mut h);
312 c.wakeup_storm_probability.to_bits().hash(&mut h);
313 c.budget_exhaust_probability.to_bits().hash(&mut h);
314 } else {
315 0u8.hash(&mut h);
316 }
317 h.finish()
318 }
319
320 #[must_use]
322 pub fn to_json(&self) -> serde_json::Value {
323 use serde_json::json;
324
325 json!({
326 "seed": self.seed,
327 "entropy_seed": self.entropy_seed,
328 "worker_count": self.worker_count,
329 "panic_on_obligation_leak": self.panic_on_obligation_leak,
330 "trace_capacity": self.trace_capacity,
331 "futurelock_max_idle_steps": self.futurelock_max_idle_steps,
332 "panic_on_futurelock": self.panic_on_futurelock,
333 "max_steps": self.max_steps,
334 "chaos": self.chaos.as_ref().map(ChaosConfigSummary::to_json),
335 "replay_recording_enabled": self.replay_recording_enabled,
336 })
337 }
338}
339
340#[derive(Debug, Clone, Copy, PartialEq)]
342pub struct ChaosConfigSummary {
343 pub seed: u64,
345 pub cancel_probability: f64,
347 pub delay_probability: f64,
349 pub io_error_probability: f64,
351 pub wakeup_storm_probability: f64,
353 pub budget_exhaust_probability: f64,
355}
356
357impl ChaosConfigSummary {
358 #[must_use]
360 pub fn from_config(config: &crate::lab::chaos::ChaosConfig) -> Self {
361 Self {
362 seed: config.seed,
363 cancel_probability: config.cancel_probability,
364 delay_probability: config.delay_probability,
365 io_error_probability: config.io_error_probability,
366 wakeup_storm_probability: config.wakeup_storm_probability,
367 budget_exhaust_probability: config.budget_exhaust_probability,
368 }
369 }
370
371 #[must_use]
373 pub fn to_json(&self) -> serde_json::Value {
374 use serde_json::json;
375
376 json!({
377 "seed": self.seed,
378 "cancel_probability": self.cancel_probability,
379 "delay_probability": self.delay_probability,
380 "io_error_probability": self.io_error_probability,
381 "wakeup_storm_probability": self.wakeup_storm_probability,
382 "budget_exhaust_probability": self.budget_exhaust_probability,
383 })
384 }
385}
386
387#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
389pub enum HarnessAttachmentKind {
390 CrashPack,
392 ReplayTrace,
394 Trace,
396 Other,
398}
399
400impl fmt::Display for HarnessAttachmentKind {
401 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
402 match self {
403 Self::CrashPack => write!(f, "crashpack"),
404 Self::ReplayTrace => write!(f, "replay_trace"),
405 Self::Trace => write!(f, "trace"),
406 Self::Other => write!(f, "other"),
407 }
408 }
409}
410
411#[derive(Debug, Clone, PartialEq, Eq, Hash)]
416pub struct HarnessAttachmentRef {
417 pub kind: HarnessAttachmentKind,
419 pub path: String,
421}
422
423impl HarnessAttachmentRef {
424 #[must_use]
426 pub fn crashpack(path: impl Into<String>) -> Self {
427 Self {
428 kind: HarnessAttachmentKind::CrashPack,
429 path: path.into(),
430 }
431 }
432
433 #[must_use]
435 pub fn replay_trace(path: impl Into<String>) -> Self {
436 Self {
437 kind: HarnessAttachmentKind::ReplayTrace,
438 path: path.into(),
439 }
440 }
441
442 #[must_use]
444 pub fn trace(path: impl Into<String>) -> Self {
445 Self {
446 kind: HarnessAttachmentKind::Trace,
447 path: path.into(),
448 }
449 }
450
451 #[must_use]
453 pub fn to_json(&self) -> serde_json::Value {
454 use serde_json::json;
455
456 json!({
457 "kind": self.kind.to_string(),
458 "path": self.path,
459 })
460 }
461}
462
463#[derive(Debug, Clone, PartialEq, Eq)]
465pub struct CrashpackLink {
466 pub path: String,
468 pub id: String,
470 pub fingerprint: u64,
472 pub replay: ReplayCommand,
474}
475
476impl CrashpackLink {
477 #[must_use]
479 pub fn to_json(&self) -> serde_json::Value {
480 use serde_json::json;
481
482 json!({
483 "path": self.path,
484 "id": self.id,
485 "fingerprint": self.fingerprint,
486 "replay": self.replay,
487 })
488 }
489}
490
491#[derive(Debug, Clone)]
498pub struct SporkHarnessReport {
499 pub schema_version: u32,
501 pub app: String,
503 pub config: LabConfigSummary,
505 pub run: LabRunReport,
507 pub attachments: Vec<HarnessAttachmentRef>,
509}
510
511impl SporkHarnessReport {
512 pub const SCHEMA_VERSION: u32 = 3;
529
530 #[must_use]
532 pub fn new(
533 app: impl Into<String>,
534 config: &LabConfig,
535 run: LabRunReport,
536 attachments: Vec<HarnessAttachmentRef>,
537 ) -> Self {
538 Self {
539 schema_version: Self::SCHEMA_VERSION,
540 app: app.into(),
541 config: LabConfigSummary::from_config(config),
542 run,
543 attachments,
544 }
545 }
546
547 #[must_use]
553 pub fn passed(&self) -> bool {
554 self.run.oracle_report.all_passed() && self.run.invariant_violations.is_empty()
555 }
556
557 #[must_use]
559 pub fn trace_fingerprint(&self) -> u64 {
560 self.run.trace_fingerprint
561 }
562
563 #[must_use]
565 pub fn seed(&self) -> u64 {
566 self.run.seed
567 }
568
569 #[must_use]
571 pub fn config_hash(&self) -> u64 {
572 self.config.config_hash()
573 }
574
575 #[must_use]
577 pub fn crashpack_path(&self) -> Option<&str> {
578 self.attachments
579 .iter()
580 .find(|a| a.kind == HarnessAttachmentKind::CrashPack)
581 .map(|a| a.path.as_str())
582 }
583
584 #[must_use]
586 pub fn crashpack_link(&self) -> Option<CrashpackLink> {
587 let path = self.crashpack_path()?.to_string();
588 let crash_config = CrashPackConfig {
589 seed: self.seed(),
590 config_hash: self.config_hash(),
591 worker_count: self.config.worker_count,
592 max_steps: self.config.max_steps,
593 commit_hash: None,
594 };
595 let replay = ReplayCommand::from_config(&crash_config, Some(path.as_str()));
596 Some(CrashpackLink {
597 id: format!(
598 "crashpack-{seed:016x}-{fingerprint:016x}",
599 seed = self.seed(),
600 fingerprint = self.trace_fingerprint()
601 ),
602 fingerprint: self.trace_fingerprint(),
603 path,
604 replay,
605 })
606 }
607
608 #[must_use]
610 pub fn oracle_failures(&self) -> Vec<String> {
611 self.run
612 .oracle_report
613 .failures()
614 .iter()
615 .map(|e| {
616 let desc = e
617 .violation
618 .as_ref()
619 .map_or_else(String::new, |v| format!(": {v}"));
620 format!("{}{desc}", e.invariant)
621 })
622 .collect()
623 }
624
625 #[must_use]
629 pub fn summary_line(&self) -> String {
630 let verdict = if self.passed() { "PASS" } else { "FAIL" };
631 let oracle = &self.run.oracle_report;
632 format!(
633 "[{verdict}] app=\"{}\" seed={} fingerprint={} oracles={}/{} invariant_violations={}",
634 self.app,
635 self.run.seed,
636 self.run.trace_fingerprint,
637 oracle.passed,
638 oracle.total,
639 self.run.invariant_violations.len(),
640 )
641 }
642
643 #[must_use]
666 pub fn to_json(&self) -> serde_json::Value {
667 use serde_json::json;
668
669 let mut attachments = self.attachments.clone();
671 attachments.sort_by(|a, b| (a.kind, &a.path).cmp(&(b.kind, &b.path)));
672
673 json!({
674 "schema_version": self.schema_version,
675 "verdict": if self.passed() { "pass" } else { "fail" },
676 "app": { "name": self.app },
677 "lab": {
678 "config": self.config.to_json(),
679 "config_hash": self.config.config_hash(),
680 },
681 "fingerprints": {
682 "trace": self.run.trace_fingerprint,
683 "event_hash": self.run.trace_certificate.event_hash,
684 "event_count": self.run.trace_certificate.event_count,
685 "schedule_hash": self.run.trace_certificate.schedule_hash,
686 },
687 "run": self.run.to_json(),
688 "crashpack": self.crashpack_link().map(|link| link.to_json()),
689 "attachments": attachments.iter().map(HarnessAttachmentRef::to_json).collect::<Vec<_>>(),
690 })
691 }
692}
693
694#[derive(Debug)]
702pub struct LabRuntime {
703 pub state: RuntimeState,
705 lab_reactor: Arc<LabReactor>,
707 seen_io_tokens: DetHashSet<usize>,
709 pub scheduler: Arc<Mutex<LabScheduler>>,
711 config: LabConfig,
713 rng: DetRng,
715 virtual_time: Time,
717 virtual_clock: Arc<VirtualClock>,
719 steps: u64,
721 chaos_rng: Option<ChaosRng>,
723 chaos_stats: ChaosStats,
725 seen_reactor_chaos_stats: ChaosStats,
727 replay_recorder: TraceRecorder,
729 deadline_monitor: Option<DeadlineMonitor>,
731 pub oracles: OracleSuite,
733 certificate: ScheduleCertificate,
735}
736
737impl LabRuntime {
738 #[must_use]
740 pub fn new(config: LabConfig) -> Self {
741 let rng = config.rng();
742 let chaos_rng = config.chaos.as_ref().map(ChaosRng::from_config);
743 let lab_reactor = config.chaos.as_ref().map_or_else(
744 || Arc::new(LabReactor::new()),
745 |chaos| Arc::new(LabReactor::with_chaos(chaos.clone())),
746 );
747 let mut state = RuntimeState::with_reactor(lab_reactor.clone());
748 state.set_logical_clock_mode(crate::trace::distributed::LogicalClockMode::Lamport);
749 state.set_obligation_leak_response(if config.panic_on_obligation_leak {
750 ObligationLeakResponse::Panic
751 } else {
752 ObligationLeakResponse::Log
753 });
754 let virtual_clock = Arc::new(VirtualClock::starting_at(Time::ZERO));
755 state.set_timer_driver(crate::time::TimerDriverHandle::with_virtual_clock(
756 virtual_clock.clone(),
757 ));
758 state.set_entropy_source(Arc::new(DetEntropy::new(config.entropy_seed)));
759 state.trace = TraceBufferHandle::new(config.trace_capacity);
760
761 let mut replay_recorder = if let Some(ref rec_config) = config.replay_recording {
763 TraceRecorder::with_config(TraceMetadata::new(config.seed), rec_config.clone())
764 } else {
765 TraceRecorder::disabled()
766 };
767
768 replay_recorder.record_rng_seed(config.seed);
770
771 crate::tracing_compat::info!("virtual clock initialized: start_time_ms=0");
772
773 Self {
774 state,
775 lab_reactor,
776 seen_io_tokens: DetHashSet::default(),
777 scheduler: Arc::new(Mutex::new(LabScheduler::new(config.worker_count))),
778 config,
779 rng,
780 virtual_time: Time::ZERO,
781 virtual_clock,
782 steps: 0,
783 chaos_rng,
784 chaos_stats: ChaosStats::new(),
785 seen_reactor_chaos_stats: ChaosStats::new(),
786 replay_recorder,
787 deadline_monitor: None,
788 oracles: OracleSuite::new(),
789 certificate: ScheduleCertificate::new(),
790 }
791 }
792
793 #[must_use]
795 pub fn with_seed(seed: u64) -> Self {
796 Self::new(LabConfig::new(seed))
797 }
798
799 #[must_use]
801 pub const fn now(&self) -> Time {
802 self.virtual_time
803 }
804
805 #[must_use]
807 pub const fn steps(&self) -> u64 {
808 self.steps
809 }
810
811 #[must_use]
813 pub const fn config(&self) -> &LabConfig {
814 &self.config
815 }
816
817 #[must_use]
819 pub fn lab_reactor(&self) -> &Arc<LabReactor> {
820 &self.lab_reactor
821 }
822
823 #[must_use]
825 pub fn trace(&self) -> &TraceBufferHandle {
826 &self.state.trace
827 }
828
829 #[must_use]
831 pub fn detected_races(&self) -> crate::trace::dpor::RaceReport {
832 crate::trace::dpor::detect_hb_races(&self.state.trace.snapshot())
833 }
834
835 #[must_use]
837 pub fn chaos_stats(&self) -> &ChaosStats {
838 &self.chaos_stats
839 }
840
841 #[must_use]
843 pub fn certificate(&self) -> &ScheduleCertificate {
844 &self.certificate
845 }
846
847 #[must_use]
849 pub fn has_replay_recording(&self) -> bool {
850 self.replay_recorder.is_enabled()
851 }
852
853 #[must_use]
855 pub fn replay_recorder(&self) -> &TraceRecorder {
856 &self.replay_recorder
857 }
858
859 pub fn take_replay_trace(&mut self) -> Option<ReplayTrace> {
863 self.replay_recorder.take()
864 }
865
866 pub fn finish_replay_trace(&mut self) -> Option<ReplayTrace> {
870 let recorder = std::mem::replace(&mut self.replay_recorder, TraceRecorder::disabled());
872 recorder.finish()
873 }
874
875 #[must_use]
877 pub fn has_chaos(&self) -> bool {
878 self.chaos_rng.is_some() && self.config.has_chaos()
879 }
880
881 #[must_use]
883 pub fn is_quiescent(&self) -> bool {
884 self.state.is_quiescent()
885 }
886
887 pub fn advance_time(&mut self, nanos: u64) {
889 let from = self.virtual_time;
890 self.virtual_time = self.virtual_time.saturating_add_nanos(nanos);
891 self.state.now = self.virtual_time;
892 self.virtual_clock.advance(nanos);
893 self.lab_reactor.advance_time(Duration::from_nanos(nanos));
894 self.replay_recorder
896 .record_time_advanced(from, self.virtual_time);
897
898 crate::tracing_compat::debug!(
899 "virtual clock advanced: delta_ms={}, new_time_ms={}",
900 nanos / 1_000_000,
901 self.virtual_time.as_nanos() / 1_000_000
902 );
903 }
904
905 pub fn advance_time_to(&mut self, time: Time) {
910 if time > self.virtual_time {
911 let from = self.virtual_time;
912 self.virtual_time = time;
913 self.state.now = self.virtual_time;
914 self.virtual_clock.advance_to(time);
915 self.lab_reactor.advance_time_to(time);
916 self.replay_recorder
918 .record_time_advanced(from, self.virtual_time);
919
920 crate::tracing_compat::debug!(
921 "virtual clock advanced: delta_ms={}, new_time_ms={}",
922 (time.as_nanos() - from.as_nanos()) / 1_000_000,
923 time.as_nanos() / 1_000_000
924 );
925 } else if time < self.virtual_time {
926 crate::tracing_compat::error!(
927 "virtual clock attempt to go backward: current_ms={}, requested_ms={}",
928 self.virtual_time.as_nanos() / 1_000_000,
929 time.as_nanos() / 1_000_000
930 );
931 }
932 }
933
934 pub fn advance_to_next_timer(&mut self) -> usize {
944 let next = self
945 .state
946 .timer_driver_handle()
947 .and_then(|h| h.next_deadline());
948
949 let Some(deadline) = next else {
950 return 0;
951 };
952
953 if deadline <= self.virtual_time {
954 return self
956 .state
957 .timer_driver_handle()
958 .map_or(0, |h| h.process_timers());
959 }
960
961 let delta_nanos = deadline.as_nanos() - self.virtual_time.as_nanos();
962 self.advance_time(delta_nanos);
963
964 let wakeups = self
965 .state
966 .timer_driver_handle()
967 .map_or(0, |h| h.process_timers());
968
969 crate::tracing_compat::debug!(
970 "virtual clock auto-advance: reason=all_tasks_blocked, \
971 next_wakeup_ms={}, delta_ms={}, wakeup_count={}",
972 deadline.as_nanos() / 1_000_000,
973 delta_nanos / 1_000_000,
974 wakeups
975 );
976
977 wakeups
978 }
979
980 #[must_use]
982 pub fn next_timer_deadline(&self) -> Option<Time> {
983 self.state
984 .timer_driver_handle()
985 .and_then(|h| h.next_deadline())
986 }
987
988 fn next_reactor_deadline(&self) -> Option<Time> {
989 self.state
990 .io_driver_handle()
991 .and_then(|_| self.lab_reactor.next_event_time())
992 }
993
994 fn next_auto_advance_deadline(&self) -> Option<Time> {
995 match (self.next_timer_deadline(), self.next_reactor_deadline()) {
996 (Some(timer), Some(reactor)) => Some(timer.min(reactor)),
997 (Some(timer), None) => Some(timer),
998 (None, Some(reactor)) => Some(reactor),
999 (None, None) => None,
1000 }
1001 }
1002
1003 fn pump_due_system_events(&mut self) -> usize {
1004 let wakeups = self
1005 .state
1006 .timer_driver_handle()
1007 .map_or(0, |h| h.process_timers());
1008 self.poll_io();
1009 self.schedule_async_finalizers();
1010 self.check_deadline_monitor();
1011 wakeups
1012 }
1013
1014 #[must_use]
1016 pub fn pending_timer_count(&self) -> usize {
1017 self.state
1018 .timer_driver_handle()
1019 .map_or(0, |h| h.pending_count())
1020 }
1021
1022 pub fn run_with_auto_advance(&mut self) -> VirtualTimeReport {
1037 let start_steps = self.steps;
1038 let mut auto_advances: u64 = 0;
1039 let mut total_wakeups: u64 = 0;
1040 let mut stuck_counter: u32 = 0;
1041 let start_time = self.virtual_time;
1042
1043 let termination = loop {
1044 if let Some(max) = self.config.max_steps {
1046 if self.steps >= max {
1047 break AutoAdvanceTermination::StepLimitReached;
1048 }
1049 }
1050
1051 let is_empty = self.scheduler.lock().is_empty();
1053 if !is_empty {
1054 stuck_counter = 0;
1055 self.step();
1056 continue;
1057 }
1058
1059 if let Some(deadline) = self.next_auto_advance_deadline() {
1061 if deadline > self.virtual_time {
1062 self.advance_time_to(deadline);
1063 let wakeups = self
1064 .state
1065 .timer_driver_handle()
1066 .map_or(0, |h| h.process_timers());
1067 auto_advances = auto_advances.saturating_add(1);
1068 total_wakeups = total_wakeups.saturating_add(wakeups as u64);
1069 continue;
1070 }
1071 total_wakeups = total_wakeups.saturating_add(self.pump_due_system_events() as u64);
1073 continue;
1074 }
1075
1076 if self.is_quiescent() {
1078 break AutoAdvanceTermination::Quiescent;
1079 }
1080
1081 stuck_counter = stuck_counter.saturating_add(1);
1084 if stuck_counter > 1000 {
1085 break AutoAdvanceTermination::StuckBailout;
1086 }
1087 self.step();
1088 };
1089
1090 VirtualTimeReport {
1091 steps: self.steps - start_steps,
1092 auto_advances,
1093 total_wakeups,
1094 time_start: start_time,
1095 time_end: self.virtual_time,
1096 virtual_elapsed_nanos: self.virtual_time.as_nanos() - start_time.as_nanos(),
1097 termination,
1098 }
1099 }
1100
1101 pub fn pause_clock(&self) {
1110 self.virtual_clock.pause();
1111 crate::tracing_compat::info!(
1112 "virtual clock paused at time_ms={}",
1113 self.virtual_time.as_nanos() / 1_000_000
1114 );
1115 }
1116
1117 pub fn resume_clock(&self) {
1119 self.virtual_clock.resume();
1120 crate::tracing_compat::info!(
1121 "virtual clock resumed at time_ms={}",
1122 self.virtual_time.as_nanos() / 1_000_000
1123 );
1124 }
1125
1126 #[must_use]
1128 pub fn is_clock_paused(&self) -> bool {
1129 self.virtual_clock.is_paused()
1130 }
1131
1132 #[allow(clippy::no_effect_underscore_binding)]
1137 pub fn inject_clock_skew(&mut self, skew_nanos: u64) {
1138 let old_nanos = self.virtual_time.as_nanos();
1140 self.advance_time(skew_nanos);
1141
1142 crate::tracing_compat::warn!(
1143 "virtual clock jump detected: old_time_ms={}, new_time_ms={}, jump_ms={} \
1144 -- may affect lease/timeout correctness",
1145 old_nanos / 1_000_000,
1146 self.virtual_time.as_nanos() / 1_000_000,
1147 skew_nanos / 1_000_000
1148 );
1149 #[cfg(not(feature = "tracing-integration"))]
1150 let _ = old_nanos;
1151 }
1152
1153 pub fn run_until_quiescent(&mut self) -> u64 {
1157 let start_steps = self.steps;
1158
1159 while !self.is_quiescent() {
1160 if let Some(max) = self.config.max_steps {
1161 if self.steps >= max {
1162 break;
1163 }
1164 }
1165 self.step();
1166 }
1167
1168 self.steps - start_steps
1169 }
1170
1171 pub fn run_until_idle(&mut self) -> u64 {
1181 let start_steps = self.steps;
1182
1183 loop {
1184 if let Some(max) = self.config.max_steps {
1185 if self.steps >= max {
1186 break;
1187 }
1188 }
1189
1190 let is_empty = self.scheduler.lock().is_empty();
1191 if is_empty {
1192 break;
1193 }
1194
1195 self.step();
1196 }
1197
1198 self.steps - start_steps
1199 }
1200
1201 #[must_use]
1203 pub fn run_until_quiescent_with_report(&mut self) -> LabRunReport {
1204 let steps_delta = self.run_until_quiescent();
1205 self.report_with_steps_delta(steps_delta)
1206 }
1207
1208 #[must_use]
1212 pub fn report(&mut self) -> LabRunReport {
1213 self.report_with_steps_delta(0)
1214 }
1215
1216 #[must_use]
1218 pub fn run_until_quiescent_spork_report(
1219 &mut self,
1220 app: impl Into<String>,
1221 attachments: Vec<HarnessAttachmentRef>,
1222 ) -> SporkHarnessReport {
1223 let run = self.run_until_quiescent_with_report();
1224 self.build_spork_report(app.into(), run, attachments)
1225 }
1226
1227 #[must_use]
1231 pub fn spork_report(
1232 &mut self,
1233 app: impl Into<String>,
1234 attachments: Vec<HarnessAttachmentRef>,
1235 ) -> SporkHarnessReport {
1236 let run = self.report();
1237 self.build_spork_report(app.into(), run, attachments)
1238 }
1239
1240 fn build_spork_report(
1241 &self,
1242 app: String,
1243 run: LabRunReport,
1244 mut attachments: Vec<HarnessAttachmentRef>,
1245 ) -> SporkHarnessReport {
1246 if let Some(auto_crashpack) = self.auto_crashpack_attachment(&run, &attachments) {
1247 attachments.push(auto_crashpack);
1248 }
1249 SporkHarnessReport::new(app, &self.config, run, attachments)
1250 }
1251
1252 fn auto_crashpack_attachment(
1253 &self,
1254 run: &LabRunReport,
1255 attachments: &[HarnessAttachmentRef],
1256 ) -> Option<HarnessAttachmentRef> {
1257 if attachments
1258 .iter()
1259 .any(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack)
1260 {
1261 return None;
1262 }
1263 let crashpack = self.build_crashpack_for_report(run)?;
1264 Some(HarnessAttachmentRef::crashpack(artifact_filename(
1265 &crashpack,
1266 )))
1267 }
1268
1269 #[must_use]
1273 pub fn build_crashpack_for_report(&self, run: &LabRunReport) -> Option<CrashPack> {
1274 let has_failure = !run.oracle_report.all_passed()
1275 || !run.invariant_violations.is_empty()
1276 || run.refinement_firewall_rule_id.is_some();
1277 if !has_failure {
1278 return None;
1279 }
1280
1281 let config_summary = LabConfigSummary::from_config(&self.config);
1282 let crash_config = CrashPackConfig {
1283 seed: run.seed,
1284 config_hash: config_summary.config_hash(),
1285 worker_count: self.config.worker_count,
1286 max_steps: self.config.max_steps,
1287 commit_hash: None,
1288 };
1289
1290 let (task, region) = self
1291 .state
1292 .tasks_iter()
1293 .find(|(_, task)| !task.state.is_terminal())
1294 .map(|(_, task)| (task.id, task.owner))
1295 .or_else(|| {
1296 self.state
1297 .obligations_iter()
1298 .find(|(_, obligation)| obligation.is_pending())
1299 .map(|(_, obligation)| (obligation.holder, obligation.region))
1300 })
1301 .or_else(|| {
1302 self.state
1303 .regions_iter()
1304 .next()
1305 .map(|(_, region)| (TaskId::testing_default(), region.id))
1306 })
1307 .or_else(|| {
1308 self.state
1309 .root_region
1310 .map(|root| (TaskId::testing_default(), root))
1311 })
1312 .unwrap_or((TaskId::testing_default(), RegionId::testing_default()));
1313
1314 let mut oracle_violations = run.invariant_violations.clone();
1315 oracle_violations.extend(
1316 run.oracle_report
1317 .failures()
1318 .iter()
1319 .map(|entry| entry.invariant.clone()),
1320 );
1321 if let Some(rule_id) = &run.refinement_firewall_rule_id {
1322 oracle_violations.push(format!("refinement_firewall:{rule_id}"));
1323 }
1324 if let Some(prefix_len) = run.refinement_counterexample_prefix_len {
1325 oracle_violations.push(format!(
1326 "refinement_firewall:minimal_counterexample_prefix_len={prefix_len}"
1327 ));
1328 }
1329 oracle_violations.sort();
1330 oracle_violations.dedup();
1331
1332 let trace_events = self.trace().snapshot();
1333 let mut builder = CrashPack::builder(crash_config.clone())
1334 .failure(FailureInfo {
1335 task,
1336 region,
1337 outcome: FailureOutcome::Err,
1338 virtual_time: Time::from_nanos(run.now_nanos),
1339 })
1340 .oracle_violations(oracle_violations)
1341 .replay(ReplayCommand::from_config(&crash_config, None));
1342
1343 let divergent_prefix = self.auto_divergent_prefix();
1344 if !divergent_prefix.is_empty() {
1345 builder = builder.divergent_prefix(divergent_prefix);
1346 }
1347
1348 builder = if trace_events.is_empty() {
1349 builder
1350 .fingerprint(run.trace_fingerprint)
1351 .event_count(run.trace_certificate.event_count)
1352 } else {
1353 builder.from_trace(&trace_events)
1354 };
1355
1356 match builder.build() {
1357 Ok(pack) => Some(pack),
1358 Err(err) => {
1359 let _ = &err;
1360 crate::tracing_compat::error!("failed to build crash pack for lab report: {err}");
1361 None
1362 }
1363 }
1364 }
1365
1366 fn auto_divergent_prefix(&self) -> Vec<ReplayEvent> {
1367 let Some(replay_trace) = self.replay_recorder.snapshot() else {
1368 return Vec::new();
1369 };
1370 if replay_trace.events.is_empty() {
1371 return Vec::new();
1372 }
1373
1374 let failure_index = replay_trace
1375 .events
1376 .iter()
1377 .position(
1378 |event| matches!(event, ReplayEvent::TaskCompleted { outcome, .. } if *outcome > 0),
1379 )
1380 .unwrap_or(replay_trace.events.len().saturating_sub(1));
1381
1382 crate::trace::minimal_divergent_prefix(&replay_trace, failure_index).events
1383 }
1384
1385 fn report_with_steps_delta(&mut self, steps_delta: u64) -> LabRunReport {
1386 let seed = self.config.seed;
1387 let quiescent = self.is_quiescent();
1388 let now = self.now();
1389
1390 let trace_events = self.trace().snapshot();
1391 let trace_len = trace_events.len();
1392
1393 let trace_fingerprint = if trace_events.is_empty() {
1394 seed_fingerprint(seed)
1397 } else {
1398 trace_fingerprint(&trace_events)
1399 };
1400
1401 let schedule_hash = self.certificate().hash();
1402 let mut certificate = TraceCertificate::new();
1403 for e in &trace_events {
1404 certificate.record_event(e);
1405 }
1406 certificate.set_schedule_hash(schedule_hash);
1407
1408 self.oracles.hydrate_temporal_from_state(&self.state, now);
1409 let oracle_report = self.oracles.report(now);
1410 let temporal_invariant_failures = oracle_report
1411 .failures()
1412 .into_iter()
1413 .filter(|entry| TEMPORAL_ORACLE_INVARIANTS.contains(&entry.invariant.as_str()))
1414 .map(|entry| entry.invariant.clone())
1415 .collect::<Vec<_>>();
1416 let temporal_counterexample_prefix_len = if temporal_invariant_failures.is_empty() {
1417 None
1418 } else {
1419 let prefix_len = self.auto_divergent_prefix().len();
1420 (prefix_len > 0).then_some(prefix_len)
1421 };
1422 let trace_total_pushed = self.trace().total_pushed();
1432 let trace_buffered_len = trace_events.len() as u64;
1433 let refinement_firewall_skipped_due_to_trace_truncation =
1434 trace_total_pushed > trace_buffered_len;
1435 let refinement_violation = if refinement_firewall_skipped_due_to_trace_truncation {
1436 None
1437 } else {
1438 check_refinement_firewall(&trace_events).first_violation
1439 };
1440 let refinement_violation = refinement_violation.as_ref();
1441 let refinement_firewall_rule_id = refinement_violation.map(|v| v.rule_id.to_owned());
1442 let refinement_firewall_event_index = refinement_violation.map(|v| v.event_index);
1443 let refinement_firewall_event_seq = refinement_violation.map(|v| v.event_seq);
1444 let refinement_counterexample_prefix_len =
1445 refinement_firewall_event_index.map(|idx| idx + 1);
1446
1447 let mut invariant_violations = self
1448 .check_invariants()
1449 .into_iter()
1450 .map(|v| v.to_string())
1451 .collect::<Vec<_>>();
1452 for invariant in &temporal_invariant_failures {
1453 invariant_violations.push(format!("temporal:{invariant}"));
1454 }
1455 if let Some(prefix_len) = temporal_counterexample_prefix_len {
1456 invariant_violations.push(format!(
1457 "temporal:minimal_divergent_prefix_len={prefix_len}"
1458 ));
1459 }
1460 if let Some(rule_id) = &refinement_firewall_rule_id {
1461 invariant_violations.push(format!("refinement_firewall:{rule_id}"));
1462 }
1463 if let Some(prefix_len) = refinement_counterexample_prefix_len {
1464 invariant_violations.push(format!(
1465 "refinement_firewall:minimal_counterexample_prefix_len={prefix_len}"
1466 ));
1467 }
1468 if refinement_firewall_skipped_due_to_trace_truncation {
1475 invariant_violations.push(format!(
1476 "refinement_firewall:scenario_failed_due_to_trace_truncation:\
1477 seed={seed},total_pushed={trace_total_pushed},buffered={trace_buffered_len}"
1478 ));
1479 }
1480 invariant_violations.sort();
1481 invariant_violations.dedup();
1482
1483 LabRunReport {
1484 seed,
1485 steps_delta,
1486 steps_total: self.steps(),
1487 quiescent,
1488 now_nanos: now.as_nanos(),
1489 trace_len,
1490 trace_fingerprint,
1491 trace_certificate: LabTraceCertificateSummary {
1492 event_hash: certificate.event_hash(),
1493 event_count: certificate.event_count(),
1494 schedule_hash: certificate.schedule_hash(),
1495 },
1496 oracle_report,
1497 invariant_violations,
1498 temporal_invariant_failures,
1499 temporal_counterexample_prefix_len,
1500 refinement_firewall_rule_id,
1501 refinement_firewall_event_index,
1502 refinement_firewall_event_seq,
1503 refinement_counterexample_prefix_len,
1504 refinement_firewall_skipped_due_to_trace_truncation,
1505 }
1506 }
1507
1508 pub fn enable_deadline_monitoring(&mut self, config: MonitorConfig) {
1510 self.enable_deadline_monitoring_with_handler(config, default_warning_handler);
1511 }
1512
1513 pub fn enable_deadline_monitoring_with_handler<F>(&mut self, config: MonitorConfig, f: F)
1515 where
1516 F: Fn(DeadlineWarning) + Send + Sync + 'static,
1517 {
1518 let mut monitor = DeadlineMonitor::new(config);
1519 monitor.on_warning(f);
1520 self.deadline_monitor = Some(monitor);
1521 }
1522
1523 pub fn deadline_monitor_mut(&mut self) -> Option<&mut DeadlineMonitor> {
1525 self.deadline_monitor.as_mut()
1526 }
1527
1528 #[allow(clippy::too_many_lines)]
1530 fn step(&mut self) {
1531 self.steps += 1;
1532 let rng_value = self.rng.next_u64();
1533 if self.steps < 50 {
1534 crate::tracing_compat::trace!(
1535 "lab runtime rng sample: rng_value={}, worker_hint={}",
1536 rng_value,
1537 (rng_value >> 32) as usize % self.config.worker_count.max(1)
1538 );
1539 }
1540 self.replay_recorder.record_rng_value(rng_value);
1541 self.check_futurelocks();
1542 if let Some(timer) = self.state.timer_driver_handle() {
1543 let _ = timer.process_timers();
1544 }
1545 self.poll_io();
1546 self.schedule_async_finalizers();
1547
1548 let worker_count = self.config.worker_count.max(1);
1550 let worker_hint = ((rng_value >> 32) as usize) % worker_count;
1552 let now = self.now();
1553 let (task_id, dispatch_lane) = {
1554 let mut sched = self.scheduler.lock();
1555 if let Some((tid, lane)) = sched.pop_for_worker(worker_hint, rng_value, now) {
1556 (tid, lane)
1557 } else if let Some(tid) = sched.steal_for_worker(worker_hint, rng_value.rotate_left(17))
1558 {
1559 (tid, DispatchLane::Stolen)
1560 } else {
1561 drop(sched);
1562 self.check_deadline_monitor();
1563 return;
1564 }
1565 };
1566
1567 self.certificate.record(task_id, dispatch_lane, self.steps);
1569 self.replay_recorder
1570 .record_task_scheduled(task_id, self.steps);
1571
1572 if self.inject_pre_poll_chaos(task_id) {
1574 return;
1576 }
1577
1578 let priority = self.state.task(task_id).map_or(0, |record| {
1580 record.cx_inner.as_ref().map_or(0, |inner| {
1581 let mut guard = inner.write();
1582
1583 if guard.budget.consume_poll().is_none() {
1585 guard.cancel_requested = true;
1586 guard
1587 .fast_cancel
1588 .store(true, std::sync::atomic::Ordering::Release);
1589 if let Some(existing) = &mut guard.cancel_reason {
1590 existing.strengthen(&crate::types::CancelReason::poll_quota());
1591 } else {
1592 guard.cancel_reason = Some(crate::types::CancelReason::poll_quota());
1593 }
1594 }
1595
1596 guard.budget.priority
1597 })
1598 });
1599
1600 let waker = Waker::from(Arc::new(TaskWaker {
1601 task_id,
1602 priority,
1603 scheduler: self.scheduler.clone(),
1604 }));
1605 let mut cx = Context::from_waker(&waker);
1606
1607 if let Some(record) = self.state.task(task_id) {
1609 if let Some(inner) = record.cx_inner.as_ref() {
1610 let cancel_waker = Waker::from(Arc::new(CancelTaskWaker {
1611 task_id,
1612 priority,
1613 scheduler: self.scheduler.clone(),
1614 }));
1615 {
1616 let mut guard = inner.write();
1617 guard.cancel_waker = Some(cancel_waker);
1618 }
1619 }
1620 }
1621
1622 let current_cx = self
1623 .state
1624 .task(task_id)
1625 .and_then(|record| record.cx.clone());
1626 let _cx_guard = crate::cx::Cx::set_current(current_cx);
1627
1628 let started_running = self
1629 .state
1630 .update_task(task_id, |record| {
1631 let old_state = record.state.clone();
1632 if record.start_running() {
1633 Some((old_state, record.state.clone()))
1634 } else {
1635 None
1636 }
1637 })
1638 .flatten();
1639
1640 if let Some((from_state, to_state)) = started_running.as_ref() {
1641 if self.config.has_cancellation_oracle() {
1642 self.notify_cancellation_oracle_task_transition(task_id, from_state, to_state);
1643 }
1644 }
1645
1646 if self.steps < 50 {
1648 crate::tracing_compat::trace!(
1649 "lab runtime executing task {:?} at step {}",
1650 task_id,
1651 self.steps
1652 );
1653 }
1654
1655 if self.config.has_cancellation_oracle() {
1657 self.notify_cancellation_oracle_task_poll(task_id);
1658 }
1659
1660 let result = if let Some(stored) = self.state.get_stored_future(task_id) {
1661 stored.poll(&mut cx)
1662 } else {
1663 return;
1665 };
1666
1667 let _ = self.state.update_task(task_id, |record| {
1669 record.mark_polled(self.steps);
1670 });
1671
1672 let cancel_ack = self.consume_cancel_ack(task_id);
1673
1674 if cancel_ack && self.config.has_cancellation_oracle() {
1676 self.notify_cancellation_oracle_cancel_ack(task_id);
1677 }
1678
1679 match result {
1681 Poll::Ready(outcome) => {
1682 self.state.remove_stored_future(task_id);
1684 self.scheduler.lock().forget_task(task_id);
1685
1686 let mut oracle_transitions = Vec::new(); let _ = self.state.update_task(task_id, |record| {
1690 if !record.state.is_terminal() {
1691 let old_state = record.state.clone();
1692 let record_outcome = match outcome {
1693 crate::types::Outcome::Ok(()) => crate::types::Outcome::Ok(()),
1694 crate::types::Outcome::Err(()) => crate::types::Outcome::Err(
1695 crate::error::Error::new(crate::error::ErrorKind::Internal),
1696 ),
1697 crate::types::Outcome::Cancelled(r) => {
1698 crate::types::Outcome::Cancelled(r)
1699 }
1700 crate::types::Outcome::Panicked(p) => {
1701 crate::types::Outcome::Panicked(p)
1702 }
1703 };
1704 let completed_via_cancel =
1705 if matches!(record_outcome, crate::types::Outcome::Ok(())) {
1706 let should_cancel = matches!(
1707 record.state,
1708 TaskState::Cancelling { .. } | TaskState::Finalizing { .. }
1709 ) || (cancel_ack
1710 && matches!(record.state, TaskState::CancelRequested { .. }));
1711 if should_cancel {
1712 if matches!(record.state, TaskState::CancelRequested { .. }) {
1713 let state_before = record.state.clone();
1714 let _ = record.acknowledge_cancel();
1715 oracle_transitions
1716 .push((state_before, record.state.clone()));
1717 }
1718 if matches!(record.state, TaskState::Cancelling { .. }) {
1719 let state_before = record.state.clone();
1720 record.cleanup_done();
1721 oracle_transitions
1722 .push((state_before, record.state.clone()));
1723 }
1724 if matches!(record.state, TaskState::Finalizing { .. }) {
1725 let state_before = record.state.clone();
1726 record.finalize_done();
1727 oracle_transitions
1728 .push((state_before, record.state.clone()));
1729 }
1730 matches!(
1731 record.state,
1732 TaskState::Completed(crate::types::Outcome::Cancelled(_))
1733 )
1734 } else {
1735 false
1736 }
1737 } else {
1738 false
1739 };
1740 if !completed_via_cancel {
1741 record.complete(record_outcome);
1742 oracle_transitions.push((old_state, record.state.clone()));
1743 }
1744 }
1745 });
1746
1747 if self.config.has_cancellation_oracle() {
1749 for (from_state, to_state) in oracle_transitions {
1750 self.notify_cancellation_oracle_task_transition(
1751 task_id,
1752 &from_state,
1753 &to_state,
1754 );
1755 }
1756 }
1757
1758 let final_severity =
1763 self.state
1764 .task(task_id)
1765 .map_or(crate::types::Severity::Ok, |record| match &record.state {
1766 TaskState::Completed(outcome) => outcome.severity(),
1767 _ => crate::types::Severity::Ok,
1768 });
1769 self.replay_recorder
1770 .record_task_completed(task_id, final_severity);
1771
1772 if let Some(monitor) = &mut self.deadline_monitor {
1773 if let Some(record) = self.state.task(task_id) {
1774 let now = self.state.now;
1775 let duration =
1776 Duration::from_nanos(now.duration_since(record.created_at()));
1777 let (task_type, deadline) = record
1778 .cx_inner
1779 .as_ref()
1780 .map(|inner| inner.read())
1781 .map_or_else(
1782 || ("default".to_string(), None),
1783 |inner| {
1784 (
1785 inner
1786 .task_type
1787 .clone()
1788 .unwrap_or_else(|| "default".to_string()),
1789 inner.budget.deadline,
1790 )
1791 },
1792 );
1793 monitor.record_completion(task_id, &task_type, duration, deadline, now);
1794 }
1795 }
1796
1797 let waiters = self.state.task_completed(task_id);
1799
1800 let scheduled: Vec<(TaskId, u8)> = waiters
1815 .into_iter()
1816 .map(|w| {
1817 let prio = self
1818 .state
1819 .task(w)
1820 .and_then(|t| t.cx_inner.as_ref())
1821 .map_or(0, |inner| inner.read().budget.priority);
1822 (w, prio)
1823 })
1824 .collect();
1825 let mut sched = self.scheduler.lock();
1826 for (waiter, prio) in scheduled {
1827 sched.schedule(waiter, prio);
1828 }
1829 }
1830 Poll::Pending => {
1831 self.replay_recorder.record_task_yielded(task_id);
1837
1838 self.inject_post_poll_chaos(task_id, priority);
1840 }
1841 }
1842
1843 self.check_deadline_monitor();
1844 }
1845
1846 fn check_deadline_monitor(&mut self) {
1847 if let Some(monitor) = &mut self.deadline_monitor {
1848 let now = self.state.now;
1849 monitor.check(now, self.state.tasks_iter().map(|(_, record)| record));
1850 }
1851 }
1852
1853 fn poll_io(&mut self) {
1854 let Some(handle) = self.state.io_driver_handle() else {
1855 return;
1856 };
1857 let now = self.state.now;
1858 let (state, recorder, seen) = (
1859 &mut self.state,
1860 &mut self.replay_recorder,
1861 &mut self.seen_io_tokens,
1862 );
1863 if let Err(error) = handle.turn_with(Some(Duration::ZERO), |event, interest| {
1864 let token = event.token.0;
1865 let interest = interest.unwrap_or(event.ready);
1866 if seen.insert(token) {
1867 state.record_trace_event(|seq| {
1868 TraceEvent::io_requested(seq, now, token as u64, interest.bits())
1869 });
1870 }
1871 state.record_trace_event(|seq| {
1872 TraceEvent::io_ready(seq, now, token as u64, event.ready.bits())
1873 });
1874 recorder.record_io_ready(
1875 token as u64,
1876 event.is_readable(),
1877 event.is_writable(),
1878 event.is_error(),
1879 event.is_hangup(),
1880 );
1881 }) {
1882 let _ = &error;
1883 crate::tracing_compat::warn!(
1884 error = ?error,
1885 "lab runtime io_driver poll failed"
1886 );
1887 }
1888 self.sync_reactor_chaos_stats();
1889 }
1890
1891 fn inject_pre_poll_chaos(&mut self, task_id: TaskId) -> bool {
1895 let Some(chaos_config) = self.config.chaos.clone() else {
1896 return false;
1897 };
1898 let Some(chaos_rng) = &mut self.chaos_rng else {
1899 return false;
1900 };
1901
1902 let cancel = chaos_rng.should_inject_cancel(&chaos_config);
1903
1904 let delay = chaos_rng
1906 .should_inject_delay(&chaos_config)
1907 .then(|| chaos_rng.next_delay(&chaos_config));
1908
1909 let budget_exhaust = chaos_rng.should_inject_budget_exhaust(&chaos_config);
1911 let skip_poll = cancel | budget_exhaust;
1912 self.chaos_stats
1913 .record_pre_poll_outcomes(cancel, delay, budget_exhaust);
1914
1915 if cancel {
1918 self.inject_cancel(task_id);
1919 }
1920
1921 if let Some(d) = delay {
1922 self.advance_time(Self::duration_nanos_saturating(d));
1923 }
1924
1925 if budget_exhaust {
1926 self.inject_budget_exhaust(task_id);
1927 }
1928
1929 if skip_poll {
1930 self.reschedule_after_chaos_skip(task_id);
1931 }
1932
1933 skip_poll
1934 }
1935
1936 #[inline]
1937 fn duration_nanos_saturating(duration: Duration) -> u64 {
1938 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
1939 }
1940
1941 fn inject_post_poll_chaos(&mut self, task_id: TaskId, priority: u8) {
1943 let Some(chaos_config) = self.config.chaos.clone() else {
1944 return;
1945 };
1946 let Some(chaos_rng) = &mut self.chaos_rng else {
1947 return;
1948 };
1949
1950 let has_open_region = self.state.live_region_count() > 0;
1957 let wakeup_count = if chaos_rng.should_inject_wakeup_storm(&chaos_config, has_open_region) {
1958 Some(chaos_rng.next_wakeup_count(&chaos_config))
1959 } else {
1960 None
1961 };
1962
1963 let target_region_open = self
1973 .state
1974 .task(task_id)
1975 .map(|t| t.owner)
1976 .and_then(|owner| self.state.region(owner))
1977 .is_some_and(|region| region.state().can_accept_work());
1978
1979 if let Some(count) = wakeup_count
1981 && target_region_open
1982 {
1983 self.chaos_stats.record_wakeup_storm(count as u64);
1984 self.inject_spurious_wakes(task_id, priority, count);
1985 } else {
1986 self.chaos_stats.record_no_injection();
1987 }
1988 }
1989
1990 fn sync_reactor_chaos_stats(&mut self) {
1991 let current = self.lab_reactor.chaos_stats();
1992 let previous = &self.seen_reactor_chaos_stats;
1993 let delta = ChaosStats {
1994 cancellations: current.cancellations.saturating_sub(previous.cancellations),
1995 delays: current.delays.saturating_sub(previous.delays),
1996 total_delay: current.total_delay.saturating_sub(previous.total_delay),
1997 io_errors: current.io_errors.saturating_sub(previous.io_errors),
1998 wakeup_storms: current.wakeup_storms.saturating_sub(previous.wakeup_storms),
1999 spurious_wakeups: current
2000 .spurious_wakeups
2001 .saturating_sub(previous.spurious_wakeups),
2002 budget_exhaustions: current
2003 .budget_exhaustions
2004 .saturating_sub(previous.budget_exhaustions),
2005 decision_points: current
2006 .decision_points
2007 .saturating_sub(previous.decision_points),
2008 };
2009 self.chaos_stats.merge(&delta);
2010 self.seen_reactor_chaos_stats = current;
2011 }
2012
2013 fn reschedule_after_chaos_skip(&self, task_id: TaskId) {
2014 let Some(record) = self.state.task(task_id) else {
2015 return;
2016 };
2017 if record.state.is_terminal() {
2018 return;
2019 }
2020 let priority = record
2021 .cx_inner
2022 .as_ref()
2023 .map_or(0, |inner| inner.read().budget.priority);
2024 let mut sched = self.scheduler.lock();
2025 sched.schedule_cancel(task_id, priority);
2026 }
2027
2028 fn schedule_async_finalizers(&mut self) {
2029 let tasks = self.state.drain_ready_async_finalizers();
2030 if tasks.is_empty() {
2031 return;
2032 }
2033 let mut sched = self.scheduler.lock();
2034 for (task_id, priority) in tasks {
2035 sched.schedule(task_id, priority);
2036 }
2037 }
2038
2039 fn consume_cancel_ack(&mut self, task_id: TaskId) -> bool {
2040 self.state
2041 .update_task(task_id, |record| {
2042 let Some(inner) = record.cx_inner.as_ref() else {
2043 return false;
2044 };
2045 let mut acknowledged = false;
2046 {
2047 let mut guard = inner.write();
2048 if guard.cancel_acknowledged {
2049 guard.cancel_acknowledged = false;
2050 drop(guard);
2051 acknowledged = true;
2052 }
2053 }
2054 if acknowledged {
2055 let _ = record.acknowledge_cancel();
2056 }
2057 acknowledged
2058 })
2059 .unwrap_or(false)
2060 }
2061
2062 fn inject_cancel(&mut self, task_id: TaskId) {
2064 use crate::types::{Budget, CancelReason};
2065
2066 self.replay_recorder.record_cancel_injection(task_id);
2068
2069 let reason = CancelReason::user("chaos-injected");
2071 if self.config.has_cancellation_oracle() {
2072 self.oracles.cancellation_protocol.on_cancel_request(
2073 task_id,
2074 reason.clone(),
2075 self.virtual_time,
2076 );
2077 }
2078
2079 let transition = self
2081 .state
2082 .update_task(task_id, |record| {
2083 if !record.state.is_terminal() {
2084 let old_state = record.state.clone();
2085 record.request_cancel_with_budget(reason, Budget::ZERO);
2086 Some((old_state, record.state.clone()))
2087 } else {
2088 None
2089 }
2090 })
2091 .flatten();
2092
2093 if let Some((old_state, new_state)) = transition {
2095 if self.config.has_cancellation_oracle() {
2096 self.oracles.cancellation_protocol.on_transition(
2097 task_id,
2098 &old_state,
2099 &new_state,
2100 self.virtual_time,
2101 );
2102 }
2103 }
2104
2105 self.state.record_trace_event(|seq| {
2107 TraceEvent::new(
2108 seq,
2109 self.virtual_time,
2110 TraceEventKind::ChaosInjection,
2111 TraceData::Chaos {
2112 kind: "cancel".to_string(),
2113 task: Some(task_id),
2114 detail: "chaos-injected cancellation".to_string(),
2115 },
2116 )
2117 });
2118 }
2119
2120 pub fn notify_cancellation_oracle_task_create(&mut self, task_id: TaskId, region_id: RegionId) {
2122 self.oracles
2123 .cancellation_protocol
2124 .on_task_create(task_id, region_id);
2125 }
2126
2127 pub fn notify_cancellation_oracle_region_create(
2129 &mut self,
2130 region_id: RegionId,
2131 parent: Option<RegionId>,
2132 ) {
2133 self.oracles
2134 .cancellation_protocol
2135 .on_region_create(region_id, parent);
2136 }
2137
2138 pub fn notify_cancellation_oracle_task_transition(
2140 &mut self,
2141 task_id: TaskId,
2142 from: &crate::record::task::TaskState,
2143 to: &crate::record::task::TaskState,
2144 ) {
2145 self.oracles
2146 .cancellation_protocol
2147 .on_transition(task_id, from, to, self.virtual_time);
2148 }
2149
2150 pub fn notify_cancellation_oracle_cancel_request(
2152 &mut self,
2153 task_id: TaskId,
2154 reason: crate::types::CancelReason,
2155 ) {
2156 self.oracles
2157 .cancellation_protocol
2158 .on_cancel_request(task_id, reason, self.virtual_time);
2159 }
2160
2161 pub fn notify_cancellation_oracle_cancel_ack(&mut self, task_id: TaskId) {
2163 self.oracles
2164 .cancellation_protocol
2165 .on_cancel_ack(task_id, self.virtual_time);
2166 }
2167
2168 pub fn notify_cancellation_oracle_task_poll(&mut self, task_id: TaskId) {
2170 self.oracles.cancellation_protocol.on_task_poll(task_id);
2171 }
2172
2173 pub fn notify_cancellation_oracle_mask_enter(&mut self, task_id: TaskId) {
2175 self.oracles
2176 .cancellation_protocol
2177 .on_mask_enter(task_id, self.virtual_time);
2178 }
2179
2180 pub fn notify_cancellation_oracle_mask_exit(&mut self, task_id: TaskId) {
2182 self.oracles
2183 .cancellation_protocol
2184 .on_mask_exit(task_id, self.virtual_time);
2185 }
2186
2187 pub fn notify_cancellation_oracle_region_cancel(
2189 &mut self,
2190 region_id: RegionId,
2191 reason: crate::types::CancelReason,
2192 ) {
2193 self.oracles
2194 .cancellation_protocol
2195 .on_region_cancel(region_id, reason, self.virtual_time);
2196 }
2197
2198 pub fn check_cancellation_protocol(
2200 &mut self,
2201 ) -> Result<(), crate::lab::oracle::CancellationProtocolViolation> {
2202 if !self.config.has_cancellation_oracle() {
2203 return Ok(());
2204 }
2205
2206 let result = self.oracles.cancellation_protocol.check();
2207
2208 if let Err(ref violation) = result {
2209 if self.config.panic_on_cancellation_violation {
2210 panic!("Cancellation protocol violation detected: {violation}");
2212 } else {
2213 crate::tracing_compat::warn!(
2215 violation = %violation,
2216 "Cancellation protocol violation detected"
2217 );
2218 }
2219 }
2220
2221 result
2222 }
2223
2224 fn inject_budget_exhaust(&mut self, task_id: TaskId) {
2226 self.replay_recorder
2228 .record_budget_exhaust_injection(task_id);
2229
2230 if let Some(record) = self.state.task(task_id) {
2232 if let Some(cx_inner) = &record.cx_inner {
2233 let mut inner = cx_inner.write();
2234 inner.budget.poll_quota = 0;
2235 inner.budget.cost_quota = Some(0);
2236 }
2237 }
2238
2239 self.state.record_trace_event(|seq| {
2241 TraceEvent::new(
2242 seq,
2243 self.virtual_time,
2244 TraceEventKind::ChaosInjection,
2245 TraceData::Chaos {
2246 kind: "budget_exhaust".to_string(),
2247 task: Some(task_id),
2248 detail: "chaos-injected budget exhaustion".to_string(),
2249 },
2250 )
2251 });
2252 }
2253
2254 fn inject_spurious_wakes(&mut self, task_id: TaskId, priority: u8, count: usize) {
2256 let owner_open = self
2264 .state
2265 .task(task_id)
2266 .map(|t| t.owner)
2267 .and_then(|owner| self.state.region(owner))
2268 .is_some_and(|region| region.state().can_accept_work());
2269 if !owner_open {
2270 return;
2271 }
2272
2273 self.replay_recorder
2275 .record_wakeup_storm_injection(task_id, u32::try_from(count).unwrap_or(u32::MAX));
2276
2277 let mut sched = self.scheduler.lock();
2279 sched.inject_spurious_wakes(task_id, priority, count);
2280 drop(sched);
2281
2282 self.state.record_trace_event(|seq| {
2284 TraceEvent::new(
2285 seq,
2286 self.virtual_time,
2287 TraceEventKind::ChaosInjection,
2288 TraceData::Chaos {
2289 kind: "wakeup_storm".to_string(),
2290 task: Some(task_id),
2291 detail: format!("chaos-injected {count} spurious wakeups"),
2292 },
2293 )
2294 });
2295 }
2296
2297 pub fn step_for_test(&mut self) {
2301 self.step();
2302 }
2303
2304 #[must_use]
2306 pub fn check_invariants(&mut self) -> Vec<InvariantViolation> {
2307 let mut violations = Vec::new();
2308
2309 if let Err(violation) = self.check_cancellation_protocol() {
2311 violations.push(InvariantViolation::CancellationProtocol {
2312 violation: violation.to_string(),
2313 });
2314 }
2315
2316 let leaks = self.obligation_leaks();
2318 if !leaks.is_empty() {
2319 for leak in &leaks {
2320 let _ = self.state.mark_obligation_leaked(leak.obligation);
2321 }
2322 violations.push(InvariantViolation::ObligationLeak { leaks });
2323 }
2324
2325 violations.extend(self.futurelock_violations());
2326 violations.extend(self.quiescence_violations());
2327
2328 let task_leak_count = self.task_leaks();
2330 if task_leak_count > 0 {
2331 violations.push(InvariantViolation::TaskLeak {
2332 count: task_leak_count,
2333 });
2334 }
2335
2336 violations
2337 }
2338
2339 fn obligation_leaks(&self) -> Vec<ObligationLeak> {
2340 let mut leaks = Vec::new();
2341
2342 for (_, obligation) in self.state.obligations_iter() {
2343 if !obligation.is_pending() {
2344 continue;
2345 }
2346
2347 let holder_terminal = self
2348 .state
2349 .task(obligation.holder)
2350 .is_none_or(|t| t.state.is_terminal());
2351 let region_closed = self
2352 .state
2353 .region(obligation.region)
2354 .is_none_or(|r| r.state().is_terminal());
2355
2356 if holder_terminal || region_closed {
2357 leaks.push(ObligationLeak {
2358 obligation: obligation.id,
2359 kind: obligation.kind,
2360 holder: obligation.holder,
2361 region: obligation.region,
2362 });
2363 }
2364 }
2365
2366 leaks
2367 }
2368
2369 fn task_leaks(&self) -> usize {
2370 self.state
2371 .tasks_iter()
2372 .filter(|(_, t)| !t.state.is_terminal())
2373 .count()
2374 }
2375
2376 fn quiescence_violations(&self) -> Vec<InvariantViolation> {
2377 let mut violations = Vec::new();
2378 for (_, region) in self.state.regions_iter() {
2379 if region.state().is_terminal() {
2380 let live_tasks = region
2382 .task_ids()
2383 .iter()
2384 .any(|&tid| self.state.task(tid).is_some_and(|t| !t.state.is_terminal()));
2385
2386 let live_children = region.child_ids().iter().any(|&rid| {
2387 self.state
2388 .region(rid)
2389 .is_some_and(|r| !r.state().is_terminal())
2390 });
2391
2392 if live_tasks || live_children {
2393 violations.push(InvariantViolation::QuiescenceViolation);
2394 }
2395 }
2396 }
2397 violations
2398 }
2399
2400 fn futurelock_violations(&self) -> Vec<InvariantViolation> {
2401 let threshold = self.config.futurelock_max_idle_steps;
2402 if threshold == 0 {
2403 return Vec::new();
2404 }
2405
2406 let current_step = self.steps;
2407 let mut violations = Vec::new();
2408
2409 for (_, task) in self.state.tasks_iter() {
2410 if task.state.is_terminal() {
2411 continue;
2412 }
2413
2414 let mut held = Vec::new();
2415 for (_, obligation) in self.state.obligations_iter() {
2416 if obligation.is_pending() && obligation.holder == task.id {
2417 held.push(obligation.id);
2418 }
2419 }
2420
2421 if held.is_empty() {
2422 continue;
2423 }
2424
2425 let idle_steps = current_step.saturating_sub(task.last_polled_step);
2426 if idle_steps > threshold {
2427 violations.push(InvariantViolation::Futurelock {
2428 task: task.id,
2429 region: task.owner,
2430 idle_steps,
2431 held,
2432 });
2433 }
2434 }
2435
2436 violations
2437 }
2438
2439 fn check_futurelocks(&self) {
2440 let violations = self.futurelock_violations();
2441 if violations.is_empty() {
2442 return;
2443 }
2444
2445 for v in violations {
2446 let InvariantViolation::Futurelock {
2447 task,
2448 region,
2449 idle_steps,
2450 held,
2451 } = v
2452 else {
2453 continue;
2454 };
2455
2456 let mut held_kinds = Vec::new();
2457 for oid in &held {
2458 for (_, obligation) in self.state.obligations_iter() {
2459 if obligation.id == *oid {
2460 held_kinds.push((obligation.id, obligation.kind));
2461 break;
2462 }
2463 }
2464 }
2465
2466 self.state.record_trace_event(|seq| {
2467 TraceEvent::new(
2468 seq,
2469 self.virtual_time,
2470 TraceEventKind::FuturelockDetected,
2471 TraceData::Futurelock {
2472 task,
2473 region,
2474 idle_steps,
2475 held: held_kinds,
2476 },
2477 )
2478 });
2479
2480 assert!(
2481 !self.config.panic_on_futurelock,
2482 "futurelock detected: {task} in {region} idle={idle_steps} held={held:?}"
2483 );
2484 }
2485 }
2486}
2487
2488const DEFAULT_LAB_CANCEL_STREAK_LIMIT: usize = 16;
2489
2490#[derive(Debug, Clone, Copy)]
2491struct PendingSpuriousWake {
2492 priority: u8,
2493 remaining: usize,
2494}
2495
2496#[derive(Debug)]
2497pub struct LabScheduler {
2502 workers: Vec<crate::runtime::scheduler::PriorityScheduler>,
2503 scheduled: DetHashSet<TaskId>,
2504 pending_spurious_wakes: DetHashMap<TaskId, PendingSpuriousWake>,
2505 assignments: Vec<Option<usize>>,
2507 next_worker: usize,
2508 cancel_streak: Vec<usize>,
2509 cancel_streak_limit: usize,
2510}
2511
2512impl LabScheduler {
2513 fn new(worker_count: usize) -> Self {
2514 let count = if worker_count == 0 { 1 } else { worker_count };
2515 let cancel_streak_limit = DEFAULT_LAB_CANCEL_STREAK_LIMIT.max(1);
2516 Self {
2517 workers: (0..count)
2518 .map(|_| crate::runtime::scheduler::PriorityScheduler::new())
2519 .collect(),
2520 scheduled: DetHashSet::default(),
2521 pending_spurious_wakes: DetHashMap::default(),
2522 assignments: Vec::new(),
2523 next_worker: 0,
2524 cancel_streak: vec![0; count],
2525 cancel_streak_limit,
2526 }
2527 }
2528
2529 #[must_use]
2531 pub fn is_empty(&self) -> bool {
2532 self.scheduled.is_empty()
2533 }
2534
2535 #[must_use]
2537 pub fn cancel_streak_limit(&self) -> usize {
2538 self.cancel_streak_limit
2539 }
2540
2541 #[inline]
2542 fn set_assignment(&mut self, task: TaskId, worker: usize) {
2543 let slot = task.arena_index().index() as usize;
2544 if slot >= self.assignments.len() {
2545 self.assignments.resize(slot + 1, None);
2546 }
2547 self.assignments[slot] = Some(worker);
2548 }
2549
2550 fn assign_worker(&mut self, task: TaskId) -> usize {
2551 let slot = task.arena_index().index() as usize;
2552 if slot < self.assignments.len() {
2553 if let Some(worker) = self.assignments[slot] {
2554 return worker;
2555 }
2556 }
2557 let worker = self.next_worker % self.workers.len();
2558 self.next_worker = self.next_worker.wrapping_add(1);
2559 if slot >= self.assignments.len() {
2560 self.assignments.resize(slot + 1, None);
2561 }
2562 self.assignments[slot] = Some(worker);
2563 worker
2564 }
2565
2566 pub fn schedule(&mut self, task: TaskId, priority: u8) {
2568 if !self.scheduled.insert(task) {
2569 crate::tracing_compat::trace!("LabScheduler already scheduled {task:?}");
2570 return;
2571 }
2572 crate::tracing_compat::trace!("LabScheduler scheduling {task:?}");
2573
2574 let worker = self.assign_worker(task);
2575 self.workers[worker].schedule(task, priority);
2576 }
2577
2578 fn inject_spurious_wakes(&mut self, task: TaskId, priority: u8, count: usize) {
2584 if count == 0 {
2585 return;
2586 }
2587
2588 let mut remaining = count;
2589 if self.scheduled.insert(task) {
2590 let worker = self.assign_worker(task);
2591 self.workers[worker].schedule(task, priority);
2592 remaining = remaining.saturating_sub(1);
2593 }
2594
2595 if remaining == 0 {
2596 return;
2597 }
2598
2599 self.pending_spurious_wakes
2600 .entry(task)
2601 .and_modify(|pending| {
2602 pending.priority = pending.priority.max(priority);
2603 pending.remaining = pending.remaining.saturating_add(remaining);
2604 })
2605 .or_insert(PendingSpuriousWake {
2606 priority,
2607 remaining,
2608 });
2609 }
2610
2611 pub fn schedule_cancel(&mut self, task: TaskId, priority: u8) {
2613 if self.scheduled.insert(task) {
2614 let worker = self.assign_worker(task);
2615 self.workers[worker].schedule_cancel(task, priority);
2616 return;
2617 }
2618
2619 let slot = task.arena_index().index() as usize;
2620 if let Some(&Some(worker)) = self.assignments.get(slot) {
2621 self.workers[worker].move_to_cancel_lane(task, priority);
2622 }
2623 }
2624
2625 pub fn schedule_timed(&mut self, task: TaskId, deadline: Time) {
2627 if !self.scheduled.insert(task) {
2628 return;
2629 }
2630
2631 let worker = self.assign_worker(task);
2632 self.workers[worker].schedule_timed(task, deadline);
2633 }
2634
2635 fn pop_for_worker(
2636 &mut self,
2637 worker: usize,
2638 rng_hint: u64,
2639 now: Time,
2640 ) -> Option<(TaskId, DispatchLane)> {
2641 if self.workers.is_empty() {
2642 return None;
2643 }
2644
2645 let worker = worker % self.workers.len();
2646 let cancel_streak = &mut self.cancel_streak[worker];
2647
2648 if *cancel_streak < self.cancel_streak_limit {
2649 if let Some((task, lane)) = self.workers[worker].pop_cancel_with_rng(rng_hint) {
2650 *cancel_streak += 1;
2651 self.scheduled.remove(&task);
2652 self.set_assignment(task, worker);
2653 self.rearm_spurious_wake(task);
2654 return Some((task, lane));
2655 }
2656 }
2657
2658 if let Some(task) = self.workers[worker].pop_timed_only_with_hint(rng_hint, now) {
2659 *cancel_streak = 0;
2660 self.scheduled.remove(&task);
2661 self.set_assignment(task, worker);
2662 self.rearm_spurious_wake(task);
2663 return Some((task, DispatchLane::Timed));
2664 }
2665
2666 if let Some(task) = self.workers[worker].pop_ready_only_with_hint(rng_hint) {
2667 *cancel_streak = 0;
2668 self.scheduled.remove(&task);
2669 self.set_assignment(task, worker);
2670 self.rearm_spurious_wake(task);
2671 return Some((task, DispatchLane::Ready));
2672 }
2673
2674 if let Some((task, lane)) = self.workers[worker].pop_cancel_with_rng(rng_hint) {
2675 *cancel_streak = 1;
2676 self.scheduled.remove(&task);
2677 self.set_assignment(task, worker);
2678 self.rearm_spurious_wake(task);
2679 return Some((task, lane));
2680 }
2681
2682 *cancel_streak = 0;
2683 None
2684 }
2685
2686 fn steal_for_worker(&mut self, thief: usize, rng_hint: u64) -> Option<TaskId> {
2687 let count = self.workers.len();
2688 if count <= 1 {
2689 return None;
2690 }
2691
2692 let thief = thief % count;
2693 let start = (rng_hint as usize) % count;
2694
2695 for offset in 0..count {
2696 let victim = (start + offset) % count;
2697 if victim == thief {
2698 continue;
2699 }
2700 if let Some(task) =
2701 self.workers[victim].pop_ready_only_with_hint(rng_hint.wrapping_add(offset as u64))
2702 {
2703 self.scheduled.remove(&task);
2704 self.set_assignment(task, thief);
2705 self.rearm_spurious_wake(task);
2706 return Some(task);
2707 }
2708 }
2709
2710 None
2711 }
2712
2713 fn forget_task(&mut self, task: TaskId) {
2714 self.scheduled.remove(&task);
2715 self.pending_spurious_wakes.remove(&task);
2716 let slot = task.arena_index().index() as usize;
2717 if slot < self.assignments.len() {
2718 self.assignments[slot] = None;
2719 }
2720 for worker in &mut self.workers {
2721 worker.remove(task);
2722 }
2723 }
2724
2725 fn rearm_spurious_wake(&mut self, task: TaskId) {
2726 let Some(mut pending) = self.pending_spurious_wakes.remove(&task) else {
2727 return;
2728 };
2729
2730 self.schedule(task, pending.priority);
2731 pending.remaining = pending.remaining.saturating_sub(1);
2732 if pending.remaining > 0 {
2733 self.pending_spurious_wakes.insert(task, pending);
2734 }
2735 }
2736}
2737
2738struct TaskWaker {
2739 task_id: crate::types::TaskId,
2740 priority: u8,
2741 scheduler: Arc<Mutex<LabScheduler>>,
2742}
2743
2744use std::task::Wake;
2745impl Wake for TaskWaker {
2746 fn wake(self: Arc<Self>) {
2747 self.scheduler.lock().schedule(self.task_id, self.priority);
2748 }
2749}
2750
2751struct CancelTaskWaker {
2756 task_id: crate::types::TaskId,
2757 priority: u8,
2758 scheduler: Arc<Mutex<LabScheduler>>,
2759}
2760
2761impl Wake for CancelTaskWaker {
2762 fn wake(self: Arc<Self>) {
2763 self.scheduler
2764 .lock()
2765 .schedule_cancel(self.task_id, self.priority);
2766 }
2767}
2768
2769#[derive(Debug, Clone, PartialEq, Eq)]
2771pub enum InvariantViolation {
2772 ObligationLeak {
2774 leaks: Vec<ObligationLeak>,
2776 },
2777 TaskLeak {
2779 count: usize,
2781 },
2782 ActorLeak {
2784 count: usize,
2786 },
2787 QuiescenceViolation,
2789 Futurelock {
2791 task: crate::types::TaskId,
2793 region: crate::types::RegionId,
2795 idle_steps: u64,
2797 held: Vec<ObligationId>,
2799 },
2800 CancellationProtocol {
2802 violation: String,
2804 },
2805 TestPanic {
2810 message: String,
2813 },
2814}
2815
2816#[derive(Debug, Clone, PartialEq, Eq)]
2818pub struct ObligationLeak {
2819 pub obligation: ObligationId,
2821 pub kind: ObligationKind,
2823 pub holder: crate::types::TaskId,
2825 pub region: crate::types::RegionId,
2827}
2828
2829impl std::fmt::Display for ObligationLeak {
2830 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2831 write!(
2832 f,
2833 "{:?} {:?} holder={:?} region={:?}",
2834 self.obligation, self.kind, self.holder, self.region
2835 )
2836 }
2837}
2838
2839impl std::fmt::Display for InvariantViolation {
2840 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2841 match self {
2842 Self::ObligationLeak { leaks } => {
2843 write!(f, "{} obligations leaked", leaks.len())
2844 }
2845 Self::TaskLeak { count } => write!(f, "{count} tasks leaked"),
2846 Self::ActorLeak { count } => write!(f, "{count} actors leaked"),
2847 Self::QuiescenceViolation => write!(f, "region closed without quiescence"),
2848 Self::Futurelock {
2849 task,
2850 region,
2851 idle_steps,
2852 held,
2853 } => write!(
2854 f,
2855 "futurelock: {task} in {region} idle={idle_steps} held={held:?}"
2856 ),
2857 Self::CancellationProtocol { violation } => {
2858 write!(f, "cancellation protocol violation: {violation}")
2859 }
2860 Self::TestPanic { message } => write!(f, "test panic: {message}"),
2861 }
2862 }
2863}
2864
2865pub fn test<F, R>(seed: u64, f: F) -> R
2867where
2868 F: FnOnce(&mut LabRuntime) -> R,
2869{
2870 let mut runtime = LabRuntime::with_seed(seed);
2871 let result = f(&mut runtime);
2872
2873 let violations = runtime.check_invariants();
2875 assert!(
2876 violations.is_empty(),
2877 "Lab runtime invariant violations: {violations:?}"
2878 );
2879
2880 result
2881}
2882
2883#[cfg(test)]
2884mod tests {
2885 #![allow(
2886 clippy::pedantic,
2887 clippy::nursery,
2888 clippy::expect_fun_call,
2889 clippy::map_unwrap_or,
2890 clippy::cast_possible_wrap,
2891 clippy::future_not_send
2892 )]
2893 use super::*;
2894 use crate::lab::chaos::ChaosConfig;
2895 use crate::record::TaskRecord;
2896 use crate::record::{ObligationAbortReason, ObligationKind};
2897 use crate::runtime::deadline_monitor::{AdaptiveDeadlineConfig, WarningReason};
2898 #[cfg(unix)]
2899 use crate::runtime::reactor::{Event, Interest};
2900 use crate::types::{Budget, CxInner, Outcome, TaskId};
2901 use crate::util::ArenaIndex;
2902 use parking_lot::Mutex;
2903 use parking_lot::RwLock;
2904 use std::sync::Arc;
2905 use std::task::Waker;
2906 use std::time::Duration;
2907
2908 #[cfg(unix)]
2909 struct TestFdSource;
2910 #[cfg(unix)]
2911 impl std::os::fd::AsRawFd for TestFdSource {
2912 fn as_raw_fd(&self) -> std::os::fd::RawFd {
2913 0
2914 }
2915 }
2916
2917 #[cfg(unix)]
2918 fn noop_waker() -> Waker {
2919 std::task::Waker::noop().clone()
2920 }
2921
2922 struct FlagWaker(Arc<std::sync::atomic::AtomicBool>);
2924 impl Wake for FlagWaker {
2925 fn wake(self: Arc<Self>) {
2926 self.0.store(true, std::sync::atomic::Ordering::SeqCst);
2927 }
2928 }
2929
2930 struct CountWaker(Arc<std::sync::atomic::AtomicU64>);
2932 impl Wake for CountWaker {
2933 fn wake(self: Arc<Self>) {
2934 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2935 }
2936 }
2937
2938 fn init_test(name: &str) {
2939 crate::test_utils::init_test_logging();
2940 crate::test_phase!(name);
2941 }
2942
2943 struct TimerAdvanceOutcome {
2944 advance_points: Vec<Time>,
2945 total_wakeups: u64,
2946 final_time: Time,
2947 cancelled_wakeups: u64,
2948 }
2949
2950 fn collect_timer_advances(
2951 deadlines_secs: &[u64],
2952 cancelled_indices: &[usize],
2953 ) -> TimerAdvanceOutcome {
2954 let mut runtime = LabRuntime::with_seed(42);
2955 let timer_handle = runtime.state.timer_driver_handle().expect("timer handle");
2956 let live_wakeups = Arc::new(std::sync::atomic::AtomicU64::new(0));
2957 let cancelled_wakeups = Arc::new(std::sync::atomic::AtomicU64::new(0));
2958 let mut handles = Vec::with_capacity(deadlines_secs.len());
2959
2960 for (idx, secs) in deadlines_secs.iter().copied().enumerate() {
2961 let counter = if cancelled_indices.contains(&idx) {
2962 cancelled_wakeups.clone()
2963 } else {
2964 live_wakeups.clone()
2965 };
2966 let waker = Waker::from(Arc::new(CountWaker(counter)));
2967 handles.push(timer_handle.register(Time::from_secs(secs), waker));
2968 }
2969
2970 for &idx in cancelled_indices {
2971 let cancelled = timer_handle.cancel(&handles[idx]);
2972 crate::assert_with_log!(
2973 cancelled,
2974 "cancelled timer handle remains removable before auto-advance",
2975 true,
2976 cancelled
2977 );
2978 }
2979
2980 let mut advance_points = Vec::new();
2981 while runtime.pending_timer_count() > 0 {
2982 let before = runtime.now();
2983 let next_deadline = runtime.next_timer_deadline().expect("pending deadline");
2984 let wakeups = runtime.advance_to_next_timer();
2985 let after = runtime.now();
2986
2987 crate::assert_with_log!(
2988 after >= before,
2989 "virtual time stays monotone while advancing timers",
2990 true,
2991 after >= before
2992 );
2993 crate::assert_with_log!(
2994 after >= next_deadline,
2995 "advance reaches or passes scheduled deadline",
2996 true,
2997 after >= next_deadline
2998 );
2999 crate::assert_with_log!(
3000 wakeups > 0,
3001 "each advance drains at least one live timer",
3002 true,
3003 wakeups > 0
3004 );
3005
3006 advance_points.push(after);
3007 }
3008
3009 TimerAdvanceOutcome {
3010 advance_points,
3011 total_wakeups: live_wakeups.load(std::sync::atomic::Ordering::SeqCst),
3012 final_time: runtime.now(),
3013 cancelled_wakeups: cancelled_wakeups.load(std::sync::atomic::Ordering::SeqCst),
3014 }
3015 }
3016
3017 #[test]
3018 fn empty_runtime_is_quiescent() {
3019 init_test("empty_runtime_is_quiescent");
3020 let runtime = LabRuntime::with_seed(42);
3021 let quiescent = runtime.is_quiescent();
3022 crate::assert_with_log!(quiescent, "quiescent", true, quiescent);
3023 crate::test_complete!("empty_runtime_is_quiescent");
3024 }
3025
3026 #[test]
3027 fn advance_time() {
3028 init_test("advance_time");
3029 let mut runtime = LabRuntime::with_seed(42);
3030 let now = runtime.now();
3031 crate::assert_with_log!(now == Time::ZERO, "now", Time::ZERO, now);
3032
3033 runtime.advance_time(1_000_000);
3034 let now = runtime.now();
3035 crate::assert_with_log!(
3036 now == Time::from_millis(1),
3037 "now",
3038 Time::from_millis(1),
3039 now
3040 );
3041 crate::test_complete!("advance_time");
3042 }
3043
3044 #[test]
3045 fn duration_nanos_saturating_clamps_large_duration() {
3046 init_test("duration_nanos_saturating_clamps_large_duration");
3047 let huge = Duration::from_secs(u64::MAX);
3048 let saturated = LabRuntime::duration_nanos_saturating(huge);
3049 crate::assert_with_log!(
3050 saturated == u64::MAX,
3051 "huge duration saturates",
3052 u64::MAX,
3053 saturated
3054 );
3055
3056 let small = Duration::from_nanos(123);
3057 let exact = LabRuntime::duration_nanos_saturating(small);
3058 crate::assert_with_log!(exact == 123, "small duration exact", 123u64, exact);
3059 crate::test_complete!("duration_nanos_saturating_clamps_large_duration");
3060 }
3061
3062 #[cfg(unix)]
3063 #[test]
3064 fn lab_runtime_records_io_ready_trace() {
3065 init_test("lab_runtime_records_io_ready_trace");
3066
3067 let mut runtime = LabRuntime::with_seed(42);
3068 let handle = runtime.state.io_driver_handle().expect("io driver");
3069 let waker = noop_waker();
3070 let source = TestFdSource;
3071
3072 let registration = handle
3073 .register(&source, Interest::READABLE, waker)
3074 .expect("register source");
3075 let token = registration.token();
3076
3077 runtime
3078 .lab_reactor()
3079 .inject_event(token, Event::readable(token), Duration::from_millis(1));
3080 runtime.advance_time(1_000_000);
3081 runtime.step_for_test();
3082
3083 let mut saw_requested = false;
3084 let mut saw_ready = false;
3085 for event in runtime.state.trace.snapshot() {
3086 if event.kind == TraceEventKind::IoRequested {
3087 saw_requested = true;
3088 }
3089 if event.kind == TraceEventKind::IoReady {
3090 saw_ready = true;
3091 }
3092 }
3093 crate::assert_with_log!(
3094 saw_requested,
3095 "io requested trace recorded",
3096 true,
3097 saw_requested
3098 );
3099 crate::assert_with_log!(saw_ready, "io ready trace recorded", true, saw_ready);
3100 crate::test_complete!("lab_runtime_records_io_ready_trace");
3101 }
3102
3103 #[cfg(unix)]
3104 #[test]
3105 fn lab_runtime_chaos_stats_include_reactor_io_error_injections() {
3106 init_test("lab_runtime_chaos_stats_include_reactor_io_error_injections");
3107
3108 let config = LabConfig::new(7).with_chaos(
3109 ChaosConfig::new(7)
3110 .with_io_error_probability(1.0)
3111 .with_io_error_kinds(vec![std::io::ErrorKind::TimedOut]),
3112 );
3113 let mut runtime = LabRuntime::new(config);
3114 let handle = runtime.state.io_driver_handle().expect("io driver");
3115 let waker = noop_waker();
3116 let source = TestFdSource;
3117
3118 let registration = handle
3119 .register(&source, Interest::READABLE, waker)
3120 .expect("register source");
3121 let token = registration.token();
3122
3123 runtime
3124 .lab_reactor()
3125 .inject_event(token, Event::readable(token), Duration::ZERO);
3126 runtime.step_for_test();
3127
3128 let stats = runtime.chaos_stats();
3129 crate::assert_with_log!(
3130 stats.io_errors == 1,
3131 "io errors aggregated",
3132 1u64,
3133 stats.io_errors
3134 );
3135 crate::assert_with_log!(
3136 stats.decision_points == 1,
3137 "reactor decision points aggregated",
3138 1u64,
3139 stats.decision_points
3140 );
3141 crate::assert_with_log!(
3142 runtime.lab_reactor().last_io_error_kind() == Some(std::io::ErrorKind::TimedOut),
3143 "reactor last error kind surfaced",
3144 Some(std::io::ErrorKind::TimedOut),
3145 runtime.lab_reactor().last_io_error_kind()
3146 );
3147
3148 crate::test_complete!("lab_runtime_chaos_stats_include_reactor_io_error_injections");
3149 }
3150
3151 #[test]
3152 fn pending_task_without_wakeup_storm_still_counts_chaos_decision_point() {
3153 init_test("pending_task_without_wakeup_storm_still_counts_chaos_decision_point");
3154
3155 let config =
3156 LabConfig::new(99).with_chaos(ChaosConfig::new(99).with_wakeup_storm_probability(0.0));
3157 let mut runtime = LabRuntime::new(config);
3158 let region = runtime.state.create_root_region(Budget::INFINITE);
3159 let (task_id, _handle) = runtime
3160 .state
3161 .create_task(region, Budget::INFINITE, async {
3162 std::future::pending::<()>().await;
3163 })
3164 .expect("create task");
3165 runtime.scheduler.lock().schedule(task_id, 0);
3166
3167 runtime.step_for_test();
3168
3169 let stats = runtime.chaos_stats();
3170 crate::assert_with_log!(
3171 stats.decision_points == 2,
3172 "pending-task decision point counted",
3173 2u64,
3174 stats.decision_points
3175 );
3176 crate::assert_with_log!(
3177 stats.wakeup_storms == 0,
3178 "no wakeup storm recorded",
3179 0u64,
3180 stats.wakeup_storms
3181 );
3182
3183 crate::test_complete!(
3184 "pending_task_without_wakeup_storm_still_counts_chaos_decision_point"
3185 );
3186 }
3187
3188 #[test]
3189 fn pre_poll_multi_injection_counts_one_chaos_decision_point() {
3190 init_test("pre_poll_multi_injection_counts_one_chaos_decision_point");
3191
3192 let config = LabConfig::new(123).with_chaos(
3193 ChaosConfig::new(123)
3194 .with_cancel_probability(1.0)
3195 .with_delay_probability(1.0)
3196 .with_delay_range(Duration::ZERO..Duration::from_nanos(2))
3197 .with_budget_exhaust_probability(1.0),
3198 );
3199 let mut runtime = LabRuntime::new(config);
3200 let region = runtime.state.create_root_region(Budget::INFINITE);
3201 let (task_id, _handle) = runtime
3202 .state
3203 .create_task(region, Budget::INFINITE, async {
3204 std::future::pending::<()>().await;
3205 })
3206 .expect("create task");
3207 runtime.scheduler.lock().schedule(task_id, 0);
3208
3209 runtime.step_for_test();
3210
3211 let stats = runtime.chaos_stats();
3212 crate::assert_with_log!(
3213 stats.decision_points == 1,
3214 "multi-injection pre-poll counts once",
3215 1u64,
3216 stats.decision_points
3217 );
3218 crate::assert_with_log!(
3219 stats.cancellations == 1,
3220 "cancel recorded",
3221 1u64,
3222 stats.cancellations
3223 );
3224 crate::assert_with_log!(stats.delays == 1, "delay recorded", 1u64, stats.delays);
3225 crate::assert_with_log!(
3226 stats.budget_exhaustions == 1,
3227 "budget exhaust recorded",
3228 1u64,
3229 stats.budget_exhaustions
3230 );
3231 crate::assert_with_log!(
3232 stats.total_delay == Duration::from_nanos(1),
3233 "positive delay preserved",
3234 Duration::from_nanos(1),
3235 stats.total_delay
3236 );
3237
3238 crate::test_complete!("pre_poll_multi_injection_counts_one_chaos_decision_point");
3239 }
3240
3241 #[test]
3242 fn deterministic_rng() {
3243 init_test("deterministic_rng");
3244 let mut r1 = LabRuntime::with_seed(42);
3245 let mut r2 = LabRuntime::with_seed(42);
3246
3247 let a = r1.rng.next_u64();
3248 let b = r2.rng.next_u64();
3249 crate::assert_with_log!(a == b, "rng", b, a);
3250 crate::test_complete!("deterministic_rng");
3251 }
3252
3253 #[test]
3254 fn lab_scheduler_pop_for_worker_respects_timed_deadlines() {
3255 init_test("lab_scheduler_pop_for_worker_respects_timed_deadlines");
3256 let mut scheduler = LabScheduler::new(1);
3257 let timed = TaskId::from_arena(ArenaIndex::new(1, 0));
3258 let ready = TaskId::from_arena(ArenaIndex::new(2, 0));
3259
3260 scheduler.schedule_timed(timed, Time::from_nanos(100));
3261 scheduler.schedule(ready, 10);
3262
3263 let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3264 crate::assert_with_log!(
3265 first == Some((ready, DispatchLane::Ready)),
3266 "ready task dispatches before not-due timed task",
3267 Some((ready, DispatchLane::Ready)),
3268 first
3269 );
3270
3271 let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3272 crate::assert_with_log!(
3273 second.is_none(),
3274 "future timed task stays queued before deadline",
3275 true,
3276 second.is_none()
3277 );
3278
3279 let third = scheduler.pop_for_worker(0, 2, Time::from_nanos(100));
3280 crate::assert_with_log!(
3281 third == Some((timed, DispatchLane::Timed)),
3282 "timed task dispatches at deadline",
3283 Some((timed, DispatchLane::Timed)),
3284 third
3285 );
3286
3287 crate::test_complete!("lab_scheduler_pop_for_worker_respects_timed_deadlines");
3288 }
3289
3290 #[test]
3291 fn lab_scheduler_steal_for_worker_only_steals_ready_tasks() {
3292 init_test("lab_scheduler_steal_for_worker_only_steals_ready_tasks");
3293 let mut scheduler = LabScheduler::new(2);
3294 let cancel = TaskId::from_arena(ArenaIndex::new(10, 0));
3295 let timed = TaskId::from_arena(ArenaIndex::new(11, 0));
3296 let ready = TaskId::from_arena(ArenaIndex::new(12, 0));
3297
3298 scheduler.schedule_cancel(cancel, 100);
3300 scheduler.schedule_timed(timed, Time::ZERO);
3301 scheduler.schedule(ready, 50);
3302
3303 let stolen = scheduler.steal_for_worker(1, 0);
3304 crate::assert_with_log!(
3305 stolen == Some(ready),
3306 "steal path takes only ready lane work",
3307 Some(ready),
3308 stolen
3309 );
3310
3311 crate::assert_with_log!(
3312 scheduler.workers[0].has_cancel_work(),
3313 "victim cancel lane remains intact after steal",
3314 true,
3315 scheduler.workers[0].has_cancel_work()
3316 );
3317
3318 let cancel_dispatch = scheduler.pop_for_worker(0, 0, Time::ZERO);
3319 crate::assert_with_log!(
3320 cancel_dispatch == Some((cancel, DispatchLane::Cancel)),
3321 "cancel lane still dispatches from victim worker",
3322 Some((cancel, DispatchLane::Cancel)),
3323 cancel_dispatch
3324 );
3325
3326 let timed_dispatch = scheduler.pop_for_worker(1, 0, Time::ZERO);
3327 crate::assert_with_log!(
3328 timed_dispatch == Some((timed, DispatchLane::Timed)),
3329 "timed lane remains on owning worker",
3330 Some((timed, DispatchLane::Timed)),
3331 timed_dispatch
3332 );
3333
3334 crate::test_complete!("lab_scheduler_steal_for_worker_only_steals_ready_tasks");
3335 }
3336
3337 #[test]
3338 fn lab_scheduler_spurious_wakes_do_not_collapse_duplicates() {
3339 init_test("lab_scheduler_spurious_wakes_do_not_collapse_duplicates");
3340 let mut scheduler = LabScheduler::new(1);
3341 let task = TaskId::from_arena(ArenaIndex::new(13, 0));
3342
3343 scheduler.inject_spurious_wakes(task, 42, 3);
3344
3345 let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3346 crate::assert_with_log!(
3347 first == Some((task, DispatchLane::Ready)),
3348 "first spurious wake dispatches",
3349 Some((task, DispatchLane::Ready)),
3350 first
3351 );
3352
3353 let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3354 crate::assert_with_log!(
3355 second == Some((task, DispatchLane::Ready)),
3356 "second spurious wake remains queued",
3357 Some((task, DispatchLane::Ready)),
3358 second
3359 );
3360
3361 let third = scheduler.pop_for_worker(0, 2, Time::ZERO);
3362 crate::assert_with_log!(
3363 third == Some((task, DispatchLane::Ready)),
3364 "third spurious wake remains queued",
3365 Some((task, DispatchLane::Ready)),
3366 third
3367 );
3368
3369 let fourth = scheduler.pop_for_worker(0, 3, Time::ZERO);
3370 crate::assert_with_log!(
3371 fourth.is_none(),
3372 "storm drains after requested wake count",
3373 true,
3374 fourth.is_none()
3375 );
3376 crate::assert_with_log!(
3377 scheduler.is_empty(),
3378 "scheduler empty after spurious storm drains",
3379 true,
3380 scheduler.is_empty()
3381 );
3382
3383 crate::test_complete!("lab_scheduler_spurious_wakes_do_not_collapse_duplicates");
3384 }
3385
3386 #[test]
3387 fn lab_scheduler_forget_task_clears_pending_spurious_wakes() {
3388 init_test("lab_scheduler_forget_task_clears_pending_spurious_wakes");
3389 let mut scheduler = LabScheduler::new(1);
3390 let task = TaskId::from_arena(ArenaIndex::new(14, 0));
3391
3392 scheduler.inject_spurious_wakes(task, 42, 3);
3393 let first = scheduler.pop_for_worker(0, 0, Time::ZERO);
3394 crate::assert_with_log!(
3395 first == Some((task, DispatchLane::Ready)),
3396 "first spurious wake dispatches before forget",
3397 Some((task, DispatchLane::Ready)),
3398 first
3399 );
3400
3401 scheduler.forget_task(task);
3402
3403 let second = scheduler.pop_for_worker(0, 1, Time::ZERO);
3404 crate::assert_with_log!(
3405 second.is_none(),
3406 "forget_task drains queued spurious wakes",
3407 true,
3408 second.is_none()
3409 );
3410 crate::assert_with_log!(
3411 scheduler.pending_spurious_wakes.is_empty(),
3412 "forget_task clears pending spurious wake budget",
3413 true,
3414 scheduler.pending_spurious_wakes.is_empty()
3415 );
3416 crate::assert_with_log!(
3417 scheduler.is_empty(),
3418 "scheduler empty after forget_task",
3419 true,
3420 scheduler.is_empty()
3421 );
3422
3423 crate::test_complete!("lab_scheduler_forget_task_clears_pending_spurious_wakes");
3424 }
3425
3426 #[test]
3427 fn lab_scheduler_steal_preserves_pending_spurious_wakes() {
3428 init_test("lab_scheduler_steal_preserves_pending_spurious_wakes");
3429 let mut scheduler = LabScheduler::new(2);
3430 let task = TaskId::from_arena(ArenaIndex::new(15, 0));
3431
3432 scheduler.inject_spurious_wakes(task, 42, 3);
3433
3434 let stolen = scheduler.steal_for_worker(1, 0);
3435 crate::assert_with_log!(
3436 stolen == Some(task),
3437 "steal dispatches first storm wake",
3438 Some(task),
3439 stolen
3440 );
3441
3442 let second = scheduler.pop_for_worker(1, 1, Time::ZERO);
3443 crate::assert_with_log!(
3444 second == Some((task, DispatchLane::Ready)),
3445 "steal path re-arms second storm wake on thief worker",
3446 Some((task, DispatchLane::Ready)),
3447 second
3448 );
3449
3450 let third = scheduler.pop_for_worker(1, 2, Time::ZERO);
3451 crate::assert_with_log!(
3452 third == Some((task, DispatchLane::Ready)),
3453 "steal path preserves final pending storm wake",
3454 Some((task, DispatchLane::Ready)),
3455 third
3456 );
3457
3458 let fourth = scheduler.pop_for_worker(1, 3, Time::ZERO);
3459 crate::assert_with_log!(
3460 fourth.is_none(),
3461 "all stolen storm wakes drain after requested count",
3462 true,
3463 fourth.is_none()
3464 );
3465 crate::assert_with_log!(
3466 scheduler.is_empty(),
3467 "scheduler empty after stolen storm drains",
3468 true,
3469 scheduler.is_empty()
3470 );
3471
3472 crate::test_complete!("lab_scheduler_steal_preserves_pending_spurious_wakes");
3473 }
3474
3475 #[test]
3476 fn deterministic_multiworker_schedule() {
3477 init_test("deterministic_multiworker_schedule");
3478 let config = LabConfig::new(7).worker_count(4);
3479
3480 crate::lab::assert_deterministic(config, |runtime| {
3481 let root = runtime.state.create_root_region(Budget::INFINITE);
3482 for _ in 0..4 {
3483 let (task_id, _handle) = runtime
3484 .state
3485 .create_task(root, Budget::INFINITE, async {
3486 crate::runtime::yield_now::yield_now().await;
3487 })
3488 .expect("create task");
3489 runtime.scheduler.lock().schedule(task_id, 0);
3490 }
3491 runtime.run_until_quiescent();
3492 });
3493
3494 crate::test_complete!("deterministic_multiworker_schedule");
3495 }
3496
3497 #[test]
3498 fn run_until_quiescent_with_report_is_deterministic() {
3499 init_test("run_until_quiescent_with_report_is_deterministic");
3500
3501 let config = LabConfig::new(123).worker_count(4).max_steps(10_000);
3502 let mut r1 = LabRuntime::new(config.clone());
3503 let mut r2 = LabRuntime::new(config);
3504
3505 let setup = |runtime: &mut LabRuntime| {
3506 let root = runtime.state.create_root_region(Budget::INFINITE);
3507 for _ in 0..4 {
3508 let (task_id, _handle) = runtime
3509 .state
3510 .create_task(root, Budget::INFINITE, async {
3511 crate::runtime::yield_now::yield_now().await;
3512 })
3513 .expect("create task");
3514 runtime.scheduler.lock().schedule(task_id, 0);
3515 }
3516 };
3517
3518 setup(&mut r1);
3519 setup(&mut r2);
3520
3521 let rep1 = r1.run_until_quiescent_with_report();
3522 let rep2 = r2.run_until_quiescent_with_report();
3523
3524 crate::assert_with_log!(rep1.quiescent, "quiescent", true, rep1.quiescent);
3525 crate::assert_with_log!(rep2.quiescent, "quiescent", true, rep2.quiescent);
3526
3527 assert_eq!(rep1.trace_fingerprint, rep2.trace_fingerprint);
3528 assert_eq!(rep1.trace_certificate, rep2.trace_certificate);
3529 assert_eq!(rep1.oracle_report.to_json(), rep2.oracle_report.to_json());
3530 assert_eq!(rep1.invariant_violations, rep2.invariant_violations);
3531
3532 crate::assert_with_log!(
3533 rep1.oracle_report.all_passed(),
3534 "oracles passed",
3535 true,
3536 rep1.oracle_report.all_passed()
3537 );
3538 crate::assert_with_log!(
3539 rep2.oracle_report.all_passed(),
3540 "oracles passed",
3541 true,
3542 rep2.oracle_report.all_passed()
3543 );
3544
3545 crate::test_complete!("run_until_quiescent_with_report_is_deterministic");
3546 }
3547
3548 #[test]
3549 fn deadline_monitor_emits_warning() {
3550 init_test("deadline_monitor_emits_warning");
3551 let mut runtime = LabRuntime::with_seed(42);
3552
3553 let warnings: Arc<Mutex<Vec<DeadlineWarning>>> = Arc::new(Mutex::new(Vec::new()));
3554 let warnings_clone = Arc::clone(&warnings);
3555
3556 let config = MonitorConfig {
3557 check_interval: Duration::from_secs(0),
3558 warning_threshold_fraction: 1.0,
3559 checkpoint_timeout: Duration::from_secs(0),
3560 adaptive: AdaptiveDeadlineConfig::default(),
3561 enabled: true,
3562 };
3563
3564 runtime.enable_deadline_monitoring_with_handler(config, move |warning| {
3565 warnings_clone.lock().push(warning);
3566 });
3567
3568 let root = runtime.state.create_root_region(Budget::INFINITE);
3569 let budget = Budget::new().with_deadline(Time::from_millis(10));
3570
3571 let task_idx = runtime.state.insert_task(TaskRecord::new_with_time(
3572 TaskId::from_arena(ArenaIndex::new(0, 0)),
3573 root,
3574 budget,
3575 runtime.state.now,
3576 ));
3577 let task_id = TaskId::from_arena(task_idx);
3578 runtime.state.task_mut(task_id).unwrap().id = task_id;
3579
3580 let mut inner = CxInner::new(root, task_id, budget);
3581 inner.checkpoint_state.last_checkpoint = None;
3582 runtime
3583 .state
3584 .task_mut(task_id)
3585 .unwrap()
3586 .set_cx_inner(Arc::new(RwLock::new(inner)));
3587
3588 runtime.step();
3589
3590 let warnings = warnings.lock();
3591 let warning = warnings.first().expect("expected warning");
3592 crate::assert_with_log!(
3593 warning.task_id == task_id,
3594 "task_id",
3595 task_id,
3596 warning.task_id
3597 );
3598 crate::assert_with_log!(
3599 warning.region_id == root,
3600 "region_id",
3601 root,
3602 warning.region_id
3603 );
3604 let ok = matches!(
3605 warning.reason,
3606 WarningReason::ApproachingDeadline | WarningReason::ApproachingDeadlineNoProgress
3607 );
3608 crate::assert_with_log!(ok, "reason", true, ok);
3609 drop(warnings);
3610 crate::test_complete!("deadline_monitor_emits_warning");
3611 }
3612
3613 #[test]
3614 fn deadline_monitor_e2e_stuck_detection() {
3615 init_test("deadline_monitor_e2e_stuck_detection");
3616 let mut runtime = LabRuntime::with_seed(42);
3617
3618 let warnings: Arc<Mutex<Vec<DeadlineWarning>>> = Arc::new(Mutex::new(Vec::new()));
3619 let warnings_clone = Arc::clone(&warnings);
3620
3621 let config = MonitorConfig {
3622 check_interval: Duration::ZERO,
3623 warning_threshold_fraction: 0.0,
3624 checkpoint_timeout: Duration::ZERO,
3625 adaptive: AdaptiveDeadlineConfig::default(),
3626 enabled: true,
3627 };
3628
3629 runtime.enable_deadline_monitoring_with_handler(config, move |warning| {
3630 warnings_clone.lock().push(warning);
3631 });
3632
3633 let root = runtime.state.create_root_region(Budget::INFINITE);
3634 let budget = Budget::new().with_deadline(Time::from_secs(10));
3635 let (task_id, _handle) = runtime
3636 .state
3637 .create_task(root, budget, async {})
3638 .expect("create task");
3639
3640 {
3641 let task = runtime.state.task_mut(task_id).unwrap();
3642 let cx = task.cx.as_ref().expect("task cx");
3643 cx.checkpoint_with("starting work").expect("checkpoint");
3644 }
3645
3646 runtime.step();
3647
3648 let warnings = warnings.lock();
3649 let warning = warnings.first().expect("expected warning");
3650 crate::assert_with_log!(
3651 warning.task_id == task_id,
3652 "task_id",
3653 task_id,
3654 warning.task_id
3655 );
3656 crate::assert_with_log!(
3657 warning.reason == WarningReason::NoProgress,
3658 "reason",
3659 WarningReason::NoProgress,
3660 warning.reason
3661 );
3662 crate::assert_with_log!(
3663 warning.last_checkpoint_message.as_deref() == Some("starting work"),
3664 "checkpoint message",
3665 Some("starting work"),
3666 warning.last_checkpoint_message.as_deref()
3667 );
3668 drop(warnings);
3669 crate::test_complete!("deadline_monitor_e2e_stuck_detection");
3670 }
3671
3672 #[test]
3673 fn futurelock_emits_trace_event() {
3674 init_test("futurelock_emits_trace_event");
3675 let config = LabConfig::new(42)
3676 .futurelock_max_idle_steps(3)
3677 .panic_on_futurelock(false);
3678 let mut runtime = LabRuntime::new(config);
3679
3680 let root = runtime.state.create_root_region(Budget::INFINITE);
3681
3682 let task_idx = runtime.state.insert_task(TaskRecord::new(
3684 TaskId::from_arena(ArenaIndex::new(0, 0)),
3685 root,
3686 Budget::INFINITE,
3687 ));
3688 let task_id = TaskId::from_arena(task_idx);
3689 runtime.state.task_mut(task_id).unwrap().id = task_id;
3690
3691 let obl_id = runtime
3693 .state
3694 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3695 .expect("create obligation");
3696
3697 for _ in 0..4 {
3698 runtime.step();
3699 }
3700
3701 let futurelock = runtime
3702 .trace()
3703 .snapshot()
3704 .into_iter()
3705 .find(|e| e.kind == TraceEventKind::FuturelockDetected)
3706 .expect("expected futurelock trace event");
3707
3708 match &futurelock.data {
3709 TraceData::Futurelock {
3710 task,
3711 region,
3712 idle_steps,
3713 held,
3714 } => {
3715 crate::assert_with_log!(*task == task_id, "task", task_id, *task);
3716 crate::assert_with_log!(*region == root, "region", root, *region);
3717 let idle_ok = *idle_steps > 3;
3718 crate::assert_with_log!(idle_ok, "idle_steps > 3", true, idle_ok);
3719 let ok = held.as_slice() == [(obl_id, ObligationKind::SendPermit)];
3720 crate::assert_with_log!(
3721 ok,
3722 "held",
3723 &[(obl_id, ObligationKind::SendPermit)],
3724 held.as_slice()
3725 );
3726 }
3727 other => panic!("unexpected trace data: {other:?}"),
3728 }
3729 crate::test_complete!("futurelock_emits_trace_event");
3730 }
3731
3732 #[test]
3733 #[should_panic(expected = "futurelock detected")]
3734 fn futurelock_can_panic() {
3735 init_test("futurelock_can_panic");
3736 let config = LabConfig::new(42).futurelock_max_idle_steps(1);
3737 let mut runtime = LabRuntime::new(config);
3738
3739 let root = runtime.state.create_root_region(Budget::INFINITE);
3740
3741 let task_idx = runtime.state.insert_task(TaskRecord::new(
3742 TaskId::from_arena(ArenaIndex::new(0, 0)),
3743 root,
3744 Budget::INFINITE,
3745 ));
3746 let task_id = TaskId::from_arena(task_idx);
3747 runtime.state.task_mut(task_id).unwrap().id = task_id;
3748
3749 let _ = runtime
3750 .state
3751 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3752 .expect("create obligation");
3753
3754 for _ in 0..3 {
3756 runtime.step();
3757 }
3758 }
3759
3760 #[test]
3766 fn polled_task_not_flagged_as_futurelocked() {
3767 init_test("polled_task_not_flagged_as_futurelocked");
3768 let config = LabConfig::new(42)
3769 .futurelock_max_idle_steps(5)
3770 .panic_on_futurelock(false);
3771 let mut runtime = LabRuntime::new(config);
3772
3773 let root = runtime.state.create_root_region(Budget::INFINITE);
3774
3775 let (task_id, _handle) = runtime
3777 .state
3778 .create_task(root, Budget::INFINITE, async {
3779 loop {
3780 crate::runtime::yield_now::yield_now().await;
3781 }
3782 })
3783 .expect("create task");
3784
3785 let _obl = runtime
3787 .state
3788 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3789 .expect("create obligation");
3790
3791 runtime.scheduler.lock().schedule(task_id, 0);
3793 for _ in 0..20 {
3794 runtime.step();
3795 }
3796
3797 let violations = runtime.futurelock_violations();
3799 crate::assert_with_log!(
3800 violations.is_empty(),
3801 "no futurelock for actively polled task",
3802 true,
3803 violations.is_empty()
3804 );
3805 crate::test_complete!("polled_task_not_flagged_as_futurelocked");
3806 }
3807
3808 #[test]
3809 fn immediate_completion_marks_running_before_completion() {
3810 init_test("immediate_completion_marks_running_before_completion");
3811 let mut runtime = LabRuntime::new(LabConfig::new(42));
3812 let root = runtime.state.create_root_region(Budget::INFINITE);
3813 let (task_id, _) = runtime
3814 .state
3815 .create_task(root, Budget::INFINITE, async {})
3816 .expect("create task");
3817 runtime.scheduler.lock().schedule(task_id, 0);
3818
3819 runtime.run_until_quiescent();
3820
3821 let protocol_ok = runtime.check_cancellation_protocol().is_ok();
3822 crate::assert_with_log!(
3823 runtime.is_quiescent(),
3824 "runtime reached quiescence after immediate completion",
3825 true,
3826 runtime.is_quiescent()
3827 );
3828 crate::assert_with_log!(
3829 protocol_ok,
3830 "cancellation oracle accepted Created -> Running -> Completed",
3831 true,
3832 protocol_ok
3833 );
3834
3835 crate::test_complete!("immediate_completion_marks_running_before_completion");
3836 }
3837
3838 #[test]
3839 fn obligation_leak_detected_when_holder_completed() {
3840 init_test("obligation_leak_detected_when_holder_completed");
3841 let mut runtime = LabRuntime::with_seed(7);
3842 let root = runtime.state.create_root_region(Budget::INFINITE);
3843
3844 let task_idx = runtime.state.insert_task(TaskRecord::new(
3845 TaskId::from_arena(ArenaIndex::new(0, 0)),
3846 root,
3847 Budget::INFINITE,
3848 ));
3849 let task_id = TaskId::from_arena(task_idx);
3850 runtime.state.task_mut(task_id).unwrap().id = task_id;
3851
3852 let obl_id = runtime
3853 .state
3854 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
3855 .expect("create obligation");
3856
3857 runtime
3858 .state
3859 .update_task(task_id, |record| record.complete(Outcome::Ok(())))
3860 .unwrap();
3861
3862 let violations = runtime.check_invariants();
3863 let mut found = false;
3864 for violation in violations {
3865 if let InvariantViolation::ObligationLeak { leaks } = violation {
3866 found = true;
3867 let len = leaks.len();
3868 crate::assert_with_log!(len == 1, "leaks len", 1, len);
3869 let leak = &leaks[0];
3870 crate::assert_with_log!(
3871 leak.obligation == obl_id,
3872 "obligation",
3873 obl_id,
3874 leak.obligation
3875 );
3876 crate::assert_with_log!(
3877 leak.kind == ObligationKind::SendPermit,
3878 "kind",
3879 ObligationKind::SendPermit,
3880 leak.kind
3881 );
3882 crate::assert_with_log!(leak.holder == task_id, "holder", task_id, leak.holder);
3883 crate::assert_with_log!(leak.region == root, "region", root, leak.region);
3884 }
3885 }
3886 crate::assert_with_log!(found, "found leak", true, found);
3887 crate::test_complete!("obligation_leak_detected_when_holder_completed");
3888 }
3889
3890 #[test]
3891 fn obligation_leak_ignored_when_resolved() {
3892 init_test("obligation_leak_ignored_when_resolved");
3893 let mut runtime = LabRuntime::with_seed(11);
3894 let root = runtime.state.create_root_region(Budget::INFINITE);
3895
3896 let task_idx = runtime.state.insert_task(TaskRecord::new(
3897 TaskId::from_arena(ArenaIndex::new(0, 0)),
3898 root,
3899 Budget::INFINITE,
3900 ));
3901 let task_id = TaskId::from_arena(task_idx);
3902 runtime.state.task_mut(task_id).unwrap().id = task_id;
3903
3904 let obl_id = runtime
3905 .state
3906 .create_obligation(ObligationKind::Ack, task_id, root, None)
3907 .expect("create obligation");
3908 runtime
3909 .state
3910 .commit_obligation(obl_id)
3911 .expect("commit obligation");
3912
3913 runtime
3914 .state
3915 .update_task(task_id, |record| record.complete(Outcome::Ok(())))
3916 .unwrap();
3917
3918 let violations = runtime.check_invariants();
3919 let has_leak = violations
3920 .iter()
3921 .any(|v| matches!(v, InvariantViolation::ObligationLeak { .. }));
3922 crate::assert_with_log!(!has_leak, "no leak", false, has_leak);
3923 crate::test_complete!("obligation_leak_ignored_when_resolved");
3924 }
3925
3926 #[test]
3927 fn report_hydrates_temporal_oracles_from_state_snapshot() {
3928 init_test("report_hydrates_temporal_oracles_from_state_snapshot");
3929 let mut runtime = LabRuntime::with_seed(31);
3930 let root = runtime.state.create_root_region(Budget::INFINITE);
3931 let (_task, _handle) = runtime
3932 .state
3933 .create_task(root, Budget::INFINITE, async {})
3934 .expect("create task");
3935
3936 runtime
3939 .state
3940 .region(root)
3941 .expect("region exists")
3942 .set_state(crate::record::region::RegionState::Closed);
3943
3944 let report = runtime.report();
3945 let task_leak = report
3946 .oracle_report
3947 .entry("task_leak")
3948 .expect("task_leak entry");
3949 let quiescence = report
3950 .oracle_report
3951 .entry("quiescence")
3952 .expect("quiescence entry");
3953
3954 crate::assert_with_log!(
3955 !task_leak.passed,
3956 "task_leak failed",
3957 false,
3958 task_leak.passed
3959 );
3960 crate::assert_with_log!(
3961 !quiescence.passed,
3962 "quiescence failed",
3963 false,
3964 quiescence.passed
3965 );
3966 let has_temporal_tag = report
3967 .invariant_violations
3968 .iter()
3969 .any(|v| v == "temporal:task_leak");
3970 crate::assert_with_log!(
3971 has_temporal_tag,
3972 "temporal marker present",
3973 true,
3974 has_temporal_tag
3975 );
3976 let temporal_failed = report
3977 .temporal_invariant_failures
3978 .iter()
3979 .any(|v| v == "task_leak");
3980 crate::assert_with_log!(
3981 temporal_failed,
3982 "temporal failure surfaced",
3983 true,
3984 temporal_failed
3985 );
3986 crate::test_complete!("report_hydrates_temporal_oracles_from_state_snapshot");
3987 }
3988
3989 #[test]
3990 fn report_hydrates_quiescence_from_finalizers_and_obligations() {
3991 init_test("report_hydrates_quiescence_from_finalizers_and_obligations");
3992 let mut runtime = LabRuntime::with_seed(32);
3993 let root = runtime.state.create_root_region(Budget::INFINITE);
3994 let (task_id, _handle) = runtime
3995 .state
3996 .create_task(root, Budget::INFINITE, async {})
3997 .expect("create task");
3998
3999 runtime.state.now = Time::from_nanos(10);
4000 let registered = runtime.state.register_sync_finalizer(root, || {});
4001 crate::assert_with_log!(registered, "registered finalizer", true, registered);
4002
4003 runtime.state.now = Time::from_nanos(20);
4004 let _obligation = runtime
4005 .state
4006 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
4007 .expect("create obligation");
4008
4009 runtime.state.now = Time::from_nanos(30);
4010 runtime
4011 .state
4012 .update_task(task_id, |record| record.complete(Outcome::Ok(())))
4013 .expect("complete task without auto-resolving obligation");
4014
4015 runtime.state.now = Time::from_nanos(40);
4016 runtime.state.record_finalizer_close_for_test(root);
4017 runtime
4018 .state
4019 .region(root)
4020 .expect("region exists")
4021 .set_state(crate::record::region::RegionState::Closed);
4022
4023 let report = runtime.report();
4024 let quiescence = report
4025 .oracle_report
4026 .entry("quiescence")
4027 .expect("quiescence entry");
4028 let finalizer = report
4029 .oracle_report
4030 .entry("finalizer")
4031 .expect("finalizer entry");
4032 let obligation_leak = report
4033 .oracle_report
4034 .entry("obligation_leak")
4035 .expect("obligation entry");
4036
4037 crate::assert_with_log!(
4038 !quiescence.passed,
4039 "quiescence failed",
4040 false,
4041 quiescence.passed
4042 );
4043 let quiescence_text = quiescence
4044 .violation
4045 .as_deref()
4046 .expect("quiescence violation text");
4047 crate::assert_with_log!(
4048 quiescence_text.contains("1 unrun finalizers"),
4049 "quiescence mentions finalizers",
4050 true,
4051 quiescence_text.contains("1 unrun finalizers")
4052 );
4053 crate::assert_with_log!(
4054 quiescence_text.contains("1 leaked obligations"),
4055 "quiescence mentions obligations",
4056 true,
4057 quiescence_text.contains("1 leaked obligations")
4058 );
4059 crate::assert_with_log!(
4060 !finalizer.passed,
4061 "finalizer failed",
4062 false,
4063 finalizer.passed
4064 );
4065 crate::assert_with_log!(
4066 !obligation_leak.passed,
4067 "obligation_leak failed",
4068 false,
4069 obligation_leak.passed
4070 );
4071 crate::test_complete!("report_hydrates_quiescence_from_finalizers_and_obligations");
4072 }
4073
4074 #[test]
4075 fn report_hydrates_cancellation_propagation_from_state_snapshot() {
4076 init_test("report_hydrates_cancellation_propagation_from_state_snapshot");
4077 let config = LabConfig::new(32).panic_on_cancellation_violation(false);
4082 let mut runtime = LabRuntime::new(config);
4083 let root = runtime.state.create_root_region(Budget::INFINITE);
4084 let _child = runtime
4085 .state
4086 .create_child_region(root, Budget::INFINITE)
4087 .expect("create child");
4088
4089 runtime
4090 .state
4091 .region(root)
4092 .expect("root exists")
4093 .cancel_request(crate::types::CancelReason::shutdown());
4094
4095 let report = runtime.report();
4096 let cancellation = report
4097 .oracle_report
4098 .entry("cancellation_protocol")
4099 .expect("cancellation_protocol entry");
4100 crate::assert_with_log!(
4101 !cancellation.passed,
4102 "cancellation_protocol failed",
4103 false,
4104 cancellation.passed
4105 );
4106 let has_temporal_tag = report
4107 .invariant_violations
4108 .iter()
4109 .any(|v| v == "temporal:cancellation_protocol");
4110 crate::assert_with_log!(
4111 has_temporal_tag,
4112 "temporal cancellation marker present",
4113 true,
4114 has_temporal_tag
4115 );
4116 crate::test_complete!("report_hydrates_cancellation_propagation_from_state_snapshot");
4117 }
4118
4119 #[test]
4120 fn report_surfaces_refinement_firewall_violation_from_trace_snapshot() {
4121 init_test("report_surfaces_refinement_firewall_violation_from_trace_snapshot");
4122 let mut runtime = LabRuntime::with_seed(33);
4123 let region = RegionId::new_for_test(41, 0);
4124 let task = TaskId::new_for_test(7, 0);
4125
4126 runtime
4127 .state
4128 .trace
4129 .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4130 runtime
4131 .state
4132 .trace
4133 .push_event(TraceEvent::spawn(2, Time::ZERO, task, region));
4134
4135 let report = runtime.report();
4136 crate::assert_with_log!(
4137 report.refinement_firewall_rule_id.as_deref() == Some("RFW-SPAWN-001"),
4138 "refinement rule id surfaced",
4139 Some("RFW-SPAWN-001"),
4140 report.refinement_firewall_rule_id.as_deref()
4141 );
4142 crate::assert_with_log!(
4143 report.refinement_firewall_event_index == Some(1),
4144 "refinement event index surfaced",
4145 Some(1usize),
4146 report.refinement_firewall_event_index
4147 );
4148 crate::assert_with_log!(
4149 report.refinement_counterexample_prefix_len == Some(2),
4150 "refinement prefix len surfaced",
4151 Some(2usize),
4152 report.refinement_counterexample_prefix_len
4153 );
4154 let has_marker = report
4155 .invariant_violations
4156 .iter()
4157 .any(|v| v == "refinement_firewall:RFW-SPAWN-001");
4158 crate::assert_with_log!(
4159 has_marker,
4160 "refinement invariant marker present",
4161 true,
4162 has_marker
4163 );
4164 let json = report.to_json();
4165 crate::assert_with_log!(
4166 json["refinement_firewall"]["rule_id"] == "RFW-SPAWN-001",
4167 "refinement json rule id",
4168 "RFW-SPAWN-001",
4169 json["refinement_firewall"]["rule_id"]
4170 );
4171 crate::assert_with_log!(
4172 json["refinement_firewall"]["counterexample_prefix_len"] == 2,
4173 "refinement json prefix len",
4174 2,
4175 json["refinement_firewall"]["counterexample_prefix_len"]
4176 );
4177 crate::assert_with_log!(
4178 json["refinement_firewall"]["skipped_due_to_trace_truncation"] == false,
4179 "refinement json not skipped",
4180 false,
4181 json["refinement_firewall"]["skipped_due_to_trace_truncation"]
4182 );
4183 crate::test_complete!("report_surfaces_refinement_firewall_violation_from_trace_snapshot");
4184 }
4185
4186 #[test]
4197 fn report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0() {
4198 init_test("report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0");
4199 let seed = 35;
4200 let config = LabConfig::new(seed).trace_capacity(1);
4201 let mut runtime = LabRuntime::new(config);
4202 let region = RegionId::new_for_test(43, 0);
4203 let task = TaskId::new_for_test(9, 0);
4204
4205 runtime
4206 .state
4207 .trace
4208 .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4209 runtime
4210 .state
4211 .trace
4212 .push_event(TraceEvent::complete(2, Time::ZERO, task, region));
4213
4214 let report = runtime.report();
4215 crate::assert_with_log!(
4216 report.refinement_firewall_skipped_due_to_trace_truncation,
4217 "refinement_firewall_skipped_due_to_trace_truncation",
4218 true,
4219 report.refinement_firewall_skipped_due_to_trace_truncation
4220 );
4221 crate::assert_with_log!(
4222 report.refinement_firewall_rule_id.is_none(),
4223 "no real rule_id when firewall could not run",
4224 true,
4225 report.refinement_firewall_rule_id.is_none()
4226 );
4227
4228 let truncation_marker = report
4233 .invariant_violations
4234 .iter()
4235 .find(|v| v.starts_with("refinement_firewall:scenario_failed_due_to_trace_truncation"));
4236 let marker = truncation_marker.expect(
4237 "truncation must surface as a hard refinement_firewall:scenario_failed_due_to_trace_truncation marker",
4238 );
4239 assert!(
4240 marker.contains(&format!("seed={seed}")),
4241 "truncation marker must embed the seed, got: {marker}"
4242 );
4243 assert!(
4244 marker.contains("total_pushed=") && marker.contains("buffered="),
4245 "truncation marker must embed total_pushed + buffered watermark, got: {marker}"
4246 );
4247 crate::assert_with_log!(
4250 !report.invariant_violations.is_empty(),
4251 "scenario must fail loudly when trace buffer truncates",
4252 true,
4253 !report.invariant_violations.is_empty()
4254 );
4255
4256 let json = report.to_json();
4257 crate::assert_with_log!(
4258 json["refinement_firewall"]["skipped_due_to_trace_truncation"] == true,
4259 "skipped flag still serialized for downstream tooling",
4260 true,
4261 json["refinement_firewall"]["skipped_due_to_trace_truncation"]
4262 );
4263 crate::test_complete!("report_fails_loudly_when_trace_buffer_is_truncated_l9ri7x0");
4264 }
4265
4266 #[test]
4273 fn inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa() {
4274 init_test("inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa");
4275
4276 let mut runtime = LabRuntime::with_seed(42);
4277 let region = runtime.state.create_root_region(Budget::INFINITE);
4278 let task_idx = runtime.state.insert_task(TaskRecord::new_with_time(
4279 TaskId::from_arena(ArenaIndex::new(0, 0)),
4280 region,
4281 Budget::INFINITE,
4282 runtime.state.now,
4283 ));
4284 let task_id = TaskId::from_arena(task_idx);
4285 runtime.state.task_mut(task_id).unwrap().id = task_id;
4286
4287 runtime.inject_spurious_wakes(task_id, 0, 2);
4289 let scheduled_open = {
4290 let mut sched = runtime.scheduler.lock();
4291 let mut count = 0u32;
4292 while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4293 count += 1;
4294 }
4295 count
4296 };
4297 crate::assert_with_log!(
4298 scheduled_open == 2,
4299 "while region is Open, 2 spurious wakes are scheduled",
4300 2u32,
4301 scheduled_open
4302 );
4303
4304 runtime
4306 .state
4307 .region(region)
4308 .expect("region exists")
4309 .set_state(crate::record::region::RegionState::Closing);
4310
4311 runtime.inject_spurious_wakes(task_id, 0, 5);
4313 let scheduled_closing = {
4314 let mut sched = runtime.scheduler.lock();
4315 let mut count = 0u32;
4316 while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4317 count += 1;
4318 }
4319 count
4320 };
4321 crate::assert_with_log!(
4322 scheduled_closing == 0,
4323 "after region.set_state(Closing), spurious wakes are suppressed",
4324 0u32,
4325 scheduled_closing
4326 );
4327
4328 for state in [
4330 crate::record::region::RegionState::Draining,
4331 crate::record::region::RegionState::Closed,
4332 ] {
4333 runtime
4334 .state
4335 .region(region)
4336 .expect("region exists")
4337 .set_state(state);
4338 runtime.inject_spurious_wakes(task_id, 0, 3);
4339 let scheduled = {
4340 let mut sched = runtime.scheduler.lock();
4341 let mut count = 0u32;
4342 while sched.pop_for_worker(0, count.into(), Time::ZERO).is_some() {
4343 count += 1;
4344 }
4345 count
4346 };
4347 assert_eq!(
4348 scheduled, 0,
4349 "spurious wakes must be suppressed in region state {state:?}"
4350 );
4351 }
4352
4353 crate::test_complete!(
4354 "inject_spurious_wakes_suppressed_when_owning_region_is_closing_l7uu7sa"
4355 );
4356 }
4357
4358 #[test]
4359 fn crashpack_includes_refinement_firewall_markers() {
4360 init_test("crashpack_includes_refinement_firewall_markers");
4361 let mut runtime = LabRuntime::with_seed(34);
4362 let region = RegionId::new_for_test(42, 0);
4363 let task = TaskId::new_for_test(8, 0);
4364
4365 runtime
4366 .state
4367 .trace
4368 .push_event(TraceEvent::spawn(1, Time::ZERO, task, region));
4369 runtime
4370 .state
4371 .trace
4372 .push_event(TraceEvent::spawn(2, Time::ZERO, task, region));
4373
4374 let run = runtime.report();
4375 let crashpack = runtime
4376 .build_crashpack_for_report(&run)
4377 .expect("refinement-firewall failure should build crashpack");
4378 let has_rule_marker = crashpack
4379 .oracle_violations
4380 .iter()
4381 .any(|entry| entry == "refinement_firewall:RFW-SPAWN-001");
4382 crate::assert_with_log!(
4383 has_rule_marker,
4384 "crashpack includes refinement rule marker",
4385 true,
4386 has_rule_marker
4387 );
4388 let has_prefix_marker = crashpack
4389 .oracle_violations
4390 .iter()
4391 .any(|entry| entry == "refinement_firewall:minimal_counterexample_prefix_len=2");
4392 crate::assert_with_log!(
4393 has_prefix_marker,
4394 "crashpack includes refinement prefix marker",
4395 true,
4396 has_prefix_marker
4397 );
4398 crate::test_complete!("crashpack_includes_refinement_firewall_markers");
4399 }
4400
4401 #[test]
4402 #[allow(clippy::too_many_lines)]
4403 fn obligation_trace_events_emitted() {
4404 init_test("obligation_trace_events_emitted");
4405 let mut runtime = LabRuntime::with_seed(21);
4406 let root = runtime.state.create_root_region(Budget::INFINITE);
4407
4408 let task_idx = runtime.state.insert_task(TaskRecord::new(
4409 TaskId::from_arena(ArenaIndex::new(0, 0)),
4410 root,
4411 Budget::INFINITE,
4412 ));
4413 let task_id = TaskId::from_arena(task_idx);
4414 runtime.state.task_mut(task_id).unwrap().id = task_id;
4415
4416 runtime.advance_time_to(Time::from_nanos(10));
4417 let ob1 = runtime
4418 .state
4419 .create_obligation(ObligationKind::SendPermit, task_id, root, None)
4420 .unwrap();
4421
4422 runtime.advance_time_to(Time::from_nanos(25));
4423 runtime.state.commit_obligation(ob1).unwrap();
4424
4425 runtime.advance_time_to(Time::from_nanos(30));
4426 let ob2 = runtime
4427 .state
4428 .create_obligation(ObligationKind::Ack, task_id, root, None)
4429 .unwrap();
4430
4431 runtime.advance_time_to(Time::from_nanos(50));
4432 runtime
4433 .state
4434 .abort_obligation(ob2, ObligationAbortReason::Cancel)
4435 .unwrap();
4436
4437 let commit_event = runtime
4438 .trace()
4439 .snapshot()
4440 .into_iter()
4441 .find(|e| e.kind == TraceEventKind::ObligationCommit)
4442 .expect("commit event");
4443 match &commit_event.data {
4444 TraceData::Obligation {
4445 obligation,
4446 task,
4447 region,
4448 kind,
4449 state,
4450 duration_ns,
4451 abort_reason,
4452 } => {
4453 crate::assert_with_log!(*obligation == ob1, "obligation", ob1, *obligation);
4454 crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4455 crate::assert_with_log!(*region == root, "region", root, *region);
4456 crate::assert_with_log!(
4457 *kind == ObligationKind::SendPermit,
4458 "kind",
4459 ObligationKind::SendPermit,
4460 *kind
4461 );
4462 crate::assert_with_log!(
4463 *state == crate::record::ObligationState::Committed,
4464 "state",
4465 crate::record::ObligationState::Committed,
4466 *state
4467 );
4468 crate::assert_with_log!(
4469 duration_ns == &Some(15),
4470 "duration",
4471 &Some(15),
4472 duration_ns
4473 );
4474 crate::assert_with_log!(
4475 abort_reason.is_none(),
4476 "abort_reason",
4477 &None::<crate::record::ObligationAbortReason>,
4478 abort_reason
4479 );
4480 }
4481 other => panic!("unexpected commit data: {other:?}"),
4482 }
4483
4484 let abort_event = runtime
4485 .trace()
4486 .snapshot()
4487 .into_iter()
4488 .find(|e| e.kind == TraceEventKind::ObligationAbort)
4489 .expect("abort event");
4490 match &abort_event.data {
4491 TraceData::Obligation {
4492 obligation,
4493 task,
4494 region,
4495 kind,
4496 state,
4497 duration_ns,
4498 abort_reason,
4499 } => {
4500 crate::assert_with_log!(*obligation == ob2, "obligation", ob2, *obligation);
4501 crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4502 crate::assert_with_log!(*region == root, "region", root, *region);
4503 crate::assert_with_log!(
4504 *kind == ObligationKind::Ack,
4505 "kind",
4506 ObligationKind::Ack,
4507 *kind
4508 );
4509 crate::assert_with_log!(
4510 *state == crate::record::ObligationState::Aborted,
4511 "state",
4512 crate::record::ObligationState::Aborted,
4513 *state
4514 );
4515 crate::assert_with_log!(
4516 duration_ns == &Some(20),
4517 "duration",
4518 &Some(20),
4519 duration_ns
4520 );
4521 crate::assert_with_log!(
4522 abort_reason == &Some(ObligationAbortReason::Cancel),
4523 "abort_reason",
4524 &Some(ObligationAbortReason::Cancel),
4525 abort_reason
4526 );
4527 }
4528 other => panic!("unexpected abort data: {other:?}"),
4529 }
4530 crate::test_complete!("obligation_trace_events_emitted");
4531 }
4532
4533 #[test]
4534 fn obligation_leak_emits_trace_event() {
4535 init_test("obligation_leak_emits_trace_event");
4536 let mut runtime = LabRuntime::with_seed(22);
4537 let root = runtime.state.create_root_region(Budget::INFINITE);
4538
4539 let task_idx = runtime.state.insert_task(TaskRecord::new(
4540 TaskId::from_arena(ArenaIndex::new(0, 0)),
4541 root,
4542 Budget::INFINITE,
4543 ));
4544 let task_id = TaskId::from_arena(task_idx);
4545 runtime.state.task_mut(task_id).unwrap().id = task_id;
4546
4547 runtime.advance_time_to(Time::from_nanos(100));
4548 let obligation = runtime
4549 .state
4550 .create_obligation(ObligationKind::Lease, task_id, root, None)
4551 .unwrap();
4552
4553 runtime.advance_time_to(Time::from_nanos(140));
4554 runtime
4555 .state
4556 .update_task(task_id, |record| record.complete(Outcome::Ok(())))
4557 .unwrap();
4558
4559 let violations = runtime.check_invariants();
4560 let has_leak = violations
4561 .iter()
4562 .any(|v| matches!(v, InvariantViolation::ObligationLeak { .. }));
4563 crate::assert_with_log!(has_leak, "has leak", true, has_leak);
4564
4565 let leak_event = runtime
4566 .trace()
4567 .snapshot()
4568 .into_iter()
4569 .find(|e| e.kind == TraceEventKind::ObligationLeak)
4570 .expect("leak event");
4571 match &leak_event.data {
4572 TraceData::Obligation {
4573 obligation: leaked,
4574 task,
4575 region,
4576 kind,
4577 state,
4578 duration_ns,
4579 abort_reason,
4580 } => {
4581 crate::assert_with_log!(*leaked == obligation, "obligation", obligation, *leaked);
4582 crate::assert_with_log!(*task == task_id, "task", task_id, *task);
4583 crate::assert_with_log!(*region == root, "region", root, *region);
4584 crate::assert_with_log!(
4585 *kind == ObligationKind::Lease,
4586 "kind",
4587 ObligationKind::Lease,
4588 *kind
4589 );
4590 crate::assert_with_log!(
4591 *state == crate::record::ObligationState::Leaked,
4592 "state",
4593 crate::record::ObligationState::Leaked,
4594 *state
4595 );
4596 crate::assert_with_log!(
4597 duration_ns == &Some(40),
4598 "duration",
4599 &Some(40),
4600 duration_ns
4601 );
4602 crate::assert_with_log!(
4603 abort_reason.is_none(),
4604 "abort_reason",
4605 &None::<crate::record::ObligationAbortReason>,
4606 abort_reason
4607 );
4608 }
4609 other => panic!("unexpected leak data: {other:?}"),
4610 }
4611 crate::test_complete!("obligation_leak_emits_trace_event");
4612 }
4613
4614 #[test]
4620 fn contract_json_has_required_top_level_keys() {
4621 crate::test_utils::init_test_logging();
4622 crate::test_phase!("contract_json_has_required_top_level_keys");
4623
4624 let app = crate::app::AppSpec::new("contract_test");
4625 let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4626 let report = harness.run_to_report().unwrap();
4627 let json = report.to_json();
4628
4629 let required_keys = [
4631 "schema_version",
4632 "verdict",
4633 "app",
4634 "lab",
4635 "fingerprints",
4636 "run",
4637 "crashpack",
4638 "attachments",
4639 ];
4640 for key in &required_keys {
4641 assert!(
4642 json.get(key).is_some(),
4643 "missing required top-level key: {key}"
4644 );
4645 }
4646
4647 assert!(json["app"]["name"].is_string(), "app.name must be a string");
4649 assert!(
4650 json["lab"]["config"].is_object(),
4651 "lab.config must be an object"
4652 );
4653 assert!(
4654 json["lab"]["config_hash"].is_u64(),
4655 "lab.config_hash must be a u64"
4656 );
4657 assert!(
4658 json["fingerprints"]["trace"].is_u64(),
4659 "fingerprints.trace must be a u64"
4660 );
4661 assert!(
4662 json["fingerprints"]["event_hash"].is_u64(),
4663 "fingerprints.event_hash must be a u64"
4664 );
4665 assert!(
4666 json["fingerprints"]["event_count"].is_u64(),
4667 "fingerprints.event_count must be a u64"
4668 );
4669 assert!(
4670 json["fingerprints"]["schedule_hash"].is_u64(),
4671 "fingerprints.schedule_hash must be a u64"
4672 );
4673 assert!(json["run"]["seed"].is_u64(), "run.seed must be a u64");
4674 assert!(
4675 json["run"]["oracles"].is_object(),
4676 "run.oracles must be an object"
4677 );
4678 assert!(
4679 json["run"]["invariants"].is_array(),
4680 "run.invariants must be an array"
4681 );
4682 assert!(
4683 json["run"]["refinement_firewall"].is_object(),
4684 "run.refinement_firewall must be an object"
4685 );
4686 assert!(
4687 json["run"]["refinement_firewall"]["skipped_due_to_trace_truncation"].is_boolean(),
4688 "run.refinement_firewall.skipped_due_to_trace_truncation must be a boolean"
4689 );
4690 assert!(
4691 json["attachments"].is_array(),
4692 "attachments must be an array"
4693 );
4694 assert!(
4695 json["crashpack"].is_null(),
4696 "passing runs should have null crashpack linkage"
4697 );
4698
4699 crate::test_complete!("contract_json_has_required_top_level_keys");
4700 }
4701
4702 #[test]
4704 fn contract_schema_version_is_current() {
4705 crate::test_utils::init_test_logging();
4706 crate::test_phase!("contract_schema_version_is_current");
4707
4708 let app = crate::app::AppSpec::new("version_test");
4709 let harness = crate::lab::SporkAppHarness::with_seed(1, app).unwrap();
4710 let report = harness.run_to_report().unwrap();
4711
4712 assert_eq!(report.schema_version, SporkHarnessReport::SCHEMA_VERSION);
4713 assert_eq!(
4714 report.to_json()["schema_version"],
4715 SporkHarnessReport::SCHEMA_VERSION
4716 );
4717
4718 crate::test_complete!("contract_schema_version_is_current");
4719 }
4720
4721 #[test]
4723 fn contract_config_hash_deterministic() {
4724 crate::test_utils::init_test_logging();
4725 crate::test_phase!("contract_config_hash_deterministic");
4726
4727 let config = LabConfig::new(42);
4728 let summary_a = LabConfigSummary::from_config(&config);
4729 let summary_b = LabConfigSummary::from_config(&config);
4730
4731 assert_eq!(summary_a.config_hash(), summary_b.config_hash());
4732
4733 let config_2 = LabConfig::new(99);
4735 let summary_c = LabConfigSummary::from_config(&config_2);
4736 assert_ne!(summary_a.config_hash(), summary_c.config_hash());
4737
4738 crate::test_complete!("contract_config_hash_deterministic");
4739 }
4740
4741 #[test]
4743 fn contract_verdict_reflects_oracle_state() {
4744 crate::test_utils::init_test_logging();
4745 crate::test_phase!("contract_verdict_reflects_oracle_state");
4746
4747 let app = crate::app::AppSpec::new("verdict_test");
4748 let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4749 let report = harness.run_to_report().unwrap();
4750
4751 assert!(report.passed());
4753 assert_eq!(report.to_json()["verdict"], "pass");
4754 assert!(report.summary_line().starts_with("[PASS]"));
4755
4756 crate::test_complete!("contract_verdict_reflects_oracle_state");
4757 }
4758
4759 #[test]
4761 fn contract_convenience_methods_consistent() {
4762 crate::test_utils::init_test_logging();
4763 crate::test_phase!("contract_convenience_methods_consistent");
4764
4765 let app = crate::app::AppSpec::new("ux_test");
4766 let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4767 let report = harness.run_to_report().unwrap();
4768 let json = report.to_json();
4769
4770 assert_eq!(
4772 report.trace_fingerprint(),
4773 json["fingerprints"]["trace"].as_u64().unwrap()
4774 );
4775
4776 assert_eq!(report.seed(), json["run"]["seed"].as_u64().unwrap());
4778
4779 assert_eq!(
4781 report.config_hash(),
4782 json["lab"]["config_hash"].as_u64().unwrap()
4783 );
4784
4785 assert!(report.crashpack_path().is_none());
4787
4788 assert!(report.oracle_failures().is_empty());
4790
4791 crate::test_complete!("contract_convenience_methods_consistent");
4792 }
4793
4794 #[test]
4796 fn contract_auto_crashpack_on_failure() {
4797 crate::test_utils::init_test_logging();
4798 crate::test_phase!("contract_auto_crashpack_on_failure");
4799
4800 let config = LabConfig::new(17).panic_on_leak(false);
4801 let mut runtime = LabRuntime::new(config);
4802 let region = runtime.state.create_root_region(Budget::INFINITE);
4803 let (task, _) = runtime
4804 .state
4805 .create_task(region, Budget::INFINITE, async {})
4806 .expect("create task");
4807 runtime.scheduler.lock().schedule(task, 0);
4808 runtime
4811 .state
4812 .create_obligation(
4813 ObligationKind::SendPermit,
4814 task,
4815 region,
4816 Some("intentional leak".to_string()),
4817 )
4818 .expect("create obligation");
4819 runtime.run_until_quiescent();
4820
4821 let report = runtime.spork_report("failing_app", Vec::new());
4822 assert!(!report.passed(), "failing run must not report PASS");
4823 let crashpack_path = report
4824 .crashpack_path()
4825 .expect("failing run should include crashpack attachment");
4826 assert!(
4827 crashpack_path.starts_with("crashpack-"),
4828 "unexpected crashpack path: {crashpack_path}"
4829 );
4830 assert!(
4831 report
4832 .attachments
4833 .iter()
4834 .any(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack),
4835 "crashpack attachment kind must be present"
4836 );
4837 let crashpack_link = report
4838 .crashpack_link()
4839 .expect("failing run should expose crashpack link metadata");
4840 assert_eq!(crashpack_link.path, crashpack_path);
4841 assert_eq!(crashpack_link.fingerprint, report.trace_fingerprint());
4842 assert!(
4843 crashpack_link.id.starts_with("crashpack-"),
4844 "unexpected crashpack id: {}",
4845 crashpack_link.id
4846 );
4847 assert!(
4848 crashpack_link.replay.command_line.contains(crashpack_path),
4849 "replay command should include crashpack path"
4850 );
4851
4852 crate::test_complete!("contract_auto_crashpack_on_failure");
4853 }
4854
4855 #[test]
4858 fn contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled() {
4859 crate::test_utils::init_test_logging();
4860 crate::test_phase!("contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled");
4861
4862 let config = LabConfig::new(1701)
4863 .panic_on_leak(false)
4864 .with_default_replay_recording();
4865 let mut runtime = LabRuntime::new(config);
4866 let region = runtime.state.create_root_region(Budget::INFINITE);
4867 let (task, _) = runtime
4868 .state
4869 .create_task(region, Budget::INFINITE, async {})
4870 .expect("create task");
4871 runtime.scheduler.lock().schedule(task, 0);
4872 runtime
4873 .state
4874 .create_obligation(
4875 ObligationKind::SendPermit,
4876 task,
4877 region,
4878 Some("intentional leak".to_string()),
4879 )
4880 .expect("create obligation");
4881 runtime.run_until_quiescent();
4882
4883 let run = runtime.report();
4884 let crashpack = runtime
4885 .build_crashpack_for_report(&run)
4886 .expect("failing run should build crashpack");
4887
4888 assert!(crashpack.has_divergent_prefix());
4889 assert!(
4890 crashpack
4891 .manifest
4892 .has_attachment(&crate::trace::crashpack::AttachmentKind::DivergentPrefix),
4893 "manifest must include divergent prefix attachment"
4894 );
4895 assert!(
4896 crashpack.replay.is_some(),
4897 "crashpack should carry replay command metadata"
4898 );
4899
4900 crate::test_complete!(
4901 "contract_auto_crashpack_contains_divergent_prefix_when_replay_enabled"
4902 );
4903 }
4904
4905 #[test]
4907 fn contract_manual_crashpack_not_duplicated_on_failure() {
4908 crate::test_utils::init_test_logging();
4909 crate::test_phase!("contract_manual_crashpack_not_duplicated_on_failure");
4910
4911 let config = LabConfig::new(18).panic_on_leak(false);
4912 let mut runtime = LabRuntime::new(config);
4913 let region = runtime.state.create_root_region(Budget::INFINITE);
4914 let (task, _) = runtime
4915 .state
4916 .create_task(region, Budget::INFINITE, async {})
4917 .expect("create task");
4918 runtime.scheduler.lock().schedule(task, 0);
4919 runtime
4920 .state
4921 .create_obligation(
4922 ObligationKind::SendPermit,
4923 task,
4924 region,
4925 Some("intentional leak".to_string()),
4926 )
4927 .expect("create obligation");
4928 runtime.run_until_quiescent();
4929
4930 let report = runtime.spork_report(
4931 "failing_app_manual",
4932 vec![HarnessAttachmentRef::crashpack("manual-crashpack.json")],
4933 );
4934 let crashpack_count = report
4935 .attachments
4936 .iter()
4937 .filter(|attachment| attachment.kind == HarnessAttachmentKind::CrashPack)
4938 .count();
4939 assert_eq!(
4940 crashpack_count, 1,
4941 "manual crashpack should not be duplicated"
4942 );
4943 assert_eq!(report.crashpack_path(), Some("manual-crashpack.json"));
4944 let crashpack_link = report
4945 .crashpack_link()
4946 .expect("manual crashpack should still produce metadata");
4947 assert_eq!(crashpack_link.path, "manual-crashpack.json");
4948 assert!(
4949 crashpack_link
4950 .replay
4951 .command_line
4952 .contains("manual-crashpack.json"),
4953 "replay command should include manual crashpack path"
4954 );
4955
4956 crate::test_complete!("contract_manual_crashpack_not_duplicated_on_failure");
4957 }
4958
4959 #[test]
4961 fn contract_json_deterministic_same_seed() {
4962 crate::test_utils::init_test_logging();
4963 crate::test_phase!("contract_json_deterministic_same_seed");
4964
4965 let json_a = {
4966 let app = crate::app::AppSpec::new("det_contract");
4967 let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4968 harness.run_to_report().unwrap().to_json()
4969 };
4970
4971 let json_b = {
4972 let app = crate::app::AppSpec::new("det_contract");
4973 let harness = crate::lab::SporkAppHarness::with_seed(42, app).unwrap();
4974 harness.run_to_report().unwrap().to_json()
4975 };
4976
4977 fn strip_event_hash(obj: &mut serde_json::Map<String, serde_json::Value>) {
4985 if obj.contains_key("event_hash") {
4986 obj.insert("event_hash".into(), serde_json::Value::Null);
4987 }
4988 for val in obj.values_mut() {
4989 if let Some(sub) = val.as_object_mut() {
4990 strip_event_hash(sub);
4991 }
4992 }
4993 }
4994 let normalize = |mut v: serde_json::Value| -> serde_json::Value {
4995 if let Some(obj) = v.as_object_mut() {
4996 strip_event_hash(obj);
4997 }
4998 v
4999 };
5000
5001 assert_eq!(
5002 normalize(json_a),
5003 normalize(json_b),
5004 "same seed must produce identical JSON (mod sequential event_hash)",
5005 );
5006
5007 crate::test_complete!("contract_json_deterministic_same_seed");
5008 }
5009
5010 #[test]
5012 fn contract_attachments_sorted_in_json() {
5013 crate::test_utils::init_test_logging();
5014 crate::test_phase!("contract_attachments_sorted_in_json");
5015
5016 let app = crate::app::AppSpec::new("attach_contract");
5017 let mut harness = crate::lab::SporkAppHarness::with_seed(7, app).unwrap();
5018 harness.attach(HarnessAttachmentRef::trace("z_trace.json"));
5020 harness.attach(HarnessAttachmentRef::crashpack("a_crash.tar"));
5021
5022 let report = harness.run_to_report().unwrap();
5023
5024 assert_eq!(report.crashpack_path(), Some("a_crash.tar"));
5026
5027 let json = report.to_json();
5028 let attachments = json["attachments"].as_array().unwrap();
5029 assert_eq!(attachments.len(), 2);
5030
5031 assert_eq!(attachments[0]["kind"], "crashpack");
5033 assert_eq!(attachments[1]["kind"], "trace");
5034
5035 crate::test_complete!("contract_attachments_sorted_in_json");
5036 }
5037
5038 #[test]
5043 fn advance_to_next_timer_empty() {
5044 init_test("advance_to_next_timer_empty");
5045 let mut runtime = LabRuntime::with_seed(42);
5046
5047 let wakeups = runtime.advance_to_next_timer();
5048 crate::assert_with_log!(wakeups == 0, "no timers → 0 wakeups", 0, wakeups);
5049
5050 let deadline = runtime.next_timer_deadline();
5051 crate::assert_with_log!(
5052 deadline.is_none(),
5053 "no pending deadline",
5054 true,
5055 deadline.is_none()
5056 );
5057 crate::test_complete!("advance_to_next_timer_empty");
5058 }
5059
5060 #[test]
5061 fn advance_to_next_timer_fires_timer() {
5062 init_test("advance_to_next_timer_fires_timer");
5063 let mut runtime = LabRuntime::with_seed(42);
5064
5065 let timer_handle = runtime.state.timer_driver_handle().unwrap();
5067 let woken = Arc::new(std::sync::atomic::AtomicBool::new(false));
5068
5069 let waker = Waker::from(Arc::new(FlagWaker(woken.clone())));
5070 let _ = timer_handle.register(Time::from_secs(1), waker);
5071
5072 let count = runtime.pending_timer_count();
5074 crate::assert_with_log!(count == 1, "1 pending timer", 1, count);
5075
5076 let wakeups = runtime.advance_to_next_timer();
5078 crate::assert_with_log!(wakeups == 1, "1 wakeup", 1, wakeups);
5079
5080 let now = runtime.now();
5082 crate::assert_with_log!(
5083 now == Time::from_secs(1),
5084 "time at 1s",
5085 Time::from_secs(1),
5086 now
5087 );
5088
5089 let was_woken = woken.load(std::sync::atomic::Ordering::SeqCst);
5091 crate::assert_with_log!(was_woken, "waker fired", true, was_woken);
5092 crate::test_complete!("advance_to_next_timer_fires_timer");
5093 }
5094
5095 #[test]
5096 fn metamorphic_timer_registration_permutation_preserves_virtual_time_progression() {
5097 init_test("metamorphic_timer_registration_permutation_preserves_virtual_time_progression");
5098
5099 let baseline = collect_timer_advances(&[5, 1, 3, 1, 8], &[]);
5100 let permuted = collect_timer_advances(&[1, 8, 5, 1, 3], &[]);
5101
5102 crate::assert_with_log!(
5103 baseline.advance_points == permuted.advance_points,
5104 "deadline multiset permutation preserves advance points",
5105 &baseline.advance_points,
5106 &permuted.advance_points
5107 );
5108 crate::assert_with_log!(
5109 baseline.total_wakeups == permuted.total_wakeups,
5110 "deadline multiset permutation preserves wakeup count",
5111 baseline.total_wakeups,
5112 permuted.total_wakeups
5113 );
5114 crate::assert_with_log!(
5115 baseline.final_time == permuted.final_time,
5116 "deadline multiset permutation preserves final virtual time",
5117 baseline.final_time,
5118 permuted.final_time
5119 );
5120 crate::assert_with_log!(
5121 baseline.advance_points
5122 == vec![
5123 Time::from_secs(1),
5124 Time::from_secs(3),
5125 Time::from_secs(5),
5126 Time::from_secs(8)
5127 ],
5128 "advance points collapse duplicate deadlines without moving backward",
5129 vec![
5130 Time::from_secs(1),
5131 Time::from_secs(3),
5132 Time::from_secs(5),
5133 Time::from_secs(8)
5134 ],
5135 baseline.advance_points.clone()
5136 );
5137
5138 crate::test_complete!(
5139 "metamorphic_timer_registration_permutation_preserves_virtual_time_progression"
5140 );
5141 }
5142
5143 #[test]
5144 fn metamorphic_cancelled_timer_does_not_skew_virtual_time_progression() {
5145 init_test("metamorphic_cancelled_timer_does_not_skew_virtual_time_progression");
5146
5147 let baseline = collect_timer_advances(&[2, 4, 9], &[]);
5148 let with_cancelled_timer = collect_timer_advances(&[2, 4, 6, 9], &[2]);
5149
5150 crate::assert_with_log!(
5151 baseline.advance_points == with_cancelled_timer.advance_points,
5152 "cancelling an intermediate timer preserves surviving advance points",
5153 &baseline.advance_points,
5154 &with_cancelled_timer.advance_points
5155 );
5156 crate::assert_with_log!(
5157 baseline.total_wakeups == with_cancelled_timer.total_wakeups,
5158 "cancelling an intermediate timer preserves surviving wakeup count",
5159 baseline.total_wakeups,
5160 with_cancelled_timer.total_wakeups
5161 );
5162 crate::assert_with_log!(
5163 baseline.final_time == with_cancelled_timer.final_time,
5164 "cancelling an intermediate timer preserves final virtual time",
5165 baseline.final_time,
5166 with_cancelled_timer.final_time
5167 );
5168 crate::assert_with_log!(
5169 with_cancelled_timer.cancelled_wakeups == 0,
5170 "cancelled timer never wakes after auto-advance",
5171 0u64,
5172 with_cancelled_timer.cancelled_wakeups
5173 );
5174
5175 crate::test_complete!("metamorphic_cancelled_timer_does_not_skew_virtual_time_progression");
5176 }
5177
5178 #[cfg(unix)]
5179 #[test]
5180 fn run_with_auto_advance_delivers_delayed_reactor_events() {
5181 init_test("run_with_auto_advance_delivers_delayed_reactor_events");
5182 let config = LabConfig::new(42).with_auto_advance().max_steps(32);
5183 let mut runtime = LabRuntime::new(config);
5184 let handle = runtime.state.io_driver_handle().expect("io driver");
5185 let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5186 let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5187 let source = TestFdSource;
5188
5189 let registration = handle
5190 .register(&source, Interest::READABLE, waker)
5191 .expect("register source");
5192 let token = registration.token();
5193
5194 runtime
5195 .lab_reactor()
5196 .inject_event(token, Event::readable(token), Duration::from_secs(1));
5197
5198 let report = runtime.run_with_auto_advance();
5199
5200 crate::assert_with_log!(
5201 report.auto_advances >= 1,
5202 "auto-advance reaches delayed reactor deadline",
5203 true,
5204 report.auto_advances >= 1
5205 );
5206 crate::assert_with_log!(
5207 runtime.now() >= Time::from_secs(1),
5208 "virtual time advanced to delayed reactor event",
5209 true,
5210 runtime.now() >= Time::from_secs(1)
5211 );
5212 let wakeups = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5213 crate::assert_with_log!(
5214 wakeups == 1,
5215 "reactor event woke registration",
5216 1u64,
5217 wakeups
5218 );
5219 let saw_ready = runtime
5220 .state
5221 .trace
5222 .snapshot()
5223 .iter()
5224 .any(|event| event.kind == TraceEventKind::IoReady);
5225 crate::assert_with_log!(saw_ready, "io ready trace recorded", true, saw_ready);
5226 let next_event = runtime.lab_reactor().next_event_time();
5227 crate::assert_with_log!(
5228 next_event.is_none(),
5229 "delayed reactor event drained",
5230 true,
5231 next_event.is_none()
5232 );
5233
5234 crate::test_complete!("run_with_auto_advance_delivers_delayed_reactor_events");
5235 }
5236
5237 #[test]
5238 fn run_with_auto_advance_basic() {
5239 init_test("run_with_auto_advance_basic");
5240 let config = LabConfig::new(42).with_auto_advance();
5241 let mut runtime = LabRuntime::new(config);
5242
5243 let report = runtime.run_with_auto_advance();
5245 crate::assert_with_log!(report.steps == 0, "0 steps", 0u64, report.steps);
5246 crate::assert_with_log!(
5247 report.auto_advances == 0,
5248 "0 auto-advances",
5249 0u64,
5250 report.auto_advances
5251 );
5252 crate::test_complete!("run_with_auto_advance_basic");
5253 }
5254
5255 #[test]
5256 fn run_with_auto_advance_jumps_past_timer_deadlines() {
5257 init_test("run_with_auto_advance_jumps_past_timer_deadlines");
5258 let config = LabConfig::new(42).with_auto_advance().max_steps(1_000);
5259 let mut runtime = LabRuntime::new(config);
5260
5261 let timer_handle = runtime.state.timer_driver_handle().unwrap();
5263 let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5264
5265 for secs in [1, 5, 10] {
5266 let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5267 let _ = timer_handle.register(Time::from_secs(secs), waker);
5268 }
5269
5270 let report = runtime.run_with_auto_advance();
5271
5272 crate::assert_with_log!(
5274 report.auto_advances >= 3,
5275 "at least 3 auto-advances",
5276 true,
5277 report.auto_advances >= 3
5278 );
5279
5280 let now = runtime.now();
5282 crate::assert_with_log!(
5283 now >= Time::from_secs(10),
5284 "time >= 10s",
5285 true,
5286 now >= Time::from_secs(10)
5287 );
5288
5289 let count = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5291 crate::assert_with_log!(count == 3, "3 wakeups", 3u64, count);
5292 crate::test_complete!("run_with_auto_advance_jumps_past_timer_deadlines");
5293 }
5294
5295 #[test]
5296 fn virtual_time_24_hour_instant_test() {
5297 init_test("virtual_time_24_hour_instant_test");
5298 let config = LabConfig::new(42).with_auto_advance().max_steps(100_000);
5300 let mut runtime = LabRuntime::new(config);
5301
5302 let timer_handle = runtime.state.timer_driver_handle().unwrap();
5304 let wake_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
5305
5306 for hour in 1..=24 {
5307 let waker = Waker::from(Arc::new(CountWaker(wake_count.clone())));
5308 let _ = timer_handle.register(Time::from_secs(hour * 3600), waker);
5309 }
5310
5311 let wall_start = std::time::Instant::now();
5312 let report = runtime.run_with_auto_advance();
5313 let wall_elapsed = wall_start.elapsed();
5314
5315 crate::assert_with_log!(
5317 report.virtual_elapsed_secs() >= 86400,
5318 "24h virtual",
5319 true,
5320 report.virtual_elapsed_secs() >= 86400
5321 );
5322
5323 let count = wake_count.load(std::sync::atomic::Ordering::SeqCst);
5325 crate::assert_with_log!(count == 24, "24 wakeups", 24u64, count);
5326
5327 let wall_ms = wall_elapsed.as_millis();
5329 crate::assert_with_log!(wall_ms < 1000, "wall time < 1s", true, wall_ms < 1000);
5330 crate::test_complete!("virtual_time_24_hour_instant_test");
5331 }
5332
5333 #[test]
5334 fn clock_pause_resume() {
5335 init_test("clock_pause_resume");
5336 let runtime = LabRuntime::with_seed(42);
5337
5338 let not_paused = !runtime.is_clock_paused();
5339 crate::assert_with_log!(not_paused, "not paused initially", true, not_paused);
5340
5341 runtime.pause_clock();
5342 let paused = runtime.is_clock_paused();
5343 crate::assert_with_log!(paused, "paused", true, paused);
5344
5345 runtime.resume_clock();
5346 let resumed = !runtime.is_clock_paused();
5347 crate::assert_with_log!(resumed, "resumed", true, resumed);
5348 crate::test_complete!("clock_pause_resume");
5349 }
5350
5351 #[test]
5352 fn inject_clock_skew() {
5353 init_test("inject_clock_skew");
5354 let mut runtime = LabRuntime::with_seed(42);
5355
5356 runtime.advance_time(1_000_000_000); let before = runtime.now();
5358
5359 runtime.inject_clock_skew(5_000_000_000);
5361 let after = runtime.now();
5362
5363 let delta = after.as_nanos() - before.as_nanos();
5364 crate::assert_with_log!(
5365 delta == 5_000_000_000,
5366 "5s skew applied",
5367 5_000_000_000u64,
5368 delta
5369 );
5370
5371 crate::assert_with_log!(
5372 after == Time::from_secs(6),
5373 "time at 6s",
5374 Time::from_secs(6),
5375 after
5376 );
5377 crate::test_complete!("inject_clock_skew");
5378 }
5379
5380 #[test]
5381 fn virtual_time_report_conversions() {
5382 init_test("virtual_time_report_conversions");
5383 let report = VirtualTimeReport {
5384 steps: 100,
5385 auto_advances: 5,
5386 total_wakeups: 10,
5387 time_start: Time::ZERO,
5388 time_end: Time::from_secs(3600),
5389 virtual_elapsed_nanos: 3_600_000_000_000,
5390 termination: AutoAdvanceTermination::Quiescent,
5391 };
5392
5393 let ms = report.virtual_elapsed_ms();
5394 crate::assert_with_log!(ms == 3_600_000, "3600000 ms", 3_600_000u64, ms);
5395
5396 let secs = report.virtual_elapsed_secs();
5397 crate::assert_with_log!(secs == 3600, "3600 secs", 3600u64, secs);
5398 crate::test_complete!("virtual_time_report_conversions");
5399 }
5400
5401 #[test]
5413 fn replay_records_correct_severity_for_cancelled_task() {
5414 init_test("replay_records_correct_severity_for_cancelled_task");
5415
5416 let config = LabConfig::new(42)
5417 .panic_on_leak(false)
5418 .with_default_replay_recording();
5419 let mut runtime = LabRuntime::new(config);
5420 let root = runtime
5421 .state
5422 .create_root_region(crate::types::Budget::INFINITE);
5423
5424 let (task_id, _) = runtime
5427 .state
5428 .create_task(root, crate::types::Budget::INFINITE, async {
5429 crate::runtime::yield_now::yield_now().await;
5430 })
5431 .expect("create task");
5432 runtime.scheduler.lock().schedule(task_id, 0);
5433
5434 runtime.step();
5436
5437 runtime
5439 .state
5440 .update_task(task_id, |record| {
5441 record.request_cancel(crate::types::CancelReason::user("test-cancel"));
5442 let _ = record.acknowledge_cancel();
5443 assert!(
5445 matches!(
5446 record.state,
5447 crate::record::task::TaskState::Cancelling { .. }
5448 ),
5449 "task should be in Cancelling state"
5450 );
5451 })
5452 .unwrap();
5453
5454 runtime.scheduler.lock().schedule(task_id, 0);
5457 runtime.run_until_quiescent();
5458
5459 if let Some(record) = runtime.state.task(task_id) {
5461 assert!(
5462 matches!(
5463 record.state,
5464 crate::record::task::TaskState::Completed(crate::types::Outcome::Cancelled(_))
5465 ),
5466 "task should be Completed(Cancelled), got {:?}",
5467 record.state
5468 );
5469 }
5470
5471 let replay = runtime
5473 .replay_recorder()
5474 .snapshot()
5475 .expect("replay recording enabled");
5476 let completed_events: Vec<_> = replay
5477 .events
5478 .iter()
5479 .filter(|e| matches!(e, crate::trace::replay::ReplayEvent::TaskCompleted { .. }))
5480 .collect();
5481
5482 assert!(
5483 !completed_events.is_empty(),
5484 "should have at least one TaskCompleted event"
5485 );
5486
5487 for event in &completed_events {
5489 if let crate::trace::replay::ReplayEvent::TaskCompleted { outcome, .. } = event {
5490 let expected = crate::types::Severity::Cancelled.as_u8();
5491 crate::assert_with_log!(
5492 *outcome == expected,
5493 "severity should be Cancelled (2)",
5494 expected,
5495 *outcome
5496 );
5497 }
5498 }
5499
5500 crate::test_complete!("replay_records_correct_severity_for_cancelled_task");
5501 }
5502
5503 #[test]
5504 fn replay_recording_metadata_is_stable_for_same_seed() {
5505 init_test("replay_recording_metadata_is_stable_for_same_seed");
5506
5507 let mut first = LabRuntime::new(LabConfig::new(42).with_default_replay_recording());
5508 let mut second = LabRuntime::new(LabConfig::new(42).with_default_replay_recording());
5509
5510 let first_trace = first
5511 .finish_replay_trace()
5512 .expect("first replay recording enabled");
5513 let second_trace = second
5514 .finish_replay_trace()
5515 .expect("second replay recording enabled");
5516
5517 crate::assert_with_log!(
5518 first_trace.metadata == second_trace.metadata,
5519 "replay metadata should match for identical seeds",
5520 &first_trace.metadata,
5521 &second_trace.metadata
5522 );
5523 crate::assert_with_log!(
5524 first_trace.events == second_trace.events,
5525 "replay events should match for identical seeds",
5526 &first_trace.events,
5527 &second_trace.events
5528 );
5529 crate::assert_with_log!(
5530 first_trace.metadata.recorded_at == 0,
5531 "recorded_at defaults to deterministic zero stamp",
5532 0u64,
5533 first_trace.metadata.recorded_at
5534 );
5535
5536 crate::test_complete!("replay_recording_metadata_is_stable_for_same_seed");
5537 }
5538
5539 #[test]
5544 fn lab_trace_certificate_summary_debug_clone_copy_eq() {
5545 let summary = LabTraceCertificateSummary {
5546 event_hash: 123,
5547 event_count: 456,
5548 schedule_hash: 789,
5549 };
5550 let copied = summary;
5551 let cloned = summary;
5552 assert_eq!(copied, cloned);
5553 assert_ne!(
5554 summary,
5555 LabTraceCertificateSummary {
5556 event_hash: 0,
5557 event_count: 456,
5558 schedule_hash: 789,
5559 }
5560 );
5561 let dbg = format!("{summary:?}");
5562 assert!(dbg.contains("LabTraceCertificateSummary"));
5563 }
5564
5565 #[test]
5566 fn virtual_time_report_debug_clone_copy_eq() {
5567 let report = VirtualTimeReport {
5568 steps: 100,
5569 auto_advances: 5,
5570 total_wakeups: 10,
5571 time_start: Time::ZERO,
5572 time_end: Time::from_millis(500),
5573 virtual_elapsed_nanos: 500_000_000,
5574 termination: AutoAdvanceTermination::Quiescent,
5575 };
5576 let copied = report;
5577 assert_eq!(copied, report);
5578 assert_eq!(report.virtual_elapsed_ms(), 500);
5579 assert_eq!(report.virtual_elapsed_secs(), 0);
5580 let dbg = format!("{report:?}");
5581 assert!(dbg.contains("VirtualTimeReport"));
5582 }
5583
5584 #[test]
5589 fn auto_advance_quiescent_termination() {
5590 init_test("auto_advance_quiescent_termination");
5591 let mut lab = LabRuntime::new(LabConfig::new(42));
5592 let report = lab.run_with_auto_advance();
5594 assert_eq!(
5595 report.termination,
5596 AutoAdvanceTermination::Quiescent,
5597 "empty runtime should terminate as quiescent"
5598 );
5599 crate::test_complete!("auto_advance_quiescent_termination");
5600 }
5601
5602 #[test]
5603 fn auto_advance_step_limit_termination() {
5604 init_test("auto_advance_step_limit_termination");
5605 let mut lab = LabRuntime::new(LabConfig::new(42).max_steps(0));
5606 let report = lab.run_with_auto_advance();
5607 assert_eq!(
5608 report.termination,
5609 AutoAdvanceTermination::StepLimitReached,
5610 "zero max_steps should terminate as step-limit-reached"
5611 );
5612 crate::test_complete!("auto_advance_step_limit_termination");
5613 }
5614
5615 #[test]
5616 fn auto_advance_stuck_bailout_termination() {
5617 init_test("auto_advance_stuck_bailout_termination");
5618 let config = LabConfig::new(42)
5619 .with_auto_advance()
5620 .no_step_limit()
5621 .futurelock_max_idle_steps(0);
5622 let mut lab = LabRuntime::new(config);
5623 let root = lab.state.create_root_region(Budget::INFINITE);
5624 let (task_id, _handle) = lab
5625 .state
5626 .create_task(root, Budget::INFINITE, async {
5627 std::future::pending::<()>().await;
5628 })
5629 .expect("create pending task");
5630 lab.scheduler.lock().schedule(task_id, 0);
5631
5632 let report = lab.run_with_auto_advance();
5633 assert_eq!(
5634 report.termination,
5635 AutoAdvanceTermination::StuckBailout,
5636 "pending task without deadlines should terminate via stuck bailout"
5637 );
5638 assert!(
5639 !lab.is_quiescent(),
5640 "stuck bailout should preserve non-quiescent state for diagnosis"
5641 );
5642 assert_eq!(
5643 report.auto_advances, 0,
5644 "stuck bailout path should not auto-advance virtual time without deadlines"
5645 );
5646 crate::test_complete!("auto_advance_stuck_bailout_termination");
5647 }
5648
5649 #[test]
5650 fn auto_advance_termination_display() {
5651 assert_eq!(
5652 format!("{}", AutoAdvanceTermination::Quiescent),
5653 "quiescent"
5654 );
5655 assert_eq!(
5656 format!("{}", AutoAdvanceTermination::StepLimitReached),
5657 "step-limit-reached"
5658 );
5659 assert_eq!(
5660 format!("{}", AutoAdvanceTermination::StuckBailout),
5661 "stuck-bailout"
5662 );
5663 }
5664
5665 #[test]
5666 fn auto_advance_termination_debug_clone_copy_eq_hash() {
5667 use std::collections::HashSet;
5668 let variants = [
5669 AutoAdvanceTermination::Quiescent,
5670 AutoAdvanceTermination::StepLimitReached,
5671 AutoAdvanceTermination::StuckBailout,
5672 ];
5673 for &v in &variants {
5675 let copied = v;
5676 let cloned = v;
5677 assert_eq!(copied, cloned);
5678 }
5679 let mut set = HashSet::new();
5681 for &v in &variants {
5682 assert!(set.insert(v));
5683 }
5684 assert_eq!(set.len(), 3);
5685 let dbg = format!("{:?}", AutoAdvanceTermination::StuckBailout);
5687 assert!(dbg.contains("StuckBailout"));
5688 }
5689
5690 #[test]
5691 fn harness_attachment_kind_debug_clone_copy_eq_hash_ord_display() {
5692 use std::collections::HashSet;
5693 let kinds = [
5694 HarnessAttachmentKind::CrashPack,
5695 HarnessAttachmentKind::ReplayTrace,
5696 HarnessAttachmentKind::Trace,
5697 HarnessAttachmentKind::Other,
5698 ];
5699 assert_eq!(format!("{}", kinds[0]), "crashpack");
5701 assert_eq!(format!("{}", kinds[1]), "replay_trace");
5702 assert_eq!(format!("{}", kinds[2]), "trace");
5703 assert_eq!(format!("{}", kinds[3]), "other");
5704 for &k in &kinds {
5706 let copied = k;
5707 let cloned = k;
5708 assert_eq!(copied, cloned);
5709 }
5710 let mut set = HashSet::new();
5712 for &k in &kinds {
5713 set.insert(k);
5714 }
5715 assert_eq!(set.len(), 4);
5716 assert!(HarnessAttachmentKind::CrashPack < HarnessAttachmentKind::ReplayTrace);
5718 assert!(HarnessAttachmentKind::ReplayTrace < HarnessAttachmentKind::Trace);
5719 assert!(HarnessAttachmentKind::Trace < HarnessAttachmentKind::Other);
5720 let mut sorted = [kinds[3], kinds[0], kinds[2], kinds[1]];
5721 sorted.sort();
5722 assert_eq!(sorted, kinds);
5723 }
5724
5725 #[test]
5726 fn harness_attachment_ref_debug_clone_eq_hash() {
5727 use std::collections::HashSet;
5728 let ref1 = HarnessAttachmentRef::crashpack("crash.bin");
5729 let ref2 = HarnessAttachmentRef::replay_trace("replay.bin");
5730 let ref3 = HarnessAttachmentRef::trace("trace.ndjson");
5731 assert_eq!(ref1.kind, HarnessAttachmentKind::CrashPack);
5732 assert_eq!(ref2.kind, HarnessAttachmentKind::ReplayTrace);
5733 assert_eq!(ref3.kind, HarnessAttachmentKind::Trace);
5734 let cloned = ref1.clone();
5735 assert_eq!(cloned, ref1);
5736 assert_ne!(ref1, ref2);
5737 let dbg = format!("{ref1:?}");
5738 assert!(dbg.contains("HarnessAttachmentRef"));
5739 let mut set = HashSet::new();
5740 set.insert(ref1.clone());
5741 set.insert(ref2);
5742 set.insert(ref1); assert_eq!(set.len(), 2);
5744 }
5745
5746 #[test]
5747 fn chaos_config_summary_debug_clone_copy_partial_eq() {
5748 let summary = ChaosConfigSummary {
5749 seed: 42,
5750 cancel_probability: 0.1,
5751 delay_probability: 0.2,
5752 io_error_probability: 0.05,
5753 wakeup_storm_probability: 0.01,
5754 budget_exhaust_probability: 0.03,
5755 };
5756 let copied = summary;
5757 let cloned = summary;
5758 assert_eq!(copied, cloned);
5759 let dbg = format!("{summary:?}");
5760 assert!(dbg.contains("ChaosConfigSummary"));
5761 }
5762
5763 #[test]
5764 fn obligation_leak_debug_clone_eq_display() {
5765 let leak = ObligationLeak {
5766 obligation: ObligationId::new_for_test(1, 0),
5767 kind: ObligationKind::SendPermit,
5768 holder: TaskId::from_arena(crate::util::ArenaIndex::new(1, 0)),
5769 region: RegionId::new_for_test(0, 0),
5770 };
5771 let cloned = leak.clone();
5772 assert_eq!(cloned, leak);
5773 let dbg = format!("{leak:?}");
5774 assert!(dbg.contains("ObligationLeak"));
5775 let display = format!("{leak}");
5776 assert!(!display.is_empty());
5777 }
5778
5779 #[test]
5798 fn conformance_identical_seed_identical_trace() {
5799 init_test("conformance_identical_seed_identical_trace");
5800
5801 let seed = 42_u64;
5802 let config = LabConfig::new(seed).worker_count(2).max_steps(1000);
5803
5804 let mut reports = Vec::new();
5806 for run_id in 0..2 {
5807 let mut runtime = LabRuntime::new(config.clone());
5808 let root = runtime.state.create_root_region(Budget::INFINITE);
5809
5810 for i in 0..5 {
5812 let (task_id, _handle) = runtime
5813 .state
5814 .create_task(root, Budget::INFINITE, async move {
5815 for j in 0..10 {
5817 futures_lite::future::yield_now().await;
5818 if (i + j) % 3 == 0 {
5819 let now =
5820 crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
5821 crate::time::sleep(now, Duration::from_millis(1)).await;
5822 }
5823 }
5824 i * 100 + run_id
5825 })
5826 .expect("create task");
5827 runtime.scheduler.lock().schedule(task_id, 0);
5828 }
5829
5830 let report = runtime.run_until_quiescent_with_report();
5831 reports.push(report);
5832 }
5833
5834 let report1 = &reports[0];
5836 let report2 = &reports[1];
5837
5838 crate::assert_with_log!(
5839 report1.seed == report2.seed,
5840 "seeds should be identical",
5841 report1.seed,
5842 report2.seed
5843 );
5844
5845 crate::assert_with_log!(
5846 report1.trace_fingerprint == report2.trace_fingerprint,
5847 "trace fingerprints should be identical",
5848 report1.trace_fingerprint,
5849 report2.trace_fingerprint
5850 );
5851
5852 crate::assert_with_log!(
5853 report1.trace_certificate.event_hash == report2.trace_certificate.event_hash,
5854 "event hashes should be identical",
5855 report1.trace_certificate.event_hash,
5856 report2.trace_certificate.event_hash
5857 );
5858
5859 crate::assert_with_log!(
5860 report1.trace_certificate.event_count == report2.trace_certificate.event_count,
5861 "event counts should be identical",
5862 report1.trace_certificate.event_count,
5863 report2.trace_certificate.event_count
5864 );
5865
5866 crate::assert_with_log!(
5867 report1.trace_certificate.schedule_hash == report2.trace_certificate.schedule_hash,
5868 "schedule hashes should be identical",
5869 report1.trace_certificate.schedule_hash,
5870 report2.trace_certificate.schedule_hash
5871 );
5872
5873 crate::test_complete!("conformance_identical_seed_identical_trace");
5874 }
5875
5876 #[test]
5881 fn conformance_virtual_time_deterministic_advancement() {
5882 init_test("conformance_virtual_time_deterministic_advancement");
5883
5884 let config = LabConfig::new(123).worker_count(1);
5885
5886 crate::lab::assert_deterministic(config, |runtime| {
5887 let root = runtime.state.create_root_region(Budget::INFINITE);
5888 let initial_time = runtime.now();
5889
5890 let durations = [
5892 Duration::from_millis(10),
5893 Duration::from_millis(5),
5894 Duration::from_millis(15),
5895 Duration::from_millis(1),
5896 ];
5897
5898 for (i, duration) in durations.iter().enumerate() {
5899 let dur = *duration;
5900 let (task_id, _handle) = runtime
5901 .state
5902 .create_task(root, Budget::INFINITE, async move {
5903 let now = crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
5904 crate::time::sleep(now, dur).await;
5905 i
5906 })
5907 .expect("create task");
5908 runtime.scheduler.lock().schedule(task_id, 0);
5909 }
5910
5911 let vtime_report = runtime.run_with_auto_advance();
5913
5914 crate::assert_with_log!(
5916 vtime_report.time_end > initial_time,
5917 "virtual time should have advanced",
5918 vtime_report.time_end,
5919 initial_time
5920 );
5921
5922 crate::assert_with_log!(
5923 vtime_report.auto_advances > 0,
5924 "should have auto-advanced virtual time",
5925 vtime_report.auto_advances,
5926 0
5927 );
5928
5929 crate::assert_with_log!(
5930 vtime_report.termination == AutoAdvanceTermination::Quiescent,
5931 "should reach quiescence",
5932 vtime_report.termination,
5933 AutoAdvanceTermination::Quiescent
5934 );
5935 });
5936
5937 crate::test_complete!("conformance_virtual_time_deterministic_advancement");
5938 }
5939
5940 #[test]
5945 fn conformance_scheduler_deterministic_lottery() {
5946 init_test("conformance_scheduler_deterministic_lottery");
5947
5948 let config = LabConfig::new(456).worker_count(4);
5949
5950 let mut schedule_sequences = Vec::new();
5951
5952 for run in 0..2 {
5954 let mut runtime = LabRuntime::new(config.clone());
5955 let root = runtime.state.create_root_region(Budget::INFINITE);
5956 let mut task_order = Vec::new();
5957
5958 for task_idx in 0..20 {
5960 let (task_id, _handle) = runtime
5961 .state
5962 .create_task(root, Budget::INFINITE, async move {
5963 for _ in 0..3 {
5965 futures_lite::future::yield_now().await;
5966 }
5967 task_idx
5968 })
5969 .expect("create task");
5970 runtime.scheduler.lock().schedule(task_id, 0);
5971 }
5972
5973 runtime.run_until_quiescent();
5975 let cert = runtime.certificate();
5976 task_order.push((run, cert.decisions(), cert.hash()));
5977 schedule_sequences.push(task_order);
5978 }
5979
5980 let seq1 = &schedule_sequences[0];
5982 let seq2 = &schedule_sequences[1];
5983
5984 crate::assert_with_log!(
5985 seq1.len() == seq2.len(),
5986 "should have same number of scheduling decision points",
5987 seq1.len(),
5988 seq2.len()
5989 );
5990
5991 for (i, ((_run1, count1, hash1), (_run2, count2, hash2))) in
5992 seq1.iter().zip(seq2.iter()).enumerate()
5993 {
5994 crate::assert_with_log!(
5995 count1 == count2,
5996 &format!("decision count should be identical at point {}", i),
5997 count1,
5998 count2
5999 );
6000
6001 crate::assert_with_log!(
6002 hash1 == hash2,
6003 &format!("schedule hash should be identical at point {}", i),
6004 hash1,
6005 hash2
6006 );
6007 }
6008
6009 crate::test_complete!("conformance_scheduler_deterministic_lottery");
6010 }
6011
6012 #[test]
6017 fn conformance_chaos_injection_deterministic() {
6018 init_test("conformance_chaos_injection_deterministic");
6019
6020 let chaos_config = crate::lab::chaos::ChaosConfig::new(789)
6021 .with_cancel_probability(0.1)
6022 .with_delay_probability(0.05)
6023 .with_io_error_probability(0.02);
6024 let config = LabConfig::new(999).with_chaos(chaos_config);
6025
6026 crate::lab::assert_deterministic(config, |runtime| {
6027 let root = runtime.state.create_root_region(Budget::INFINITE);
6028
6029 for i in 0..10 {
6031 let (task_id, _handle) = runtime
6032 .state
6033 .create_task(root, Budget::INFINITE, async move {
6034 for j in 0..20 {
6036 futures_lite::future::yield_now().await;
6037 if j % 5 == 0 {
6038 let now =
6039 crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
6040 crate::time::sleep(now, Duration::from_millis(1)).await;
6041 }
6042 }
6043 i
6044 })
6045 .expect("create task");
6046 runtime.scheduler.lock().schedule(task_id, 0);
6047 }
6048
6049 runtime.run_until_quiescent();
6050
6051 let chaos_stats = runtime.chaos_stats();
6053 let total_decisions = chaos_stats.decision_points;
6054
6055 crate::assert_with_log!(
6056 total_decisions > 0,
6057 "chaos should have made some decisions",
6058 total_decisions,
6059 0
6060 );
6061 });
6062
6063 crate::test_complete!("conformance_chaos_injection_deterministic");
6064 }
6065
6066 #[test]
6071 fn conformance_panic_semantics_deterministic() {
6072 init_test("conformance_panic_semantics_deterministic");
6073
6074 let config = LabConfig::new(333)
6075 .worker_count(3)
6076 .panic_on_leak(false)
6077 .max_steps(10_000); crate::lab::assert_deterministic(config, |runtime| {
6080 let root = runtime.state.create_root_region(Budget::INFINITE);
6081
6082 for i in 0..5 {
6084 let (task_id, _handle) = runtime
6085 .state
6086 .create_task(root, Budget::INFINITE, async move {
6087 assert!(i != 2, "deterministic panic in task {}", i);
6089
6090 for _j in 0..10 {
6092 futures_lite::future::yield_now().await;
6093 }
6094 i * 10
6095 })
6096 .expect("create task");
6097 runtime.scheduler.lock().schedule(task_id, 0);
6098 }
6099
6100 let report = runtime.run_until_quiescent_with_report();
6105 let trace_events = runtime.trace().snapshot();
6106 let complete_events = trace_events
6107 .iter()
6108 .filter(|event| event.kind == TraceEventKind::Complete)
6109 .count();
6110
6111 crate::assert_with_log!(
6112 complete_events > 0,
6113 "should have recorded task completion activity",
6114 complete_events,
6115 0
6116 );
6117
6118 crate::assert_with_log!(
6120 report.quiescent,
6121 "runtime should reach quiescence despite panic",
6122 report.quiescent,
6123 false
6124 );
6125 });
6126
6127 crate::test_complete!("conformance_panic_semantics_deterministic");
6128 }
6129
6130 #[test]
6135 fn conformance_comprehensive_determinism_stress() {
6136 init_test("conformance_comprehensive_determinism_stress");
6137
6138 let chaos_config = crate::lab::chaos::ChaosConfig::new(555)
6139 .with_cancel_probability(0.05)
6140 .with_delay_probability(0.03);
6141 let config = LabConfig::new(777)
6142 .worker_count(4)
6143 .with_chaos(chaos_config)
6144 .max_steps(5000);
6145
6146 crate::lab::assert_deterministic_multi(&config, 3, |runtime| {
6148 let root = runtime.state.create_root_region(Budget::INFINITE);
6149
6150 for i in 0..15 {
6152 let (task_id, _handle) = runtime
6153 .state
6154 .create_task(root, Budget::INFINITE, async move {
6155 for j in 0..30 {
6157 match (i + j) % 4 {
6158 0 => futures_lite::future::yield_now().await,
6159 1 => {
6160 let now =
6161 crate::cx::Cx::current().map_or(Time::ZERO, |cx| cx.now());
6162 crate::time::sleep(now, Duration::from_millis(j as u64 % 5))
6163 .await;
6164 }
6165 2 => {
6166 let mut sum = 0_u64;
6168 for k in 0..100 {
6169 sum = sum.wrapping_add(k);
6170 }
6171 let _ = sum;
6172 }
6173 _ => futures_lite::future::yield_now().await,
6174 }
6175 }
6176 i * 1000 + 42
6177 })
6178 .expect("create task");
6179 runtime.scheduler.lock().schedule(task_id, 0);
6180 }
6181
6182 let vtime_report = runtime.run_with_auto_advance();
6184
6185 crate::assert_with_log!(
6186 vtime_report.termination == AutoAdvanceTermination::Quiescent,
6187 "comprehensive workload should reach quiescence",
6188 vtime_report.termination,
6189 AutoAdvanceTermination::Quiescent
6190 );
6191 });
6192
6193 crate::test_complete!("conformance_comprehensive_determinism_stress");
6194 }
6195
6196 #[test]
6197 #[allow(clippy::literal_string_with_formatting_args)]
6198 fn non_test_lab_runtime_paths_do_not_use_stray_stdout_debug_prints() {
6199 let source =
6200 std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(file!()))
6201 .expect("lab runtime source must be readable");
6202
6203 for message in [
6204 "LabScheduler already scheduled {task:?}",
6205 "LabScheduler scheduling {task:?}",
6206 "Executing {:?} at step {}",
6207 "rng_value = {}, worker_hint = {}",
6208 ] {
6209 let stdout_call = format!("print{}!(\"{message}\"", "ln");
6210 assert!(
6211 !source.contains(&stdout_call),
6212 "non-test LabRuntime debug print regressed: {message}"
6213 );
6214 }
6215 }
6216}