moonpool_sim/runner/
report.rs

1//! Simulation metrics and reporting.
2//!
3//! This module provides types for collecting and reporting simulation results.
4
5use std::collections::HashMap;
6use std::fmt;
7use std::time::Duration;
8
9use moonpool_assertions::AssertKind;
10
11use crate::SimulationResult;
12use crate::chaos::AssertionStats;
13
14/// Format a fork-recipe (`(rng_call_count, seed)` pairs) as a human-readable
15/// timeline: each segment `count@seed`, joined by ` -> ` (matches
16/// `moonpool_explorer::format_timeline`, but available without the backend).
17pub(crate) fn format_recipe(recipe: &[(u64, u64)]) -> String {
18    recipe
19        .iter()
20        .map(|(count, seed)| format!("{count}@{seed}"))
21        .collect::<Vec<_>>()
22        .join(" -> ")
23}
24
25/// Core metrics collected during a simulation run.
26#[derive(Debug, Clone, PartialEq)]
27pub struct SimulationMetrics {
28    /// Wall-clock time taken for the simulation
29    pub wall_time: Duration,
30    /// Simulated logical time elapsed
31    pub simulated_time: Duration,
32    /// Number of events processed
33    pub events_processed: u64,
34}
35
36impl Default for SimulationMetrics {
37    fn default() -> Self {
38        Self {
39            wall_time: Duration::ZERO,
40            simulated_time: Duration::ZERO,
41            events_processed: 0,
42        }
43    }
44}
45
46/// A captured bug recipe with its root seed for deterministic replay.
47#[derive(Debug, Clone, PartialEq)]
48pub struct BugRecipe {
49    /// The root seed that was active when this bug was discovered.
50    pub seed: u64,
51    /// Fork path: `(rng_call_count, child_seed)` pairs.
52    pub recipe: Vec<(u64, u64)>,
53}
54
55/// Report from fork-based exploration.
56#[derive(Debug, Clone)]
57pub struct ExplorationReport {
58    /// Total timelines explored across all forks.
59    pub total_timelines: u64,
60    /// Total fork points triggered.
61    pub fork_points: u64,
62    /// Number of bugs found (children exiting with code 42).
63    pub bugs_found: u64,
64    /// Bug recipes captured during exploration (one per seed that found bugs).
65    pub bug_recipes: Vec<BugRecipe>,
66    /// Remaining global energy after exploration.
67    pub energy_remaining: i64,
68    /// Energy in the reallocation pool.
69    pub realloc_pool_remaining: i64,
70    /// Number of bits set in the explored coverage map.
71    pub coverage_bits: u32,
72    /// Total number of bits in the coverage map (8192).
73    pub coverage_total: u32,
74    /// Total instrumented code edges (from LLVM sancov). 0 when sancov unavailable.
75    pub sancov_edges_total: usize,
76    /// Code edges covered across all timelines. 0 when sancov unavailable.
77    pub sancov_edges_covered: usize,
78    /// Whether the multi-seed loop stopped because convergence was detected.
79    pub converged: bool,
80    /// Timelines explored per seed (parallel to `seeds_used`).
81    pub per_seed_timelines: Vec<u64>,
82}
83
84/// Which progress signal an [`UntilCoverageStable`] run plateaued on.
85///
86/// [`UntilCoverageStable`]: crate::IterationControl::UntilCoverageStable
87#[derive(Debug, Clone, Copy, PartialEq, Eq)]
88pub enum SaturationSignal {
89    /// Real LLVM sancov edge coverage (binary built via `cargo xtask sim run`).
90    CodeCoverage,
91    /// Count of distinct reached sometimes/reachable assertion slots
92    /// (fallback when the binary is not sancov-instrumented).
93    AssertionCoverage,
94}
95
96/// Outcome of an [`UntilCoverageStable`] run: which signal it plateaued on and
97/// the final coverage numbers, so the report can name the stop reason
98/// explicitly.
99///
100/// [`UntilCoverageStable`]: crate::IterationControl::UntilCoverageStable
101#[derive(Debug, Clone)]
102pub struct SaturationReport {
103    /// The progress signal counted toward the plateau.
104    pub signal: SaturationSignal,
105    /// Code edges covered (meaningful only when `signal == CodeCoverage`).
106    pub edges_covered: usize,
107    /// Total instrumented code edges (meaningful only when `signal == CodeCoverage`).
108    pub edges_total: usize,
109    /// Number of observed sometimes/reachable assertions that fired.
110    pub sometimes_hit: usize,
111    /// Total number of observed sometimes/reachable assertions.
112    pub sometimes_total: usize,
113    /// Consecutive quiet seeds required to declare saturation.
114    pub plateau_seeds: usize,
115}
116
117/// Pass/fail/miss status for an assertion in the report.
118#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
119pub enum AssertionStatus {
120    /// Assertion contract violated (always-type failed, or unreachable reached).
121    Fail,
122    /// Coverage assertion never satisfied (sometimes/reachable never triggered).
123    Miss,
124    /// Assertion contract satisfied.
125    Pass,
126}
127
128/// Detailed information about a single assertion slot.
129#[derive(Debug, Clone)]
130pub struct AssertionDetail {
131    /// Human-readable assertion message.
132    pub msg: String,
133    /// The kind of assertion.
134    pub kind: AssertKind,
135    /// Number of times the assertion passed.
136    pub pass_count: u64,
137    /// Number of times the assertion failed.
138    pub fail_count: u64,
139    /// Best watermark value (for numeric assertions).
140    pub watermark: i64,
141    /// Frontier value (for `BooleanSometimesAll`).
142    pub frontier: u8,
143    /// Computed status based on kind and counts.
144    pub status: AssertionStatus,
145}
146
147/// Summary of one `assert_sometimes_each!` site (grouped by msg).
148#[derive(Debug, Clone)]
149pub struct BucketSiteSummary {
150    /// Assertion message identifying the site.
151    pub msg: String,
152    /// Number of unique identity-key buckets discovered.
153    pub buckets_discovered: usize,
154    /// Total hit count across all buckets.
155    pub total_hits: u64,
156}
157
158/// Comprehensive report of a simulation run with statistical analysis.
159#[derive(Debug, Clone)]
160pub struct SimulationReport {
161    /// Number of iterations executed
162    pub iterations: usize,
163    /// Number of successful runs
164    pub successful_runs: usize,
165    /// Number of failed runs
166    pub failed_runs: usize,
167    /// Aggregated metrics across all runs
168    pub metrics: SimulationMetrics,
169    /// Individual metrics for each iteration
170    pub individual_metrics: Vec<SimulationResult<SimulationMetrics>>,
171    /// Seeds used for each iteration
172    pub seeds_used: Vec<u64>,
173    /// failed seeds
174    pub seeds_failing: Vec<u64>,
175    /// Aggregated assertion results across all iterations
176    pub assertion_results: HashMap<String, AssertionStats>,
177    /// Always-type assertion violations (definite bugs).
178    pub assertion_violations: Vec<String>,
179    /// Coverage assertion violations (sometimes/reachable not satisfied).
180    pub coverage_violations: Vec<String>,
181    /// Exploration report (present when fork-based exploration was enabled).
182    pub exploration: Option<ExplorationReport>,
183    /// Per-assertion detailed breakdown from shared memory slots.
184    pub assertion_details: Vec<AssertionDetail>,
185    /// Per-site summaries of `assert_sometimes_each!` buckets.
186    pub bucket_summaries: Vec<BucketSiteSummary>,
187    /// True when `UntilCoverageStable` hit its iteration cap without saturating.
188    pub convergence_timeout: bool,
189    /// Saturation outcome for an `UntilCoverageStable` run (signal + coverage
190    /// numbers); `None` for other iteration-control modes.
191    pub saturation: Option<SaturationReport>,
192}
193
194impl SimulationReport {
195    /// Calculate the success rate as a percentage.
196    #[must_use]
197    pub fn success_rate(&self) -> f64 {
198        if self.iterations == 0 {
199            0.0
200        } else {
201            // Precision loss acceptable: simulation counts fit well within 2^52.
202            let successful = u32::try_from(self.successful_runs).map_or(f64::INFINITY, f64::from);
203            let total = u32::try_from(self.iterations).map_or(f64::INFINITY, f64::from);
204            (successful / total) * 100.0
205        }
206    }
207
208    /// Get the average wall time per iteration.
209    #[must_use]
210    pub fn average_wall_time(&self) -> Duration {
211        if self.successful_runs == 0 {
212            Duration::ZERO
213        } else {
214            let runs = u32::try_from(self.successful_runs).unwrap_or(u32::MAX);
215            self.metrics.wall_time / runs
216        }
217    }
218
219    /// Get the average simulated time per iteration.
220    #[must_use]
221    pub fn average_simulated_time(&self) -> Duration {
222        if self.successful_runs == 0 {
223            Duration::ZERO
224        } else {
225            let runs = u32::try_from(self.successful_runs).unwrap_or(u32::MAX);
226            self.metrics.simulated_time / runs
227        }
228    }
229
230    /// Get the average number of events processed per iteration.
231    #[must_use]
232    pub fn average_events_processed(&self) -> f64 {
233        if self.successful_runs == 0 {
234            0.0
235        } else {
236            // Precision loss acceptable: simulation counts fit within 2^52.
237            let events =
238                u32::try_from(self.metrics.events_processed).map_or(f64::INFINITY, f64::from);
239            let runs = u32::try_from(self.successful_runs).map_or(f64::INFINITY, f64::from);
240            events / runs
241        }
242    }
243
244    /// Print the report to stderr with colors when the terminal supports it.
245    ///
246    /// Falls back to the plain `Display` output when stderr is not a TTY
247    /// or `NO_COLOR` is set.
248    pub fn eprint(&self) {
249        super::display::eprint_report(self);
250    }
251}
252
253// ---------------------------------------------------------------------------
254// Display helpers
255// ---------------------------------------------------------------------------
256
257/// Convert a non-negative finite `f64` to a saturated `u64`.
258///
259/// Returns 0 for NaN / negative values and `u64::MAX` for values that exceed
260/// `u64::MAX`. Smaller magnitudes round to the nearest integer.
261fn f64_to_u64_saturating(v: f64) -> u64 {
262    // `2^64` as an `f64` — values at or above this saturate to `u64::MAX`.
263    const TWO_POW_64: f64 = 18_446_744_073_709_551_616.0;
264    if !v.is_finite() || v <= 0.0 {
265        0
266    } else if v >= TWO_POW_64 {
267        u64::MAX
268    } else {
269        // SAFETY: `v` is finite, non-negative, and strictly below `2^64`;
270        // `to_int_unchecked` is therefore well-defined.
271        unsafe { v.round().to_int_unchecked::<u64>() }
272    }
273}
274
275/// Format a number with comma separators (e.g., 123456 -> "123,456").
276fn fmt_num(n: u64) -> String {
277    let s = n.to_string();
278    let mut result = String::with_capacity(s.len() + s.len() / 3);
279    for (i, c) in s.chars().rev().enumerate() {
280        if i > 0 && i % 3 == 0 {
281            result.push(',');
282        }
283        result.push(c);
284    }
285    result.chars().rev().collect()
286}
287
288/// Format an `i64` with comma separators (handles negatives).
289fn fmt_i64(n: i64) -> String {
290    if n < 0 {
291        format!("-{}", fmt_num(n.unsigned_abs()))
292    } else {
293        fmt_num(n.unsigned_abs())
294    }
295}
296
297/// Format a duration as a human-readable string.
298fn fmt_duration(d: Duration) -> String {
299    let total_ms = d.as_millis();
300    if total_ms < 1000 {
301        format!("{total_ms}ms")
302    } else if total_ms < 60_000 {
303        format!("{:.2}s", d.as_secs_f64())
304    } else {
305        let mins = d.as_secs() / 60;
306        let secs = d.as_secs() % 60;
307        format!("{mins}m {secs:02}s")
308    }
309}
310
311/// Short human-readable label for an assertion kind.
312fn kind_label(kind: AssertKind) -> &'static str {
313    match kind {
314        AssertKind::Always => "always",
315        AssertKind::AlwaysOrUnreachable => "always?",
316        AssertKind::Sometimes => "sometimes",
317        AssertKind::Reachable => "reachable",
318        AssertKind::Unreachable => "unreachable",
319        AssertKind::NumericAlways => "num-always",
320        AssertKind::NumericSometimes => "numeric",
321        AssertKind::BooleanSometimesAll => "frontier",
322    }
323}
324
325/// Sort key for grouping assertion kinds in display.
326fn kind_sort_order(kind: AssertKind) -> u8 {
327    match kind {
328        AssertKind::Always => 0,
329        AssertKind::AlwaysOrUnreachable => 1,
330        AssertKind::Unreachable => 2,
331        AssertKind::NumericAlways => 3,
332        AssertKind::Sometimes => 4,
333        AssertKind::Reachable => 5,
334        AssertKind::NumericSometimes => 6,
335        AssertKind::BooleanSometimesAll => 7,
336    }
337}
338
339// ---------------------------------------------------------------------------
340// Display impl
341// ---------------------------------------------------------------------------
342
343fn fmt_summary(f: &mut fmt::Formatter<'_>, report: &SimulationReport) -> fmt::Result {
344    writeln!(f, "=== Simulation Report ===")?;
345    writeln!(
346        f,
347        "  Iterations: {}  |  Passed: {}  |  Failed: {}  |  Rate: {:.1}%",
348        report.iterations,
349        report.successful_runs,
350        report.failed_runs,
351        report.success_rate()
352    )?;
353    writeln!(f)
354}
355
356fn fmt_timing(f: &mut fmt::Formatter<'_>, report: &SimulationReport) -> fmt::Result {
357    writeln!(
358        f,
359        "  Avg Wall Time:     {:<14}Total: {}",
360        fmt_duration(report.average_wall_time()),
361        fmt_duration(report.metrics.wall_time)
362    )?;
363    writeln!(
364        f,
365        "  Avg Sim Time:      {}",
366        fmt_duration(report.average_simulated_time())
367    )?;
368    writeln!(
369        f,
370        "  Avg Events:        {}",
371        fmt_num(f64_to_u64_saturating(report.average_events_processed()))
372    )
373}
374
375/// Render the `UntilCoverageStable` saturation outcome: which signal it
376/// plateaued on, the coverage numbers, and whether the cap was hit first.
377fn fmt_saturation(f: &mut fmt::Formatter<'_>, report: &SimulationReport) -> fmt::Result {
378    let Some(sat) = &report.saturation else {
379        return Ok(());
380    };
381    writeln!(f)?;
382    let status = if report.convergence_timeout {
383        "NOT saturated"
384    } else {
385        "saturated"
386    };
387    match sat.signal {
388        SaturationSignal::CodeCoverage => writeln!(
389            f,
390            "  {status}: {}/{} sometimes hit · code coverage stable at {}/{} edges for {} seeds",
391            sat.sometimes_hit,
392            sat.sometimes_total,
393            sat.edges_covered,
394            sat.edges_total,
395            sat.plateau_seeds,
396        )?,
397        SaturationSignal::AssertionCoverage => writeln!(
398            f,
399            "  {status}: {}/{} sometimes hit · assertion coverage stable for {} seeds (no sancov — run via 'cargo xtask sim run' for code-coverage-driven stop)",
400            sat.sometimes_hit, sat.sometimes_total, sat.plateau_seeds,
401        )?,
402    }
403    if report.convergence_timeout {
404        writeln!(
405            f,
406            "  UntilCoverageStable hit the iteration cap without saturating."
407        )?;
408    }
409    Ok(())
410}
411
412fn fmt_exploration(f: &mut fmt::Formatter<'_>, exp: &ExplorationReport) -> fmt::Result {
413    writeln!(f)?;
414    writeln!(f, "--- Exploration ---")?;
415    writeln!(
416        f,
417        "  Timelines:    {:<18}Bugs found:     {}",
418        fmt_num(exp.total_timelines),
419        fmt_num(exp.bugs_found)
420    )?;
421    writeln!(
422        f,
423        "  Fork points:  {:<18}Coverage:       {} / {} bits ({:.1}%)",
424        fmt_num(exp.fork_points),
425        fmt_num(u64::from(exp.coverage_bits)),
426        fmt_num(u64::from(exp.coverage_total)),
427        if exp.coverage_total > 0 {
428            (f64::from(exp.coverage_bits) / f64::from(exp.coverage_total)) * 100.0
429        } else {
430            0.0
431        }
432    )?;
433    if exp.sancov_edges_total > 0 {
434        let covered = u64::try_from(exp.sancov_edges_covered).unwrap_or(u64::MAX);
435        let total = u64::try_from(exp.sancov_edges_total).unwrap_or(u64::MAX);
436        let covered_u32 = u32::try_from(exp.sancov_edges_covered).unwrap_or(u32::MAX);
437        let total_u32 = u32::try_from(exp.sancov_edges_total).unwrap_or(u32::MAX);
438        writeln!(
439            f,
440            "  Sancov:       {} / {} edges ({:.1}%)",
441            fmt_num(covered),
442            fmt_num(total),
443            (f64::from(covered_u32) / f64::from(total_u32)) * 100.0
444        )?;
445    }
446    writeln!(
447        f,
448        "  Energy left:  {:<18}Realloc pool:   {}",
449        fmt_i64(exp.energy_remaining),
450        fmt_i64(exp.realloc_pool_remaining)
451    )?;
452    for br in &exp.bug_recipes {
453        writeln!(
454            f,
455            "  Bug recipe (seed={}): {}",
456            br.seed,
457            format_recipe(&br.recipe)
458        )?;
459    }
460    Ok(())
461}
462
463fn fmt_assertion_detail(f: &mut fmt::Formatter<'_>, detail: &AssertionDetail) -> fmt::Result {
464    let status_tag = match detail.status {
465        AssertionStatus::Pass => "PASS",
466        AssertionStatus::Fail => "FAIL",
467        AssertionStatus::Miss => "MISS",
468    };
469    let kind_tag = kind_label(detail.kind);
470    let quoted_msg = format!("\"{}\"", detail.msg);
471
472    match detail.kind {
473        AssertKind::Sometimes | AssertKind::Reachable => {
474            let total = detail.pass_count + detail.fail_count;
475            let rate = if total > 0 {
476                let pass_u32 = u32::try_from(detail.pass_count).unwrap_or(u32::MAX);
477                let total_u32 = u32::try_from(total).unwrap_or(u32::MAX);
478                (f64::from(pass_u32) / f64::from(total_u32)) * 100.0
479            } else {
480                0.0
481            };
482            writeln!(
483                f,
484                "  {}  [{:<10}]  {:<34}  {} / {} ({:.1}%)",
485                status_tag,
486                kind_tag,
487                quoted_msg,
488                fmt_num(detail.pass_count),
489                fmt_num(total),
490                rate
491            )
492        }
493        AssertKind::NumericSometimes | AssertKind::NumericAlways => writeln!(
494            f,
495            "  {}  [{:<10}]  {:<34}  {} pass  {} fail  watermark: {}",
496            status_tag,
497            kind_tag,
498            quoted_msg,
499            fmt_num(detail.pass_count),
500            fmt_num(detail.fail_count),
501            detail.watermark
502        ),
503        AssertKind::BooleanSometimesAll => writeln!(
504            f,
505            "  {}  [{:<10}]  {:<34}  {} calls  frontier: {}",
506            status_tag,
507            kind_tag,
508            quoted_msg,
509            fmt_num(detail.pass_count),
510            detail.frontier
511        ),
512        _ => writeln!(
513            f,
514            "  {}  [{:<10}]  {:<34}  {} pass  {} fail",
515            status_tag,
516            kind_tag,
517            quoted_msg,
518            fmt_num(detail.pass_count),
519            fmt_num(detail.fail_count)
520        ),
521    }
522}
523
524fn fmt_assertion_details(f: &mut fmt::Formatter<'_>, details: &[AssertionDetail]) -> fmt::Result {
525    if details.is_empty() {
526        return Ok(());
527    }
528    writeln!(f)?;
529    writeln!(f, "--- Assertions ({}) ---", details.len())?;
530
531    let mut sorted: Vec<&AssertionDetail> = details.iter().collect();
532    sorted.sort_by(|a, b| {
533        kind_sort_order(a.kind)
534            .cmp(&kind_sort_order(b.kind))
535            .then(a.status.cmp(&b.status))
536            .then(a.msg.cmp(&b.msg))
537    });
538
539    for detail in &sorted {
540        fmt_assertion_detail(f, detail)?;
541    }
542    Ok(())
543}
544
545impl fmt::Display for SimulationReport {
546    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
547        fmt_summary(f, self)?;
548        fmt_timing(f, self)?;
549
550        // === Faulty Seeds ===
551        if !self.seeds_failing.is_empty() {
552            writeln!(f)?;
553            writeln!(f, "  Faulty seeds: {:?}", self.seeds_failing)?;
554        }
555
556        if let Some(ref exp) = self.exploration {
557            fmt_exploration(f, exp)?;
558        }
559
560        fmt_assertion_details(f, &self.assertion_details)?;
561
562        // === Assertion Violations ===
563        if !self.assertion_violations.is_empty() {
564            writeln!(f)?;
565            writeln!(f, "--- Assertion Violations ---")?;
566            for v in &self.assertion_violations {
567                writeln!(f, "  - {v}")?;
568            }
569        }
570
571        // === Coverage Gaps ===
572        if !self.coverage_violations.is_empty() {
573            writeln!(f)?;
574            writeln!(f, "--- Coverage Gaps ---")?;
575            for v in &self.coverage_violations {
576                writeln!(f, "  - {v}")?;
577            }
578        }
579
580        // === Buckets ===
581        if !self.bucket_summaries.is_empty() {
582            let total_buckets: usize = self
583                .bucket_summaries
584                .iter()
585                .map(|s| s.buckets_discovered)
586                .sum();
587            writeln!(f)?;
588            writeln!(
589                f,
590                "--- Buckets ({} across {} sites) ---",
591                total_buckets,
592                self.bucket_summaries.len()
593            )?;
594            for bs in &self.bucket_summaries {
595                writeln!(
596                    f,
597                    "  {:<34}  {:>3} buckets  {:>8} hits",
598                    format!("\"{}\"", bs.msg),
599                    bs.buckets_discovered,
600                    fmt_num(bs.total_hits)
601                )?;
602            }
603        }
604
605        // === Saturation (UntilCoverageStable) ===
606        fmt_saturation(f, self)?;
607        if self.saturation.is_none() && self.convergence_timeout {
608            writeln!(f)?;
609            writeln!(
610                f,
611                "  UntilCoverageStable hit the iteration cap without saturating."
612            )?;
613        }
614
615        // === Per-Seed Metrics ===
616        if self.seeds_used.len() > 1 {
617            writeln!(f)?;
618            writeln!(f, "--- Seeds ---")?;
619            let per_seed_tl = self.exploration.as_ref().map(|e| &e.per_seed_timelines);
620            for (i, seed) in self.seeds_used.iter().enumerate() {
621                if let Some(Ok(m)) = self.individual_metrics.get(i) {
622                    let tl_suffix = per_seed_tl
623                        .and_then(|v| v.get(i))
624                        .map(|t| format!("  timelines={}", fmt_num(*t)))
625                        .unwrap_or_default();
626                    writeln!(
627                        f,
628                        "  #{:<3}  seed={:<14}  wall={:<10}  sim={:<10}  events={}{}",
629                        i + 1,
630                        seed,
631                        fmt_duration(m.wall_time),
632                        fmt_duration(m.simulated_time),
633                        fmt_num(m.events_processed),
634                        tl_suffix,
635                    )?;
636                } else if let Some(Err(_)) = self.individual_metrics.get(i) {
637                    writeln!(f, "  #{:<3}  seed={:<14}  FAILED", i + 1, seed)?;
638                }
639            }
640        }
641
642        writeln!(f)?;
643        Ok(())
644    }
645}
moonpool_sim/runner/report.rs

moonpool_sim/runner/
report.rs
