Skip to main content

magi/
stats.rs

1//! Aggregate statistics over every recorded run.
2//!
3//! These tables are a by-product of running the graph, not a benchmark. The
4//! seat assignment rotates, the task distribution is whatever the operator
5//! happened to ask for, and a model that draws harder tasks looks worse. Read
6//! them as "relative performance on my workload", which is the only claim the
7//! data supports.
8use std::collections::BTreeMap;
9use std::path::{Path, PathBuf};
10
11use crate::run::{RunState, RunStatus, list_ids};
12
13/// Implementation record for one agent.
14#[derive(Debug, Clone, Default)]
15pub struct AgentStats {
16    /// Agent id.
17    pub agent: String,
18    /// Candidates it produced that were judged.
19    pub entered: usize,
20    /// Competitions it won.
21    pub wins: usize,
22    /// Candidates that produced no change at all.
23    pub empty: usize,
24}
25
26impl AgentStats {
27    /// Win rate over entries, as a percentage.
28    pub fn win_rate(&self) -> f64 {
29        if self.entered == 0 {
30            0.0
31        } else {
32            100.0 * self.wins as f64 / self.entered as f64
33        }
34    }
35}
36
37/// Review record for one agent.
38#[derive(Debug, Clone, Default)]
39pub struct ReviewerStats {
40    /// Agent id.
41    pub agent: String,
42    /// Review rounds it sat in whose adoption could be scored — a round whose
43    /// fixer never reported back is excluded, so this is the denominator of
44    /// [`Self::adopted_per_round`], not a headcount of appearances. For that,
45    /// see [`Self::seated`].
46    pub rounds: usize,
47    /// Review rounds it was on the panel for at all, scoreable or not.
48    /// Whether a seat answered is a fact about the seat and does not depend
49    /// on what later became of the fixer's report, so this — not `rounds` —
50    /// is the honest denominator for [`Self::timeout_rate`].
51    pub seated: usize,
52    /// Findings it submitted.
53    pub submitted: usize,
54    /// Findings the fixer acted on.
55    pub adopted: usize,
56    /// Findings no other reviewer in the same round also raised.
57    pub unique: usize,
58    /// Rounds it was seated in but never answered (timeout, crash, unparsable
59    /// output) — kept apart from `submitted`/`adopted` so a silent seat
60    /// cannot read as a seat with nothing to say.
61    pub timeouts: usize,
62}
63
64impl ReviewerStats {
65    /// Adopted findings per round: how much signal one seat produces.
66    pub fn adopted_per_round(&self) -> f64 {
67        if self.rounds == 0 {
68            0.0
69        } else {
70            self.adopted as f64 / self.rounds as f64
71        }
72    }
73
74    /// Adopted over submitted: how often its findings are real. Rounds where
75    /// the seat never answered are not in `submitted`, so a timeout cannot
76    /// dilute (or hide behind) this rate.
77    pub fn precision(&self) -> f64 {
78        if self.submitted == 0 {
79            0.0
80        } else {
81            100.0 * self.adopted as f64 / self.submitted as f64
82        }
83    }
84
85    /// Share of its findings that only it saw.
86    pub fn unique_rate(&self) -> f64 {
87        if self.submitted == 0 {
88            0.0
89        } else {
90            100.0 * self.unique as f64 / self.submitted as f64
91        }
92    }
93
94    /// Share of the rounds it was seated in where it never answered.
95    pub fn timeout_rate(&self) -> f64 {
96        if self.seated == 0 {
97            0.0
98        } else {
99            100.0 * self.timeouts as f64 / self.seated as f64
100        }
101    }
102}
103
104/// Design-deliberation record for one agent.
105///
106/// Approximate by construction, same as [`crate::advise::Reflection`] itself:
107/// `strong`/`faint` come from a word-overlap heuristic against the synthesis
108/// brief, not from an explicit attribution, so [`Self::reflection_rate`] is a
109/// rough read on whose ideas seemed to land, not a precise credit split.
110#[derive(Debug, Clone, Default)]
111pub struct AdvisorStats {
112    /// Agent id.
113    pub agent: String,
114    /// Advisor seats it occupied, across every run with `[graph] advise` on.
115    pub seated: usize,
116    /// Seats where it produced a usable proposal (`record.proposal.is_some()`).
117    pub proposed: usize,
118    /// Seats where it produced no usable proposal at all (crashed, timed
119    /// out, or answered with nothing a proposal could be parsed out of).
120    /// Counted from `record.proposal.is_none()` directly rather than from
121    /// `record.reflection == Absent` — `reflection` is `#[serde(default)]`
122    /// and can be left at its default on a record from before
123    /// `apply_reflection` ran, which would otherwise double as a false
124    /// "absent".
125    pub absent: usize,
126    /// Proposals the synthesis brief carried little or no recognisable
127    /// trace of.
128    pub faint: usize,
129    /// Proposals the synthesis brief named outright or carried enough of to
130    /// count as a clear match.
131    pub strong: usize,
132}
133
134impl AdvisorStats {
135    /// Share of its proposals rated `Strong`, as a percentage. The
136    /// denominator is `proposed`, not `seated` — a seat that never produced
137    /// a proposal had nothing for the synthesis to reflect, so it cannot
138    /// count against this rate any more than a reviewer's silence counts
139    /// against [`ReviewerStats::precision`].
140    ///
141    /// Same caveat as the struct itself: this is a heuristic read on
142    /// reflection, not a precise attribution.
143    pub fn reflection_rate(&self) -> f64 {
144        if self.proposed == 0 {
145            0.0
146        } else {
147            100.0 * self.strong as f64 / self.proposed as f64
148        }
149    }
150}
151
152/// What real-machine verification caught that static review did not.
153#[derive(Debug, Clone, Default)]
154pub struct E2eStats {
155    /// Rounds where E2E commands ran.
156    pub rounds: usize,
157    /// Rounds where E2E failed.
158    pub failures: usize,
159    /// Rounds where E2E failed and no reviewer had raised a blocking finding —
160    /// a runtime defect that only execution found.
161    pub sole_detections: usize,
162    /// Rounds where E2E was deferred rather than run: blocking findings
163    /// already required a fix, so the round went straight to the fixer
164    /// instead of spending a full verify run on a head about to change. Kept
165    /// separate from [`Self::rounds`] on purpose — a deferred round never
166    /// ran anything, so counting it there would misreport how often E2E
167    /// actually executed.
168    pub deferred: usize,
169}
170
171impl E2eStats {
172    /// Share of E2E failures that static review had missed entirely.
173    pub fn sole_rate(&self) -> f64 {
174        if self.failures == 0 {
175            0.0
176        } else {
177            100.0 * self.sole_detections as f64 / self.failures as f64
178        }
179    }
180}
181
182/// Run-level counters.
183#[derive(Debug, Clone, Default)]
184pub struct Totals {
185    /// Runs on disk.
186    pub runs: usize,
187    /// Reached a merge.
188    pub merged: usize,
189    /// Passed the gate, merge not requested.
190    pub ready: usize,
191    /// Stopped with findings open or a red gate.
192    pub blocked: usize,
193    /// Could not complete.
194    pub failed: usize,
195    /// The judging panel never reached a quorum; the work is kept but the
196    /// verdict is not trustworthy. Never counted as `blocked` — see
197    /// `RunStatus::Stalled`'s own doc — so it needs its own counter to stay
198    /// visible in a breakdown rather than vanishing from every bucket.
199    pub stalled: usize,
200    /// Every candidate wrote nothing, and said why in a way that survived
201    /// the adoption guard: a claim, not a failure, and not yet verified.
202    pub verified_noop: usize,
203    /// A later attempt at the same task already finished it; this run's own
204    /// `blocked`/`stalled` no longer needs anyone's attention.
205    pub superseded: usize,
206    /// Still moving: any non-terminal status (prep through landing). Kept as
207    /// one bucket rather than one counter per node — the per-node position of
208    /// a live run belongs to `magi show`/the deck, not to a workload-wide
209    /// tally that is read well after the run in question has finished.
210    pub in_progress: usize,
211    /// Runs that reached a tally.
212    pub tallied: usize,
213    /// Tallies where the judges' first choices disagreed.
214    pub split: usize,
215    /// Tallies that went through deliberation.
216    pub deliberated: usize,
217    /// Deliberated runs where at least one judge moved.
218    pub minds_changed: usize,
219    /// Deliberated runs that ended unanimous.
220    pub converged: usize,
221    /// Review rounds across all runs.
222    pub review_rounds: usize,
223}
224
225impl Totals {
226    /// Merged or ready over all runs.
227    pub fn completion_rate(&self) -> f64 {
228        if self.runs == 0 {
229            0.0
230        } else {
231            100.0 * (self.merged + self.ready) as f64 / self.runs as f64
232        }
233    }
234
235    /// Share of tallies that were split.
236    pub fn split_rate(&self) -> f64 {
237        if self.tallied == 0 {
238            0.0
239        } else {
240            100.0 * self.split as f64 / self.tallied as f64
241        }
242    }
243}
244
245/// Per-node duration breakdown, aggregated across every loaded run.
246///
247/// A duration here is the span between a node's *first* and *last* recorded
248/// event within one run — not time actually spent working. A node visited
249/// more than twice in one run (a retry, or a park/resume gap) has any idle
250/// time in between folded into that span, so this reads as an upper bound on
251/// the node's wall-clock cost, not a measurement of it.
252#[derive(Debug, Clone, Default, PartialEq)]
253pub struct NodeDuration {
254    /// Node name, as it appears in `Event::node`.
255    pub node: String,
256    /// Runs where the node's span could be measured (at least two events).
257    pub runs: usize,
258    /// Sum of every measured run's span, in seconds.
259    pub total_secs: i64,
260    /// The longest single run's span, in seconds.
261    pub max_secs: i64,
262    /// Runs where the node had exactly one event — visited, but with no
263    /// second timestamp to measure a span against. Counted apart from `runs`
264    /// so a caller never has to guess whether a low mean hides unmeasured
265    /// visits.
266    pub single: usize,
267}
268
269impl NodeDuration {
270    /// Mean span across the runs that could be measured, in seconds.
271    ///
272    /// `0.0` when `runs` is zero, never a division by zero.
273    pub fn mean_secs(&self) -> f64 {
274        if self.runs == 0 {
275            0.0
276        } else {
277            self.total_secs as f64 / self.runs as f64
278        }
279    }
280}
281
282/// Derive a per-node duration breakdown from every loaded run's events.
283///
284/// For each run, a node's span is the gap between its earliest and latest
285/// [`crate::run::Event`] in that run. A node with fewer than two events in a
286/// run contributes no span (see [`NodeDuration::single`] for the one-event
287/// case); a run with no events for a node contributes nothing at all. Spans
288/// from every run are then summed per node.
289pub fn node_durations<'a>(states: impl IntoIterator<Item = &'a RunState>) -> Vec<NodeDuration> {
290    let mut nodes: BTreeMap<String, NodeDuration> = BTreeMap::new();
291
292    for state in states {
293        let mut spans: BTreeMap<&str, (jiff::Timestamp, jiff::Timestamp, usize)> = BTreeMap::new();
294        for e in &state.events {
295            spans
296                .entry(e.node.as_str())
297                .and_modify(|(min, max, count)| {
298                    if e.at < *min {
299                        *min = e.at;
300                    }
301                    if e.at > *max {
302                        *max = e.at;
303                    }
304                    *count += 1;
305                })
306                .or_insert((e.at, e.at, 1));
307        }
308
309        for (node, (min, max, count)) in spans {
310            let entry = nodes
311                .entry(node.to_owned())
312                .or_insert_with(|| NodeDuration {
313                    node: node.to_owned(),
314                    ..NodeDuration::default()
315                });
316            if count < 2 {
317                entry.single += 1;
318                continue;
319            }
320            // `max - min` is never negative: both are the extremes of the
321            // same event set, and equal timestamps yield a zero-second span
322            // rather than being mistaken for an unmeasured visit.
323            let span_secs = (max - min).get_seconds();
324            entry.runs += 1;
325            entry.total_secs += span_secs;
326            if span_secs > entry.max_secs {
327                entry.max_secs = span_secs;
328            }
329        }
330    }
331
332    let mut nodes: Vec<NodeDuration> = nodes.into_values().collect();
333    nodes.sort_by(|a, b| b.total_secs.cmp(&a.total_secs).then(a.node.cmp(&b.node)));
334    nodes
335}
336
337/// What the post-merge release-bump step did, across every merged run.
338///
339/// `merged` is the denominator for [`Self::coverage_rate`]: a repository
340/// that never uses the release-bump step (no `auto-tag.yml`, see
341/// `kata:agents:rust:*`) should read as "0 of N merged runs recorded a
342/// bump", not vanish from the report the way it would if `recorded` were the
343/// denominator instead — silently excluding those runs would hide the
344/// coverage gap itself.
345#[derive(Debug, Clone, Default)]
346pub struct ReleaseBumpStats {
347    /// Merged runs, the denominator for [`Self::coverage_rate`].
348    pub merged: usize,
349    /// Merged runs that carry a [`crate::run::ReleaseBump`] record at all.
350    pub recorded: usize,
351    /// Recorded bumps that opened a release pull request.
352    pub pr_opened: usize,
353    /// Recorded bumps whose `automerge_enabled` is `true` at the point the
354    /// bump step finished. This is the *final* state, not "was automerge
355    /// enabled at some point" — `bump::surface_problem` flips it back to
356    /// `false` when GitHub later rejects automerge, so a bump that briefly
357    /// enabled it and then had it rejected counts here as not enabled.
358    pub automerge_enabled: usize,
359    /// Recorded bumps magi merged directly because GitHub refused automerge
360    /// on an already-green pull request. Counted apart from
361    /// `automerge_enabled` on purpose: a `merged_directly` bump needed no
362    /// human even though automerge itself never took, so folding it into
363    /// (or leaving it out of) the automerge count would misstate either
364    /// number.
365    pub merged_directly: usize,
366    /// Recorded bumps that ended with [`crate::run::RunState::needs_attention`]
367    /// true — a human has something to do.
368    pub needs_attention: usize,
369}
370
371impl ReleaseBumpStats {
372    /// Share of merged runs that recorded a release bump at all.
373    pub fn coverage_rate(&self) -> f64 {
374        if self.merged == 0 {
375            0.0
376        } else {
377            100.0 * self.recorded as f64 / self.merged as f64
378        }
379    }
380
381    /// Share of opened release PRs that ended with automerge enabled.
382    pub fn automerge_rate(&self) -> f64 {
383        if self.pr_opened == 0 {
384            0.0
385        } else {
386            100.0 * self.automerge_enabled as f64 / self.pr_opened as f64
387        }
388    }
389
390    /// Share of recorded bumps that needed a human.
391    pub fn attention_rate(&self) -> f64 {
392        if self.recorded == 0 {
393            0.0
394        } else {
395            100.0 * self.needs_attention as f64 / self.recorded as f64
396        }
397    }
398
399    /// Recorded bumps that finished with nothing for a human to do.
400    ///
401    /// Deliberately the difference `recorded - needs_attention`, not a
402    /// separate counter kept in step with `automerge_enabled`: a bump that
403    /// GitHub refused automerge on but that magi merged directly
404    /// (`merged_directly`) is clean — nobody had to act — even though
405    /// `automerge_enabled` is `false` for it. Counting clean bumps as
406    /// `automerge_enabled` alone would leave `merged_directly` cases in
407    /// neither the clean nor the attention bucket, and the two would stop
408    /// summing to `recorded`.
409    pub fn clean(&self) -> usize {
410        self.recorded.saturating_sub(self.needs_attention)
411    }
412}
413
414/// Everything, aggregated.
415#[derive(Debug, Clone, Default)]
416pub struct Stats {
417    /// Run counters.
418    pub totals: Totals,
419    /// Per-agent implementation record, best win rate first.
420    pub agents: Vec<AgentStats>,
421    /// Per-agent review record, most adopted-per-round first.
422    pub reviewers: Vec<ReviewerStats>,
423    /// Per-agent design-deliberation record, highest reflection rate first.
424    /// Only runs where `[graph] advise` produced an [`crate::advise::Advice`]
425    /// contribute — a run with the stage off carries no signal either way,
426    /// and counting it would water down every agent's rate with seats that
427    /// were never asked.
428    pub advisors: Vec<AdvisorStats>,
429    /// Verification record.
430    pub e2e: E2eStats,
431    /// Per-node duration breakdown, longest total first.
432    pub nodes: Vec<NodeDuration>,
433    /// Post-merge release-bump record, over every merged run.
434    pub release_bumps: ReleaseBumpStats,
435}
436
437/// Load every run on disk, skipping any that cannot be read.
438pub fn load_all() -> Vec<RunState> {
439    list_ids()
440        .into_iter()
441        .filter_map(|id| RunState::load(&id).ok())
442        .collect()
443}
444
445/// One repository's [`Stats`], as grouped by [`by_repo`].
446#[derive(Debug, Clone)]
447pub struct RepoStats {
448    /// The grouping key: `RunState.repo` exactly as recorded, a normalised
449    /// full path. Never a display name — this is what a caller (the CLI's
450    /// `--repo` fallback, the web `?repo=` query) matches back against, and
451    /// matching by name would conflate two different checkouts that happen
452    /// to share a leaf directory.
453    pub repo: PathBuf,
454    /// Display name: `repo`'s file name, or the full path when it has none
455    /// (e.g. `/`). Collisions between repositories are the caller's problem
456    /// to disambiguate (see `report::repo_summary`), not this struct's.
457    pub name: String,
458    /// This repository's own aggregate, counted exactly as [`collect`]
459    /// counts the whole workload.
460    pub stats: Stats,
461}
462
463/// Group `states` by [`RunState::repo`] and aggregate each group with
464/// [`collect_refs`] — the same counting logic as [`collect`], just scoped to
465/// one repository at a time.
466///
467/// Sorted by run count descending, then by repo path ascending on ties, so
468/// the busiest repository leads the summary table.
469pub fn by_repo(states: &[RunState]) -> Vec<RepoStats> {
470    let mut groups: BTreeMap<PathBuf, Vec<&RunState>> = BTreeMap::new();
471    for state in states {
472        groups.entry(state.repo.clone()).or_default().push(state);
473    }
474
475    let mut out: Vec<RepoStats> = groups
476        .into_iter()
477        .map(|(repo, group)| {
478            let stats = collect_refs(group);
479            let name = repo
480                .file_name()
481                .map(|n| n.to_string_lossy().into_owned())
482                .unwrap_or_else(|| repo.to_string_lossy().into_owned());
483            RepoStats { repo, name, stats }
484        })
485        .collect();
486    out.sort_by(|a, b| {
487        b.stats
488            .totals
489            .runs
490            .cmp(&a.stats.totals.runs)
491            .then(a.repo.cmp(&b.repo))
492    });
493    out
494}
495
496/// Narrow `states` to those recorded against `repo` exactly.
497///
498/// Matches [`RunState::repo`] by full-path equality only, never by name — the
499/// name-fallback resolution a caller may want (a checkout that has since
500/// moved or been deleted) belongs one layer up, where a config or filesystem
501/// lookup can decide what "the same repository" means; this function has no
502/// such context and would otherwise risk conflating two different checkouts
503/// that happen to share a leaf directory.
504pub fn filter_repo<'a>(states: &'a [RunState], repo: &Path) -> Vec<&'a RunState> {
505    states.iter().filter(|s| s.repo == repo).collect()
506}
507
508/// Aggregate `states`.
509pub fn collect(states: &[RunState]) -> Stats {
510    collect_refs(states)
511}
512
513/// Aggregate `states`, over any iterator of references rather than a slice —
514/// what [`by_repo`] uses to run the same counting logic over each repository's
515/// own group without collecting it into an owned `Vec<RunState>` first.
516/// [`collect`] is a thin wrapper around this for the common whole-slice case.
517pub fn collect_refs<'a>(states: impl IntoIterator<Item = &'a RunState>) -> Stats {
518    let states: Vec<&'a RunState> = states.into_iter().collect();
519    let mut totals = Totals::default();
520    let mut agents: BTreeMap<String, AgentStats> = BTreeMap::new();
521    let mut reviewers: BTreeMap<String, ReviewerStats> = BTreeMap::new();
522    let mut advisors: BTreeMap<String, AdvisorStats> = BTreeMap::new();
523    let mut e2e = E2eStats::default();
524    let mut release_bumps = ReleaseBumpStats::default();
525
526    for state in &states {
527        totals.runs += 1;
528        if state.status == RunStatus::Merged {
529            release_bumps.merged += 1;
530            if let Some(b) = &state.release_bump {
531                release_bumps.recorded += 1;
532                if b.pr_url.is_some() {
533                    release_bumps.pr_opened += 1;
534                }
535                if b.automerge_enabled {
536                    release_bumps.automerge_enabled += 1;
537                }
538                if b.merged_directly {
539                    release_bumps.merged_directly += 1;
540                }
541            }
542            if state.needs_attention() {
543                release_bumps.needs_attention += 1;
544            }
545        }
546        match state.status {
547            RunStatus::Merged => totals.merged += 1,
548            RunStatus::Ready => totals.ready += 1,
549            RunStatus::Blocked => totals.blocked += 1,
550            RunStatus::Failed => totals.failed += 1,
551            RunStatus::Stalled => totals.stalled += 1,
552            RunStatus::VerifiedNoop => totals.verified_noop += 1,
553            // Counted on its own rather than folded into `blocked`/`stalled`:
554            // the task it belongs to already landed through a later run,
555            // which is the one this tally counts as the merge/ready outcome,
556            // and folding it back in would inflate the denominator with two
557            // outcomes for one task.
558            //
559            // `AlreadyInBase` is the same story from the other side: the
560            // change reached the base by another route, so it shares the
561            // bucket instead of becoming a second outcome for one change.
562            RunStatus::Superseded | RunStatus::AlreadyInBase => totals.superseded += 1,
563            RunStatus::Prep
564            | RunStatus::Implementing
565            | RunStatus::Judging
566            | RunStatus::Deliberating
567            | RunStatus::Voting
568            | RunStatus::Reviewing
569            | RunStatus::Gating
570            | RunStatus::Landing => totals.in_progress += 1,
571        }
572
573        for c in &state.candidates {
574            let entry = agents.entry(c.agent.clone()).or_insert_with(|| AgentStats {
575                agent: c.agent.clone(),
576                ..AgentStats::default()
577            });
578            // A verified no-op is not counted as the ordinary empty loss it
579            // would otherwise look like: the candidate gave evidence for
580            // writing nothing, which `entry.empty` exists to flag the
581            // *absence* of.
582            if c.empty && c.verified_noop.is_none() {
583                entry.empty += 1;
584            }
585            if c.viable() {
586                entry.entered += 1;
587            }
588        }
589
590        if let Some(t) = &state.tally {
591            // A tally with no panel (`uncontested`) never split, never
592            // deliberated and never converged — it never happened, and
593            // folding it into the denominator would understate the real
594            // split rate with runs that carry no panel-agreement signal at
595            // all. The winner still earns its agent a win either way: an
596            // uncontested candidate is still the one that shipped.
597            if t.uncontested.is_none() {
598                totals.tallied += 1;
599                if !t.unanimous_initial {
600                    totals.split += 1;
601                }
602                if t.deliberated {
603                    totals.deliberated += 1;
604                    if t.changed_votes > 0 {
605                        totals.minds_changed += 1;
606                    }
607                    if t.unanimous_final {
608                        totals.converged += 1;
609                    }
610                }
611            }
612            if let Some(w) = state.candidates.iter().find(|c| c.label == t.winner) {
613                agents
614                    .entry(w.agent.clone())
615                    .or_insert_with(|| AgentStats {
616                        agent: w.agent.clone(),
617                        ..AgentStats::default()
618                    })
619                    .wins += 1;
620            }
621        }
622
623        if let Some(advice) = &state.advice {
624            for rec in &advice.records {
625                let entry = advisors
626                    .entry(rec.agent.clone())
627                    .or_insert_with(|| AdvisorStats {
628                        agent: rec.agent.clone(),
629                        ..AdvisorStats::default()
630                    });
631                entry.seated += 1;
632                if rec.proposal.is_none() {
633                    entry.absent += 1;
634                    continue;
635                }
636                entry.proposed += 1;
637                match rec.reflection {
638                    crate::advise::Reflection::Strong => entry.strong += 1,
639                    crate::advise::Reflection::Faint => entry.faint += 1,
640                    // A proposal exists, so this is not a real "no proposal"
641                    // reading — see `AdvisorStats::absent`'s own doc for why
642                    // that count comes from `proposal.is_none()` instead of
643                    // this field. `graph::Runner::advise` always calls
644                    // `apply_reflection` before saving, so the only way a
645                    // proposed record keeps the default `Absent` is a run.json
646                    // predating the `reflection` field. Fold it into `faint`
647                    // rather than dropping it from the breakdown entirely:
648                    // that is what `classify()` itself falls back to when
649                    // there is nothing to score against.
650                    crate::advise::Reflection::Absent => entry.faint += 1,
651                }
652            }
653        }
654
655        for round in &state.reviews {
656            totals.review_rounds += 1;
657
658            // A round whose fixer never reported back (crashed, timed out, or
659            // replied with something magi could not parse) leaves adoption
660            // unknown, not zero. Counting it would score every reviewer in
661            // that round as having been ignored, when the truth is simply
662            // unrecorded — so it stays out of the adoption-rate denominator
663            // entirely rather than silently becoming a round of 0 adoptions.
664            let report_lost = round.fix.as_ref().is_some_and(|f| f.failed.is_some());
665            let adopted: Vec<&String> = round
666                .fix
667                .as_ref()
668                .map(|f| f.addressed.iter().collect())
669                .unwrap_or_default();
670
671            for rec in &round.reviews {
672                let entry = reviewers
673                    .entry(rec.agent.clone())
674                    .or_insert_with(|| ReviewerStats {
675                        agent: rec.agent.clone(),
676                        ..ReviewerStats::default()
677                    });
678                // Seating and answering are facts about the seat itself: they
679                // hold whether or not this round's adoption is scoreable, so
680                // they are counted before the lost-report guard. A seat that
681                // never answered stays out of every scoring denominator —
682                // silence is not a review that found nothing.
683                entry.seated += 1;
684                if rec.failed.is_some() {
685                    entry.timeouts += 1;
686                    continue;
687                }
688                if report_lost {
689                    continue;
690                }
691                entry.rounds += 1;
692                entry.submitted += rec.findings.len();
693                for f in &rec.findings {
694                    if adopted.iter().any(|a| **a == f.id) {
695                        entry.adopted += 1;
696                    }
697                    let overlapped = round
698                        .reviews
699                        .iter()
700                        .filter(|other| other.reviewer != rec.reviewer)
701                        .flat_map(|other| other.findings.iter())
702                        .any(|g| same_defect(f, g));
703                    if !overlapped {
704                        entry.unique += 1;
705                    }
706                }
707            }
708
709            if round.e2e_deferred {
710                e2e.deferred += 1;
711            } else if !round.e2e.is_empty() {
712                e2e.rounds += 1;
713                if round.e2e.iter().any(|o| !o.ok()) {
714                    e2e.failures += 1;
715                    if round.blocking == 0 {
716                        e2e.sole_detections += 1;
717                    }
718                }
719            }
720        }
721    }
722
723    let mut agents: Vec<AgentStats> = agents.into_values().collect();
724    agents.sort_by(|a, b| {
725        b.win_rate()
726            .total_cmp(&a.win_rate())
727            .then(b.entered.cmp(&a.entered))
728    });
729    let mut reviewers: Vec<ReviewerStats> = reviewers.into_values().collect();
730    // A seat only sighted in rounds whose adoption could not be scored has
731    // nothing to report: no scoreable round, no silence to flag. It stays out
732    // of the table entirely rather than appearing as a row of zeroes, which
733    // would read as a reviewer that produced nothing.
734    reviewers.retain(|r| r.rounds > 0 || r.timeouts > 0);
735    reviewers.sort_by(|a, b| {
736        b.adopted_per_round()
737            .total_cmp(&a.adopted_per_round())
738            .then(b.rounds.cmp(&a.rounds))
739    });
740
741    let mut advisors: Vec<AdvisorStats> = advisors.into_values().collect();
742    advisors.sort_by(|a, b| {
743        b.reflection_rate()
744            .total_cmp(&a.reflection_rate())
745            .then(b.proposed.cmp(&a.proposed))
746    });
747
748    let nodes = node_durations(states);
749
750    Stats {
751        totals,
752        agents,
753        reviewers,
754        advisors,
755        e2e,
756        nodes,
757        release_bumps,
758    }
759}
760
761/// Do two findings describe the same defect?
762///
763/// A deliberate heuristic: same normalised title, or the same file within five
764/// lines. Two reviewers rarely word a finding identically, and exact matching
765/// would report every overlap as a unique find.
766fn same_defect(a: &crate::verdict::Finding, b: &crate::verdict::Finding) -> bool {
767    if normalize(&a.title) == normalize(&b.title) {
768        return true;
769    }
770    match (&a.file, &b.file) {
771        (Some(fa), Some(fb)) if fa == fb => match (a.line, b.line) {
772            (Some(la), Some(lb)) => la.abs_diff(lb) <= 5,
773            _ => false,
774        },
775        _ => false,
776    }
777}
778
779fn normalize(title: &str) -> String {
780    title
781        .chars()
782        .filter(|c| c.is_alphanumeric())
783        .map(|c| c.to_ascii_lowercase())
784        .collect()
785}
786
787#[cfg(test)]
788mod tests {
789    use super::*;
790    use crate::config::Config;
791    use crate::run::{Candidate, CommandOutcome, FixRecord, ReviewRecord, ReviewRound, Tally};
792    use crate::verdict::{Finding, Severity};
793    use std::path::PathBuf;
794
795    fn finding(id: &str, file: &str, line: u32, title: &str, sev: Severity) -> Finding {
796        Finding {
797            id: id.to_owned(),
798            severity: sev,
799            file: Some(file.to_owned()),
800            line: Some(line),
801            title: title.to_owned(),
802            detail: String::new(),
803        }
804    }
805
806    fn candidate(label: char, agent: &str) -> Candidate {
807        Candidate {
808            index: 0,
809            label,
810            agent: agent.to_owned(),
811            branch: format!("magi/x/{label}"),
812            worktree: PathBuf::from("/w"),
813            summary: String::new(),
814            stat: String::new(),
815            files: 1,
816            commits: 1,
817            empty: false,
818            failed: None,
819            verified_noop: None,
820            duration_ms: 0,
821            folded: false,
822        }
823    }
824
825    fn state_with(reviews: Vec<ReviewRound>, winner: char, status: RunStatus) -> RunState {
826        let mut s = RunState::new(
827            PathBuf::from("/repo"),
828            "main".to_owned(),
829            "abcdef".to_owned(),
830            "task".to_owned(),
831            Config::default(),
832        );
833        s.candidates = vec![candidate('A', "alpha"), candidate('B', "beta")];
834        s.tally = Some(Tally {
835            first_choice: BTreeMap::from([('A', 1), ('B', 2)]),
836            borda: BTreeMap::new(),
837            winner,
838            rankings: 3,
839            unanimous_initial: false,
840            deliberated: true,
841            changed_votes: 1,
842            unanimous_final: true,
843            tie_break: None,
844            judges: 3,
845            present: 3,
846            quorum: 2,
847            met_quorum: true,
848            uncontested: None,
849        });
850        s.reviews = reviews;
851        s.status = status;
852        s
853    }
854
855    fn state_with_repo(
856        reviews: Vec<ReviewRound>,
857        winner: char,
858        status: RunStatus,
859        repo: &str,
860    ) -> RunState {
861        let mut s = state_with(reviews, winner, status);
862        s.repo = PathBuf::from(repo);
863        s
864    }
865
866    #[test]
867    fn win_rates_and_completion_are_counted_per_agent() {
868        let states = vec![
869            state_with(Vec::new(), 'B', RunStatus::Merged),
870            state_with(Vec::new(), 'A', RunStatus::Blocked),
871        ];
872        let stats = collect(&states);
873        assert_eq!(stats.totals.runs, 2);
874        assert_eq!(stats.totals.merged, 1);
875        assert_eq!(stats.totals.blocked, 1);
876        assert_eq!(stats.totals.completion_rate(), 50.0);
877        assert_eq!(stats.totals.split, 2);
878        assert_eq!(stats.totals.minds_changed, 2);
879        assert_eq!(stats.totals.converged, 2);
880
881        let beta = stats.agents.iter().find(|a| a.agent == "beta").unwrap();
882        assert_eq!(beta.entered, 2);
883        assert_eq!(beta.wins, 1);
884        assert_eq!(beta.win_rate(), 50.0);
885    }
886
887    #[test]
888    fn reviewer_precision_and_uniqueness() {
889        let round = ReviewRound {
890            round: 1,
891            head: "h".to_owned(),
892            verified_head: None,
893            verified_at: None,
894            reviews: vec![
895                ReviewRecord {
896                    attempts: 0,
897                    reviewer: 1,
898                    agent: "alpha".to_owned(),
899                    summary: String::new(),
900                    findings: vec![
901                        finding(
902                            "R1-1-1",
903                            "src/a.rs",
904                            10,
905                            "panics on empty",
906                            Severity::Blocker,
907                        ),
908                        finding("R1-1-2", "src/b.rs", 40, "leaks a handle", Severity::Major),
909                    ],
910                    vote: None,
911                    failed: None,
912                    duration_ms: 0,
913                },
914                ReviewRecord {
915                    attempts: 0,
916                    reviewer: 2,
917                    agent: "beta".to_owned(),
918                    summary: String::new(),
919                    // Same defect as R1-1-1, three lines off: an overlap.
920                    findings: vec![finding(
921                        "R1-2-1",
922                        "src/a.rs",
923                        13,
924                        "empty input panic",
925                        Severity::Blocker,
926                    )],
927                    vote: None,
928                    failed: None,
929                    duration_ms: 0,
930                },
931            ],
932            e2e: Vec::new(),
933            verify_retried: false,
934            e2e_deferred: false,
935            e2e_defer_reason: None,
936            fix: Some(FixRecord {
937                agent: "alpha".to_owned(),
938                addressed: vec!["R1-1-1".to_owned()],
939                rejected: Vec::new(),
940                notes: String::new(),
941                committed: true,
942                failed: None,
943                duration_ms: 0,
944                continuation: None,
945            }),
946            blocking: 3,
947            answered: 2,
948            expected: 2,
949            clean: false,
950            progressed: true,
951            vote_split: false,
952            reconsideration: Vec::new(),
953            verdict: None,
954        };
955        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Ready)]);
956        let alpha = stats.reviewers.iter().find(|r| r.agent == "alpha").unwrap();
957        assert_eq!(alpha.submitted, 2);
958        assert_eq!(alpha.adopted, 1);
959        assert_eq!(alpha.precision(), 50.0);
960        assert_eq!(alpha.adopted_per_round(), 1.0);
961        // The src/a.rs finding overlaps beta's; src/b.rs does not.
962        assert_eq!(alpha.unique, 1);
963
964        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
965        assert_eq!(beta.submitted, 1);
966        assert_eq!(beta.adopted, 0);
967        assert_eq!(beta.unique, 0);
968    }
969
970    #[test]
971    fn a_lost_fix_report_does_not_count_as_zero_adoption() {
972        let submitted = ReviewRound {
973            round: 1,
974            head: "h".to_owned(),
975            verified_head: None,
976            verified_at: None,
977            reviews: vec![ReviewRecord {
978                attempts: 0,
979                reviewer: 1,
980                agent: "alpha".to_owned(),
981                summary: String::new(),
982                findings: vec![finding(
983                    "R1-1-1",
984                    "src/a.rs",
985                    10,
986                    "panics on empty",
987                    Severity::Blocker,
988                )],
989                vote: None,
990                failed: None,
991                duration_ms: 0,
992            }],
993            e2e: Vec::new(),
994            verify_retried: false,
995            e2e_deferred: false,
996            e2e_defer_reason: None,
997            // The fixer's diff may well have landed (blocking counts do fall
998            // round over round) — only its adoption report never came back.
999            fix: Some(FixRecord {
1000                agent: "alpha".to_owned(),
1001                addressed: Vec::new(),
1002                rejected: Vec::new(),
1003                notes: String::new(),
1004                committed: true,
1005                failed: Some("unparsable fix report".to_owned()),
1006                duration_ms: 0,
1007                continuation: None,
1008            }),
1009            blocking: 4,
1010            answered: 1,
1011            expected: 1,
1012            clean: false,
1013            progressed: false,
1014            vote_split: false,
1015            reconsideration: Vec::new(),
1016            verdict: None,
1017        };
1018        let stats = collect(&[state_with(vec![submitted], 'A', RunStatus::Ready)]);
1019        assert!(
1020            stats.reviewers.is_empty(),
1021            "a round with no adoption signal must not enter any reviewer's \
1022             denominator: {:?}",
1023            stats.reviewers
1024        );
1025    }
1026
1027    #[test]
1028    fn timed_out_seat_counts_as_a_timeout_not_a_clean_submission() {
1029        let round = ReviewRound {
1030            round: 1,
1031            head: "h".to_owned(),
1032            verified_head: None,
1033            verified_at: None,
1034            reviews: vec![
1035                ReviewRecord {
1036                    attempts: 0,
1037                    reviewer: 1,
1038                    agent: "alpha".to_owned(),
1039                    summary: String::new(),
1040                    findings: Vec::new(),
1041                    vote: None,
1042                    failed: None,
1043                    duration_ms: 0,
1044                },
1045                ReviewRecord {
1046                    attempts: 0,
1047                    reviewer: 2,
1048                    agent: "beta".to_owned(),
1049                    summary: String::new(),
1050                    findings: Vec::new(),
1051                    vote: None,
1052                    failed: Some("agent timed out".to_owned()),
1053                    duration_ms: 0,
1054                },
1055            ],
1056            e2e: Vec::new(),
1057            verify_retried: false,
1058            e2e_deferred: false,
1059            e2e_defer_reason: None,
1060            fix: None,
1061            blocking: 0,
1062            answered: 1,
1063            expected: 2,
1064            clean: false,
1065            progressed: false,
1066            vote_split: false,
1067            reconsideration: Vec::new(),
1068            verdict: None,
1069        };
1070        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Blocked)]);
1071
1072        let alpha = stats.reviewers.iter().find(|r| r.agent == "alpha").unwrap();
1073        assert_eq!(alpha.seated, 1);
1074        assert_eq!(alpha.rounds, 1);
1075        assert_eq!(alpha.timeouts, 0);
1076        assert_eq!(alpha.submitted, 0);
1077
1078        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
1079        assert_eq!(beta.seated, 1);
1080        assert_eq!(beta.timeouts, 1);
1081        assert_eq!(beta.submitted, 0);
1082        // A timeout must never read as a submission with nothing found: it
1083        // stays out of the scoring denominators entirely rather than becoming
1084        // a 0/0 that looks identical to a reviewer who answered and passed.
1085        assert_eq!(beta.rounds, 0);
1086        assert_eq!(beta.timeout_rate(), 100.0);
1087    }
1088
1089    #[test]
1090    fn a_timeout_is_still_recorded_when_the_round_also_lost_its_fix_report() {
1091        // Two independent gaps in one round: `beta` never answered, and the
1092        // fixer's adoption report never came back. The lost report suppresses
1093        // adoption scoring (see `a_lost_fix_report_does_not_count_as_zero_
1094        // adoption`) — it must not also swallow the fact that a seat was
1095        // silent, which is a property of the seat and not of the fixer.
1096        let round = ReviewRound {
1097            round: 1,
1098            head: "h".to_owned(),
1099            verified_head: None,
1100            verified_at: None,
1101            reviews: vec![
1102                ReviewRecord {
1103                    attempts: 0,
1104                    reviewer: 1,
1105                    agent: "alpha".to_owned(),
1106                    summary: String::new(),
1107                    findings: vec![finding(
1108                        "R1-1-1",
1109                        "src/a.rs",
1110                        10,
1111                        "panics on empty",
1112                        Severity::Blocker,
1113                    )],
1114                    vote: None,
1115                    failed: None,
1116                    duration_ms: 0,
1117                },
1118                ReviewRecord {
1119                    attempts: 0,
1120                    reviewer: 2,
1121                    agent: "beta".to_owned(),
1122                    summary: String::new(),
1123                    findings: Vec::new(),
1124                    vote: None,
1125                    failed: Some("agent timed out".to_owned()),
1126                    duration_ms: 0,
1127                },
1128            ],
1129            e2e: Vec::new(),
1130            verify_retried: false,
1131            e2e_deferred: false,
1132            e2e_defer_reason: None,
1133            fix: Some(FixRecord {
1134                agent: "alpha".to_owned(),
1135                addressed: Vec::new(),
1136                rejected: Vec::new(),
1137                notes: String::new(),
1138                committed: true,
1139                failed: Some("unparsable fix report".to_owned()),
1140                duration_ms: 0,
1141                continuation: None,
1142            }),
1143            blocking: 1,
1144            answered: 1,
1145            expected: 2,
1146            clean: false,
1147            progressed: false,
1148            vote_split: false,
1149            reconsideration: Vec::new(),
1150            verdict: None,
1151        };
1152        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Blocked)]);
1153
1154        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
1155        assert_eq!(beta.timeouts, 1);
1156        assert_eq!(beta.timeout_rate(), 100.0);
1157        // `alpha` answered, so the lost report keeps it out of the table
1158        // altogether — nothing about its findings can be scored.
1159        assert!(
1160            !stats.reviewers.iter().any(|r| r.agent == "alpha"),
1161            "{:?}",
1162            stats.reviewers
1163        );
1164    }
1165
1166    #[test]
1167    fn e2e_sole_detection_needs_a_clean_static_review() {
1168        let fail = CommandOutcome {
1169            command: "cargo test".to_owned(),
1170            code: Some(101),
1171            output_tail: "boom".to_owned(),
1172            duration_ms: 1,
1173            resource_blocked: false,
1174        };
1175        let sole = ReviewRound {
1176            round: 1,
1177            head: "h".to_owned(),
1178            verified_head: None,
1179            verified_at: None,
1180            reviews: Vec::new(),
1181            e2e: vec![fail.clone()],
1182            verify_retried: false,
1183            e2e_deferred: false,
1184            e2e_defer_reason: None,
1185            fix: None,
1186            blocking: 0,
1187            answered: 0,
1188            expected: 0,
1189            clean: false,
1190            progressed: false,
1191            vote_split: false,
1192            reconsideration: Vec::new(),
1193            verdict: None,
1194        };
1195        let alongside = ReviewRound {
1196            round: 2,
1197            head: "h".to_owned(),
1198            verified_head: None,
1199            verified_at: None,
1200            reviews: Vec::new(),
1201            e2e: vec![fail],
1202            verify_retried: false,
1203            e2e_deferred: false,
1204            e2e_defer_reason: None,
1205            fix: None,
1206            blocking: 2,
1207            answered: 0,
1208            expected: 0,
1209            clean: false,
1210            progressed: false,
1211            vote_split: false,
1212            reconsideration: Vec::new(),
1213            verdict: None,
1214        };
1215        let stats = collect(&[state_with(vec![sole, alongside], 'A', RunStatus::Ready)]);
1216        assert_eq!(stats.e2e.rounds, 2);
1217        assert_eq!(stats.e2e.failures, 2);
1218        assert_eq!(stats.e2e.sole_detections, 1);
1219        assert_eq!(stats.e2e.sole_rate(), 50.0);
1220    }
1221
1222    #[test]
1223    fn every_run_status_lands_in_exactly_one_breakdown_bucket() {
1224        let states = vec![
1225            state_with(Vec::new(), 'A', RunStatus::Merged),
1226            state_with(Vec::new(), 'A', RunStatus::Ready),
1227            state_with(Vec::new(), 'A', RunStatus::Blocked),
1228            state_with(Vec::new(), 'A', RunStatus::Failed),
1229            state_with(Vec::new(), 'A', RunStatus::Stalled),
1230            state_with(Vec::new(), 'A', RunStatus::VerifiedNoop),
1231            state_with(Vec::new(), 'A', RunStatus::Superseded),
1232            state_with(Vec::new(), 'A', RunStatus::Implementing),
1233            state_with(Vec::new(), 'A', RunStatus::Landing),
1234        ];
1235        let stats = collect(&states);
1236        let t = &stats.totals;
1237        assert_eq!(t.runs, 9);
1238        assert_eq!(t.merged, 1);
1239        assert_eq!(t.ready, 1);
1240        assert_eq!(t.blocked, 1);
1241        assert_eq!(t.failed, 1);
1242        assert_eq!(t.stalled, 1);
1243        assert_eq!(t.verified_noop, 1);
1244        assert_eq!(t.superseded, 1);
1245        // `Implementing` and `Landing` both fall into the one non-terminal
1246        // bucket.
1247        assert_eq!(t.in_progress, 2);
1248        assert_eq!(
1249            t.merged
1250                + t.ready
1251                + t.blocked
1252                + t.failed
1253                + t.stalled
1254                + t.verified_noop
1255                + t.superseded
1256                + t.in_progress,
1257            t.runs,
1258            "every run must land in exactly one bucket of the breakdown"
1259        );
1260    }
1261
1262    #[test]
1263    fn empty_input_yields_zeroed_rates_not_nan() {
1264        let stats = collect(&[]);
1265        assert_eq!(stats.totals.completion_rate(), 0.0);
1266        assert_eq!(stats.totals.split_rate(), 0.0);
1267        assert_eq!(stats.e2e.sole_rate(), 0.0);
1268        assert!(stats.agents.is_empty());
1269        assert!(stats.advisors.is_empty());
1270        assert_eq!(AdvisorStats::default().reflection_rate(), 0.0);
1271    }
1272
1273    fn advisor_record(
1274        seat: &str,
1275        agent: &str,
1276        proposal: Option<crate::verdict::Proposal>,
1277        reflection: crate::advise::Reflection,
1278    ) -> crate::advise::AdvisorRecord {
1279        crate::advise::AdvisorRecord {
1280            seat: seat.to_owned(),
1281            agent: agent.to_owned(),
1282            proposal,
1283            error: None,
1284            duration_ms: 0,
1285            reflection,
1286        }
1287    }
1288
1289    fn a_proposal() -> crate::verdict::Proposal {
1290        crate::verdict::Proposal {
1291            approach: "do the thing".to_owned(),
1292            key_tradeoff: "speed over memory".to_owned(),
1293            risks: Vec::new(),
1294            touches: Vec::new(),
1295            why_not_naive: "the naive version breaks under load".to_owned(),
1296        }
1297    }
1298
1299    #[test]
1300    fn advisor_stats_count_proposed_absent_and_reflection_split() {
1301        use crate::advise::{Advice, Reflection};
1302
1303        let mut s = state_with(Vec::new(), 'A', RunStatus::Merged);
1304        s.advice = Some(Advice {
1305            records: vec![
1306                advisor_record("advisor-1", "alpha", Some(a_proposal()), Reflection::Strong),
1307                advisor_record("advisor-2", "alpha", Some(a_proposal()), Reflection::Faint),
1308                advisor_record("advisor-3", "alpha", None, Reflection::Absent),
1309            ],
1310            synthesis: Some("blended brief".to_owned()),
1311        });
1312
1313        let stats = collect(&[s]);
1314        let alpha = stats.advisors.iter().find(|a| a.agent == "alpha").unwrap();
1315        assert_eq!(alpha.seated, 3);
1316        assert_eq!(alpha.proposed, 2);
1317        assert_eq!(alpha.absent, 1);
1318        assert_eq!(alpha.strong, 1);
1319        assert_eq!(alpha.faint, 1);
1320        assert_eq!(alpha.reflection_rate(), 50.0);
1321    }
1322
1323    #[test]
1324    fn advisor_stats_count_absent_from_the_proposal_not_the_reflection_default() {
1325        // A record whose `proposal` is `None` but whose `reflection` was
1326        // never classified (predates `apply_reflection`, or the field's own
1327        // serde default) must still count as `absent` — and a run whose
1328        // synthesis never ran leaves every *proposed* record `Faint` by the
1329        // same default, which must not spill into `absent` either.
1330        //
1331        // A third case: a proposed record whose `reflection` was *never*
1332        // classified at all (a run.json predating the `reflection` field)
1333        // must not vanish from the breakdown either — it has to land
1334        // somewhere in faint/strong, not be silently dropped from all three
1335        // counters while still counting toward `proposed`.
1336        use crate::advise::{Advice, Reflection};
1337
1338        let mut s = state_with(Vec::new(), 'A', RunStatus::Merged);
1339        s.advice = Some(Advice {
1340            records: vec![
1341                advisor_record("advisor-1", "alpha", None, Reflection::Absent),
1342                advisor_record("advisor-2", "alpha", Some(a_proposal()), Reflection::Faint),
1343                advisor_record("advisor-3", "alpha", Some(a_proposal()), Reflection::Absent),
1344            ],
1345            synthesis: None,
1346        });
1347
1348        let stats = collect(&[s]);
1349        let alpha = stats.advisors.iter().find(|a| a.agent == "alpha").unwrap();
1350        assert_eq!(alpha.seated, 3);
1351        assert_eq!(alpha.proposed, 2);
1352        assert_eq!(alpha.absent, 1);
1353        assert_eq!(alpha.faint, 2);
1354        assert_eq!(alpha.strong, 0);
1355    }
1356
1357    #[test]
1358    fn advisor_stats_ignore_runs_with_advise_off() {
1359        let s = state_with(Vec::new(), 'A', RunStatus::Merged);
1360        assert!(s.advice.is_none());
1361        let stats = collect(&[s]);
1362        assert!(stats.advisors.is_empty());
1363    }
1364
1365    #[test]
1366    fn same_defect_matches_titles_across_files() {
1367        let a = finding("1", "src/a.rs", 1, "Panics On Empty!", Severity::Major);
1368        let b = finding("2", "src/z.rs", 900, "panics on empty", Severity::Nit);
1369        assert!(same_defect(&a, &b));
1370        let c = finding("3", "src/z.rs", 900, "totally different", Severity::Nit);
1371        assert!(!same_defect(&a, &c));
1372    }
1373
1374    #[test]
1375    fn release_bump_stats_split_clean_from_attention_and_track_coverage() {
1376        use crate::run::ReleaseBump;
1377
1378        // Not recorded at all: uses the release-bump step? unknown.
1379        let unrecorded = state_with(Vec::new(), 'A', RunStatus::Merged);
1380
1381        // Clean: automerge worked, nobody had to look at it.
1382        let mut automerged = state_with(Vec::new(), 'A', RunStatus::Merged);
1383        automerged.release_bump = Some(ReleaseBump {
1384            pr_url: Some("https://github.com/o/r/pull/1".to_owned()),
1385            version: Some("1.2.3".to_owned()),
1386            automerge_enabled: true,
1387            merged_directly: false,
1388            problem: None,
1389            action_required: None,
1390        });
1391
1392        // Also clean, but automerge itself was rejected by GitHub and magi
1393        // merged the already-green PR directly - `automerge_enabled` reads
1394        // false here, and that must not make this count as needing a human.
1395        let mut merged_directly = state_with(Vec::new(), 'A', RunStatus::Merged);
1396        merged_directly.release_bump = Some(ReleaseBump {
1397            pr_url: Some("https://github.com/o/r/pull/2".to_owned()),
1398            version: Some("1.2.4".to_owned()),
1399            automerge_enabled: false,
1400            merged_directly: true,
1401            problem: None,
1402            action_required: None,
1403        });
1404
1405        // Needs a human, PR opened.
1406        let mut blocked_with_pr = state_with(Vec::new(), 'A', RunStatus::Merged);
1407        blocked_with_pr.release_bump = Some(ReleaseBump {
1408            pr_url: Some("https://github.com/o/r/pull/3".to_owned()),
1409            version: Some("1.2.5".to_owned()),
1410            automerge_enabled: false,
1411            merged_directly: false,
1412            problem: Some("checks red".to_owned()),
1413            action_required: Some("look at the PR".to_owned()),
1414        });
1415
1416        // Needs a human, no PR ever opened.
1417        let mut blocked_without_pr = state_with(Vec::new(), 'A', RunStatus::Merged);
1418        blocked_without_pr.release_bump = Some(ReleaseBump {
1419            pr_url: None,
1420            version: Some("1.2.6".to_owned()),
1421            automerge_enabled: false,
1422            merged_directly: false,
1423            problem: Some("gh pr create failed".to_owned()),
1424            action_required: Some("open the PR by hand".to_owned()),
1425        });
1426
1427        let stats = collect(&[
1428            unrecorded,
1429            automerged,
1430            merged_directly,
1431            blocked_with_pr,
1432            blocked_without_pr,
1433        ]);
1434        let b = &stats.release_bumps;
1435        assert_eq!(b.merged, 5);
1436        assert_eq!(b.recorded, 4);
1437        assert_eq!(b.pr_opened, 3);
1438        assert_eq!(b.automerge_enabled, 1);
1439        assert_eq!(b.merged_directly, 1);
1440        assert_eq!(b.needs_attention, 2);
1441        assert_eq!(b.clean(), 2);
1442        // Clean and attention must always split `recorded` exactly.
1443        assert_eq!(b.clean() + b.needs_attention, b.recorded);
1444        assert_eq!(b.coverage_rate(), 80.0);
1445        assert!((b.automerge_rate() - 33.333_333_333_333_336).abs() < 1e-9);
1446        assert_eq!(b.attention_rate(), 50.0);
1447    }
1448
1449    #[test]
1450    fn release_bump_ignores_runs_that_are_not_merged() {
1451        use crate::run::ReleaseBump;
1452
1453        let mut blocked = state_with(Vec::new(), 'A', RunStatus::Blocked);
1454        blocked.release_bump = Some(ReleaseBump {
1455            pr_url: Some("https://github.com/o/r/pull/9".to_owned()),
1456            version: Some("9.9.9".to_owned()),
1457            automerge_enabled: true,
1458            merged_directly: false,
1459            problem: None,
1460            action_required: None,
1461        });
1462
1463        let stats = collect(&[blocked]);
1464        let b = &stats.release_bumps;
1465        assert_eq!(b.merged, 0);
1466        assert_eq!(b.recorded, 0);
1467        assert_eq!(b.pr_opened, 0);
1468    }
1469
1470    #[test]
1471    fn release_bump_stats_are_zero_on_merged_runs_with_no_bump_or_no_runs() {
1472        let stats = collect(&[state_with(Vec::new(), 'A', RunStatus::Merged)]);
1473        let b = &stats.release_bumps;
1474        assert_eq!(b.merged, 1);
1475        assert_eq!(b.recorded, 0);
1476        assert_eq!(b.coverage_rate(), 0.0);
1477        assert_eq!(b.automerge_rate(), 0.0);
1478        assert_eq!(b.attention_rate(), 0.0);
1479        assert_eq!(b.clean(), 0);
1480
1481        let empty = collect(&[]);
1482        let b = &empty.release_bumps;
1483        assert_eq!(b.merged, 0);
1484        assert_eq!(b.coverage_rate(), 0.0);
1485        assert_eq!(b.automerge_rate(), 0.0);
1486        assert_eq!(b.attention_rate(), 0.0);
1487    }
1488
1489    fn event(node: &str, at_secs: i64, message: &str) -> crate::run::Event {
1490        crate::run::Event {
1491            at: jiff::Timestamp::from_second(at_secs).unwrap(),
1492            node: node.to_owned(),
1493            message: message.to_owned(),
1494        }
1495    }
1496
1497    fn state_with_events(events: Vec<crate::run::Event>) -> RunState {
1498        let mut s = RunState::new(
1499            PathBuf::from("/repo"),
1500            "main".to_owned(),
1501            "abcdef".to_owned(),
1502            "task".to_owned(),
1503            Config::default(),
1504        );
1505        s.events = events;
1506        s
1507    }
1508
1509    #[test]
1510    fn a_node_with_multiple_events_spans_first_to_last() {
1511        let s = state_with_events(vec![
1512            event("implement", 1_000, "start"),
1513            event("implement", 1_030, "still running"),
1514            event("implement", 1_090, "done"),
1515        ]);
1516        let nodes = node_durations(&[s]);
1517        assert_eq!(nodes.len(), 1);
1518        assert_eq!(nodes[0].node, "implement");
1519        assert_eq!(nodes[0].runs, 1);
1520        assert_eq!(nodes[0].total_secs, 90);
1521        assert_eq!(nodes[0].max_secs, 90);
1522        assert_eq!(nodes[0].single, 0);
1523        assert_eq!(nodes[0].mean_secs(), 90.0);
1524    }
1525
1526    #[test]
1527    fn a_node_with_a_single_event_is_unmeasured_not_zero() {
1528        let s = state_with_events(vec![event("gate", 2_000, "ran once")]);
1529        let nodes = node_durations(&[s]);
1530        assert_eq!(nodes.len(), 1);
1531        assert_eq!(nodes[0].node, "gate");
1532        assert_eq!(nodes[0].runs, 0);
1533        assert_eq!(nodes[0].total_secs, 0);
1534        assert_eq!(nodes[0].single, 1);
1535        // No measured runs: the mean must read as zero, never NaN or a
1536        // fabricated span from the lone event.
1537        assert_eq!(nodes[0].mean_secs(), 0.0);
1538    }
1539
1540    #[test]
1541    fn a_run_with_no_events_produces_no_node_rows() {
1542        let s = state_with_events(vec![]);
1543        let nodes = node_durations(&[s]);
1544        assert!(nodes.is_empty());
1545    }
1546
1547    #[test]
1548    fn multiple_runs_aggregate_the_same_node() {
1549        let a = state_with_events(vec![event("judge", 0, "start"), event("judge", 60, "done")]);
1550        let b = state_with_events(vec![
1551            event("judge", 0, "start"),
1552            event("judge", 200, "done"),
1553        ]);
1554        // A single-event run for the same node must add to `single` without
1555        // disturbing the measured runs' total or max.
1556        let c = state_with_events(vec![event("judge", 5, "start")]);
1557        let nodes = node_durations(&[a, b, c]);
1558        assert_eq!(nodes.len(), 1);
1559        let judge = &nodes[0];
1560        assert_eq!(judge.node, "judge");
1561        assert_eq!(judge.runs, 2);
1562        assert_eq!(judge.total_secs, 260);
1563        assert_eq!(judge.max_secs, 200);
1564        assert_eq!(judge.single, 1);
1565        assert_eq!(judge.mean_secs(), 130.0);
1566    }
1567
1568    #[test]
1569    fn by_repo_splits_states_and_group_totals_sum_to_the_whole() {
1570        let states = vec![
1571            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1572            state_with_repo(Vec::new(), 'A', RunStatus::Blocked, "/repos/a"),
1573            state_with_repo(Vec::new(), 'B', RunStatus::Merged, "/repos/b"),
1574        ];
1575        let groups = by_repo(&states);
1576        assert_eq!(groups.len(), 2);
1577
1578        let total_runs: usize = groups.iter().map(|g| g.stats.totals.runs).sum();
1579        assert_eq!(total_runs, collect(&states).totals.runs);
1580        let total_merged: usize = groups.iter().map(|g| g.stats.totals.merged).sum();
1581        assert_eq!(total_merged, collect(&states).totals.merged);
1582
1583        // Busiest repository (2 runs) sorts first.
1584        assert_eq!(groups[0].repo, PathBuf::from("/repos/a"));
1585        assert_eq!(groups[0].name, "a");
1586        assert_eq!(groups[0].stats.totals.runs, 2);
1587        assert_eq!(groups[1].repo, PathBuf::from("/repos/b"));
1588        assert_eq!(groups[1].name, "b");
1589        assert_eq!(groups[1].stats.totals.runs, 1);
1590    }
1591
1592    #[test]
1593    fn by_repo_breaks_a_run_count_tie_by_path() {
1594        let states = vec![
1595            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/z"),
1596            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1597        ];
1598        let groups = by_repo(&states);
1599        assert_eq!(groups.len(), 2);
1600        assert_eq!(groups[0].repo, PathBuf::from("/repos/a"));
1601        assert_eq!(groups[1].repo, PathBuf::from("/repos/z"));
1602    }
1603
1604    #[test]
1605    fn by_repo_on_empty_input_yields_no_groups() {
1606        assert!(by_repo(&[]).is_empty());
1607    }
1608
1609    #[test]
1610    fn filter_repo_matches_the_full_path_exactly() {
1611        let states = vec![
1612            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1613            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/ab"),
1614        ];
1615        let hits = filter_repo(&states, Path::new("/repos/a"));
1616        assert_eq!(hits.len(), 1);
1617        assert_eq!(hits[0].repo, PathBuf::from("/repos/a"));
1618    }
1619
1620    #[test]
1621    fn filter_repo_returns_nothing_for_an_unknown_repo_or_empty_input() {
1622        let states = vec![state_with_repo(
1623            Vec::new(),
1624            'A',
1625            RunStatus::Merged,
1626            "/repos/a",
1627        )];
1628        assert!(filter_repo(&states, Path::new("/repos/nope")).is_empty());
1629        assert!(filter_repo(&[], Path::new("/repos/a")).is_empty());
1630    }
1631}