Skip to main content

magi/
stats.rs

1//! Aggregate statistics over every recorded run.
2//!
3//! These tables are a by-product of running the graph, not a benchmark. The
4//! seat assignment rotates, the task distribution is whatever the operator
5//! happened to ask for, and a model that draws harder tasks looks worse. Read
6//! them as "relative performance on my workload", which is the only claim the
7//! data supports.
8use std::collections::BTreeMap;
9use std::path::{Path, PathBuf};
10
11use crate::run::{RunState, RunStatus, list_ids};
12
13/// Implementation record for one agent.
14#[derive(Debug, Clone, Default)]
15pub struct AgentStats {
16    /// Agent id.
17    pub agent: String,
18    /// Candidates it produced that were judged.
19    pub entered: usize,
20    /// Competitions it won.
21    pub wins: usize,
22    /// Candidates that produced no change at all.
23    pub empty: usize,
24}
25
26impl AgentStats {
27    /// Win rate over entries, as a percentage.
28    pub fn win_rate(&self) -> f64 {
29        if self.entered == 0 {
30            0.0
31        } else {
32            100.0 * self.wins as f64 / self.entered as f64
33        }
34    }
35}
36
37/// Review record for one agent.
38#[derive(Debug, Clone, Default)]
39pub struct ReviewerStats {
40    /// Agent id.
41    pub agent: String,
42    /// Review rounds it sat in whose adoption could be scored — a round whose
43    /// fixer never reported back is excluded, so this is the denominator of
44    /// [`Self::adopted_per_round`], not a headcount of appearances. For that,
45    /// see [`Self::seated`].
46    pub rounds: usize,
47    /// Review rounds it was on the panel for at all, scoreable or not.
48    /// Whether a seat answered is a fact about the seat and does not depend
49    /// on what later became of the fixer's report, so this — not `rounds` —
50    /// is the honest denominator for [`Self::timeout_rate`].
51    pub seated: usize,
52    /// Findings it submitted.
53    pub submitted: usize,
54    /// Findings the fixer acted on.
55    pub adopted: usize,
56    /// Findings no other reviewer in the same round also raised.
57    pub unique: usize,
58    /// Rounds it was seated in but never answered (timeout, crash, unparsable
59    /// output) — kept apart from `submitted`/`adopted` so a silent seat
60    /// cannot read as a seat with nothing to say.
61    pub timeouts: usize,
62}
63
64impl ReviewerStats {
65    /// Adopted findings per round: how much signal one seat produces.
66    pub fn adopted_per_round(&self) -> f64 {
67        if self.rounds == 0 {
68            0.0
69        } else {
70            self.adopted as f64 / self.rounds as f64
71        }
72    }
73
74    /// Adopted over submitted: how often its findings are real. Rounds where
75    /// the seat never answered are not in `submitted`, so a timeout cannot
76    /// dilute (or hide behind) this rate.
77    pub fn precision(&self) -> f64 {
78        if self.submitted == 0 {
79            0.0
80        } else {
81            100.0 * self.adopted as f64 / self.submitted as f64
82        }
83    }
84
85    /// Share of its findings that only it saw.
86    pub fn unique_rate(&self) -> f64 {
87        if self.submitted == 0 {
88            0.0
89        } else {
90            100.0 * self.unique as f64 / self.submitted as f64
91        }
92    }
93
94    /// Share of the rounds it was seated in where it never answered.
95    pub fn timeout_rate(&self) -> f64 {
96        if self.seated == 0 {
97            0.0
98        } else {
99            100.0 * self.timeouts as f64 / self.seated as f64
100        }
101    }
102}
103
104/// Design-deliberation record for one agent.
105///
106/// Approximate by construction, same as [`crate::advise::Reflection`] itself:
107/// `strong`/`faint` come from a word-overlap heuristic against the synthesis
108/// brief, not from an explicit attribution, so [`Self::reflection_rate`] is a
109/// rough read on whose ideas seemed to land, not a precise credit split.
110#[derive(Debug, Clone, Default)]
111pub struct AdvisorStats {
112    /// Agent id.
113    pub agent: String,
114    /// Advisor seats it occupied, across every run with `[graph] advise` on.
115    pub seated: usize,
116    /// Seats where it produced a usable proposal (`record.proposal.is_some()`).
117    pub proposed: usize,
118    /// Seats where it produced no usable proposal at all (crashed, timed
119    /// out, or answered with nothing a proposal could be parsed out of).
120    /// Counted from `record.proposal.is_none()` directly rather than from
121    /// `record.reflection == Absent` — `reflection` is `#[serde(default)]`
122    /// and can be left at its default on a record from before
123    /// `apply_reflection` ran, which would otherwise double as a false
124    /// "absent".
125    pub absent: usize,
126    /// Proposals the synthesis brief carried little or no recognisable
127    /// trace of.
128    pub faint: usize,
129    /// Proposals the synthesis brief named outright or carried enough of to
130    /// count as a clear match.
131    pub strong: usize,
132}
133
134impl AdvisorStats {
135    /// Share of its proposals rated `Strong`, as a percentage. The
136    /// denominator is `proposed`, not `seated` — a seat that never produced
137    /// a proposal had nothing for the synthesis to reflect, so it cannot
138    /// count against this rate any more than a reviewer's silence counts
139    /// against [`ReviewerStats::precision`].
140    ///
141    /// Same caveat as the struct itself: this is a heuristic read on
142    /// reflection, not a precise attribution.
143    pub fn reflection_rate(&self) -> f64 {
144        if self.proposed == 0 {
145            0.0
146        } else {
147            100.0 * self.strong as f64 / self.proposed as f64
148        }
149    }
150}
151
152/// What real-machine verification caught that static review did not.
153#[derive(Debug, Clone, Default)]
154pub struct E2eStats {
155    /// Rounds where E2E commands ran.
156    pub rounds: usize,
157    /// Rounds where E2E failed.
158    pub failures: usize,
159    /// Rounds where E2E failed and no reviewer had raised a blocking finding —
160    /// a runtime defect that only execution found.
161    pub sole_detections: usize,
162    /// Rounds where E2E was deferred rather than run: blocking findings
163    /// already required a fix, so the round went straight to the fixer
164    /// instead of spending a full verify run on a head about to change. Kept
165    /// separate from [`Self::rounds`] on purpose — a deferred round never
166    /// ran anything, so counting it there would misreport how often E2E
167    /// actually executed.
168    pub deferred: usize,
169}
170
171impl E2eStats {
172    /// Share of E2E failures that static review had missed entirely.
173    pub fn sole_rate(&self) -> f64 {
174        if self.failures == 0 {
175            0.0
176        } else {
177            100.0 * self.sole_detections as f64 / self.failures as f64
178        }
179    }
180}
181
182/// Run-level counters.
183#[derive(Debug, Clone, Default)]
184pub struct Totals {
185    /// Runs on disk.
186    pub runs: usize,
187    /// Reached a merge.
188    pub merged: usize,
189    /// Passed the gate, merge not requested.
190    pub ready: usize,
191    /// Stopped with findings open or a red gate.
192    pub blocked: usize,
193    /// Could not complete.
194    pub failed: usize,
195    /// The judging panel never reached a quorum; the work is kept but the
196    /// verdict is not trustworthy. Never counted as `blocked` — see
197    /// `RunStatus::Stalled`'s own doc — so it needs its own counter to stay
198    /// visible in a breakdown rather than vanishing from every bucket.
199    pub stalled: usize,
200    /// Every candidate wrote nothing, and said why in a way that survived
201    /// the adoption guard: a claim, not a failure, and not yet verified.
202    pub verified_noop: usize,
203    /// A later attempt at the same task already finished it; this run's own
204    /// `blocked`/`stalled` no longer needs anyone's attention.
205    pub superseded: usize,
206    /// Still moving: any non-terminal status (prep through landing). Kept as
207    /// one bucket rather than one counter per node — the per-node position of
208    /// a live run belongs to `magi show`/the deck, not to a workload-wide
209    /// tally that is read well after the run in question has finished.
210    pub in_progress: usize,
211    /// Runs that reached a tally.
212    pub tallied: usize,
213    /// Tallies where the judges' first choices disagreed.
214    pub split: usize,
215    /// Tallies that went through deliberation.
216    pub deliberated: usize,
217    /// Deliberated runs where at least one judge moved.
218    pub minds_changed: usize,
219    /// Deliberated runs that ended unanimous.
220    pub converged: usize,
221    /// Review rounds across all runs.
222    pub review_rounds: usize,
223}
224
225impl Totals {
226    /// Merged or ready over all runs.
227    pub fn completion_rate(&self) -> f64 {
228        if self.runs == 0 {
229            0.0
230        } else {
231            100.0 * (self.merged + self.ready) as f64 / self.runs as f64
232        }
233    }
234
235    /// Share of tallies that were split.
236    pub fn split_rate(&self) -> f64 {
237        if self.tallied == 0 {
238            0.0
239        } else {
240            100.0 * self.split as f64 / self.tallied as f64
241        }
242    }
243}
244
245/// Per-node duration breakdown, aggregated across every loaded run.
246///
247/// A duration here is the span between a node's *first* and *last* recorded
248/// event within one run — not time actually spent working. A node visited
249/// more than twice in one run (a retry, or a park/resume gap) has any idle
250/// time in between folded into that span, so this reads as an upper bound on
251/// the node's wall-clock cost, not a measurement of it.
252#[derive(Debug, Clone, Default, PartialEq)]
253pub struct NodeDuration {
254    /// Node name, as it appears in `Event::node`.
255    pub node: String,
256    /// Runs where the node's span could be measured (at least two events).
257    pub runs: usize,
258    /// Sum of every measured run's span, in seconds.
259    pub total_secs: i64,
260    /// The longest single run's span, in seconds.
261    pub max_secs: i64,
262    /// Runs where the node had exactly one event — visited, but with no
263    /// second timestamp to measure a span against. Counted apart from `runs`
264    /// so a caller never has to guess whether a low mean hides unmeasured
265    /// visits.
266    pub single: usize,
267}
268
269impl NodeDuration {
270    /// Mean span across the runs that could be measured, in seconds.
271    ///
272    /// `0.0` when `runs` is zero, never a division by zero.
273    pub fn mean_secs(&self) -> f64 {
274        if self.runs == 0 {
275            0.0
276        } else {
277            self.total_secs as f64 / self.runs as f64
278        }
279    }
280}
281
282/// Derive a per-node duration breakdown from every loaded run's events.
283///
284/// For each run, a node's span is the gap between its earliest and latest
285/// [`crate::run::Event`] in that run. A node with fewer than two events in a
286/// run contributes no span (see [`NodeDuration::single`] for the one-event
287/// case); a run with no events for a node contributes nothing at all. Spans
288/// from every run are then summed per node.
289pub fn node_durations<'a>(states: impl IntoIterator<Item = &'a RunState>) -> Vec<NodeDuration> {
290    let mut nodes: BTreeMap<String, NodeDuration> = BTreeMap::new();
291
292    for state in states {
293        let mut spans: BTreeMap<&str, (jiff::Timestamp, jiff::Timestamp, usize)> = BTreeMap::new();
294        for e in &state.events {
295            spans
296                .entry(e.node.as_str())
297                .and_modify(|(min, max, count)| {
298                    if e.at < *min {
299                        *min = e.at;
300                    }
301                    if e.at > *max {
302                        *max = e.at;
303                    }
304                    *count += 1;
305                })
306                .or_insert((e.at, e.at, 1));
307        }
308
309        for (node, (min, max, count)) in spans {
310            let entry = nodes
311                .entry(node.to_owned())
312                .or_insert_with(|| NodeDuration {
313                    node: node.to_owned(),
314                    ..NodeDuration::default()
315                });
316            if count < 2 {
317                entry.single += 1;
318                continue;
319            }
320            // `max - min` is never negative: both are the extremes of the
321            // same event set, and equal timestamps yield a zero-second span
322            // rather than being mistaken for an unmeasured visit.
323            let span_secs = (max - min).get_seconds();
324            entry.runs += 1;
325            entry.total_secs += span_secs;
326            if span_secs > entry.max_secs {
327                entry.max_secs = span_secs;
328            }
329        }
330    }
331
332    let mut nodes: Vec<NodeDuration> = nodes.into_values().collect();
333    nodes.sort_by(|a, b| b.total_secs.cmp(&a.total_secs).then(a.node.cmp(&b.node)));
334    nodes
335}
336
337/// What the post-merge release-bump step did, across every merged run.
338///
339/// `merged` is the denominator for [`Self::coverage_rate`]: a repository
340/// that never uses the release-bump step (no `auto-tag.yml`, see
341/// `kata:agents:rust:*`) should read as "0 of N merged runs recorded a
342/// bump", not vanish from the report the way it would if `recorded` were the
343/// denominator instead — silently excluding those runs would hide the
344/// coverage gap itself.
345#[derive(Debug, Clone, Default)]
346pub struct ReleaseBumpStats {
347    /// Merged runs, the denominator for [`Self::coverage_rate`].
348    pub merged: usize,
349    /// Merged runs that carry a [`crate::run::ReleaseBump`] record at all.
350    pub recorded: usize,
351    /// Recorded bumps that opened a release pull request.
352    pub pr_opened: usize,
353    /// Recorded bumps whose `automerge_enabled` is `true` at the point the
354    /// bump step finished. This is the *final* state, not "was automerge
355    /// enabled at some point" — `bump::surface_problem` flips it back to
356    /// `false` when GitHub later rejects automerge, so a bump that briefly
357    /// enabled it and then had it rejected counts here as not enabled.
358    pub automerge_enabled: usize,
359    /// Recorded bumps magi merged directly because GitHub refused automerge
360    /// on an already-green pull request. Counted apart from
361    /// `automerge_enabled` on purpose: a `merged_directly` bump needed no
362    /// human even though automerge itself never took, so folding it into
363    /// (or leaving it out of) the automerge count would misstate either
364    /// number.
365    pub merged_directly: usize,
366    /// Recorded bumps that ended with [`crate::run::RunState::needs_attention`]
367    /// true — a human has something to do.
368    pub needs_attention: usize,
369}
370
371impl ReleaseBumpStats {
372    /// Share of merged runs that recorded a release bump at all.
373    pub fn coverage_rate(&self) -> f64 {
374        if self.merged == 0 {
375            0.0
376        } else {
377            100.0 * self.recorded as f64 / self.merged as f64
378        }
379    }
380
381    /// Share of opened release PRs that ended with automerge enabled.
382    pub fn automerge_rate(&self) -> f64 {
383        if self.pr_opened == 0 {
384            0.0
385        } else {
386            100.0 * self.automerge_enabled as f64 / self.pr_opened as f64
387        }
388    }
389
390    /// Share of recorded bumps that needed a human.
391    pub fn attention_rate(&self) -> f64 {
392        if self.recorded == 0 {
393            0.0
394        } else {
395            100.0 * self.needs_attention as f64 / self.recorded as f64
396        }
397    }
398
399    /// Recorded bumps that finished with nothing for a human to do.
400    ///
401    /// Deliberately the difference `recorded - needs_attention`, not a
402    /// separate counter kept in step with `automerge_enabled`: a bump that
403    /// GitHub refused automerge on but that magi merged directly
404    /// (`merged_directly`) is clean — nobody had to act — even though
405    /// `automerge_enabled` is `false` for it. Counting clean bumps as
406    /// `automerge_enabled` alone would leave `merged_directly` cases in
407    /// neither the clean nor the attention bucket, and the two would stop
408    /// summing to `recorded`.
409    pub fn clean(&self) -> usize {
410        self.recorded.saturating_sub(self.needs_attention)
411    }
412}
413
414/// Everything, aggregated.
415#[derive(Debug, Clone, Default)]
416pub struct Stats {
417    /// Run counters.
418    pub totals: Totals,
419    /// Per-agent implementation record, best win rate first.
420    pub agents: Vec<AgentStats>,
421    /// Per-agent review record, most adopted-per-round first.
422    pub reviewers: Vec<ReviewerStats>,
423    /// Per-agent design-deliberation record, highest reflection rate first.
424    /// Only runs where `[graph] advise` produced an [`crate::advise::Advice`]
425    /// contribute — a run with the stage off carries no signal either way,
426    /// and counting it would water down every agent's rate with seats that
427    /// were never asked.
428    pub advisors: Vec<AdvisorStats>,
429    /// Verification record.
430    pub e2e: E2eStats,
431    /// Per-node duration breakdown, longest total first.
432    pub nodes: Vec<NodeDuration>,
433    /// Post-merge release-bump record, over every merged run.
434    pub release_bumps: ReleaseBumpStats,
435}
436
437/// Load every run on disk, skipping any that cannot be read.
438pub fn load_all() -> Vec<RunState> {
439    list_ids()
440        .into_iter()
441        .filter_map(|id| RunState::load(&id).ok())
442        .collect()
443}
444
445/// One repository's [`Stats`], as grouped by [`by_repo`].
446#[derive(Debug, Clone)]
447pub struct RepoStats {
448    /// The grouping key: `RunState.repo` exactly as recorded, a normalised
449    /// full path. Never a display name — this is what a caller (the CLI's
450    /// `--repo` fallback, the web `?repo=` query) matches back against, and
451    /// matching by name would conflate two different checkouts that happen
452    /// to share a leaf directory.
453    pub repo: PathBuf,
454    /// Display name: `repo`'s file name, or the full path when it has none
455    /// (e.g. `/`). Collisions between repositories are the caller's problem
456    /// to disambiguate (see `report::repo_summary`), not this struct's.
457    pub name: String,
458    /// This repository's own aggregate, counted exactly as [`collect`]
459    /// counts the whole workload.
460    pub stats: Stats,
461}
462
463/// Group `states` by [`RunState::repo`] and aggregate each group with
464/// [`collect_refs`] — the same counting logic as [`collect`], just scoped to
465/// one repository at a time.
466///
467/// Sorted by run count descending, then by repo path ascending on ties, so
468/// the busiest repository leads the summary table.
469pub fn by_repo(states: &[RunState]) -> Vec<RepoStats> {
470    let mut groups: BTreeMap<PathBuf, Vec<&RunState>> = BTreeMap::new();
471    for state in states {
472        groups.entry(state.repo.clone()).or_default().push(state);
473    }
474
475    let mut out: Vec<RepoStats> = groups
476        .into_iter()
477        .map(|(repo, group)| {
478            let stats = collect_refs(group);
479            let name = repo
480                .file_name()
481                .map(|n| n.to_string_lossy().into_owned())
482                .unwrap_or_else(|| repo.to_string_lossy().into_owned());
483            RepoStats { repo, name, stats }
484        })
485        .collect();
486    out.sort_by(|a, b| {
487        b.stats
488            .totals
489            .runs
490            .cmp(&a.stats.totals.runs)
491            .then(a.repo.cmp(&b.repo))
492    });
493    out
494}
495
496/// Narrow `states` to those recorded against `repo` exactly.
497///
498/// Matches [`RunState::repo`] by full-path equality only, never by name — the
499/// name-fallback resolution a caller may want (a checkout that has since
500/// moved or been deleted) belongs one layer up, where a config or filesystem
501/// lookup can decide what "the same repository" means; this function has no
502/// such context and would otherwise risk conflating two different checkouts
503/// that happen to share a leaf directory.
504pub fn filter_repo<'a>(states: &'a [RunState], repo: &Path) -> Vec<&'a RunState> {
505    states.iter().filter(|s| s.repo == repo).collect()
506}
507
508/// Aggregate `states`.
509pub fn collect(states: &[RunState]) -> Stats {
510    collect_refs(states)
511}
512
513/// Aggregate `states`, over any iterator of references rather than a slice —
514/// what [`by_repo`] uses to run the same counting logic over each repository's
515/// own group without collecting it into an owned `Vec<RunState>` first.
516/// [`collect`] is a thin wrapper around this for the common whole-slice case.
517pub fn collect_refs<'a>(states: impl IntoIterator<Item = &'a RunState>) -> Stats {
518    let states: Vec<&'a RunState> = states.into_iter().collect();
519    let mut totals = Totals::default();
520    let mut agents: BTreeMap<String, AgentStats> = BTreeMap::new();
521    let mut reviewers: BTreeMap<String, ReviewerStats> = BTreeMap::new();
522    let mut advisors: BTreeMap<String, AdvisorStats> = BTreeMap::new();
523    let mut e2e = E2eStats::default();
524    let mut release_bumps = ReleaseBumpStats::default();
525
526    for state in &states {
527        totals.runs += 1;
528        if state.status == RunStatus::Merged {
529            release_bumps.merged += 1;
530            if let Some(b) = &state.release_bump {
531                release_bumps.recorded += 1;
532                if b.pr_url.is_some() {
533                    release_bumps.pr_opened += 1;
534                }
535                if b.automerge_enabled {
536                    release_bumps.automerge_enabled += 1;
537                }
538                if b.merged_directly {
539                    release_bumps.merged_directly += 1;
540                }
541            }
542            if state.needs_attention() {
543                release_bumps.needs_attention += 1;
544            }
545        }
546        match state.status {
547            RunStatus::Merged => totals.merged += 1,
548            RunStatus::Ready => totals.ready += 1,
549            RunStatus::Blocked => totals.blocked += 1,
550            RunStatus::Failed => totals.failed += 1,
551            RunStatus::Stalled => totals.stalled += 1,
552            RunStatus::VerifiedNoop => totals.verified_noop += 1,
553            // Counted on its own rather than folded into `blocked`/`stalled`:
554            // the task it belongs to already landed through a later run,
555            // which is the one this tally counts as the merge/ready outcome,
556            // and folding it back in would inflate the denominator with two
557            // outcomes for one task.
558            RunStatus::Superseded => totals.superseded += 1,
559            RunStatus::Prep
560            | RunStatus::Implementing
561            | RunStatus::Judging
562            | RunStatus::Deliberating
563            | RunStatus::Voting
564            | RunStatus::Reviewing
565            | RunStatus::Gating
566            | RunStatus::Landing => totals.in_progress += 1,
567        }
568
569        for c in &state.candidates {
570            let entry = agents.entry(c.agent.clone()).or_insert_with(|| AgentStats {
571                agent: c.agent.clone(),
572                ..AgentStats::default()
573            });
574            // A verified no-op is not counted as the ordinary empty loss it
575            // would otherwise look like: the candidate gave evidence for
576            // writing nothing, which `entry.empty` exists to flag the
577            // *absence* of.
578            if c.empty && c.verified_noop.is_none() {
579                entry.empty += 1;
580            }
581            if c.viable() {
582                entry.entered += 1;
583            }
584        }
585
586        if let Some(t) = &state.tally {
587            // A tally with no panel (`uncontested`) never split, never
588            // deliberated and never converged — it never happened, and
589            // folding it into the denominator would understate the real
590            // split rate with runs that carry no panel-agreement signal at
591            // all. The winner still earns its agent a win either way: an
592            // uncontested candidate is still the one that shipped.
593            if t.uncontested.is_none() {
594                totals.tallied += 1;
595                if !t.unanimous_initial {
596                    totals.split += 1;
597                }
598                if t.deliberated {
599                    totals.deliberated += 1;
600                    if t.changed_votes > 0 {
601                        totals.minds_changed += 1;
602                    }
603                    if t.unanimous_final {
604                        totals.converged += 1;
605                    }
606                }
607            }
608            if let Some(w) = state.candidates.iter().find(|c| c.label == t.winner) {
609                agents
610                    .entry(w.agent.clone())
611                    .or_insert_with(|| AgentStats {
612                        agent: w.agent.clone(),
613                        ..AgentStats::default()
614                    })
615                    .wins += 1;
616            }
617        }
618
619        if let Some(advice) = &state.advice {
620            for rec in &advice.records {
621                let entry = advisors
622                    .entry(rec.agent.clone())
623                    .or_insert_with(|| AdvisorStats {
624                        agent: rec.agent.clone(),
625                        ..AdvisorStats::default()
626                    });
627                entry.seated += 1;
628                if rec.proposal.is_none() {
629                    entry.absent += 1;
630                    continue;
631                }
632                entry.proposed += 1;
633                match rec.reflection {
634                    crate::advise::Reflection::Strong => entry.strong += 1,
635                    crate::advise::Reflection::Faint => entry.faint += 1,
636                    // A proposal exists, so this is not a real "no proposal"
637                    // reading — see `AdvisorStats::absent`'s own doc for why
638                    // that count comes from `proposal.is_none()` instead of
639                    // this field. `graph::Runner::advise` always calls
640                    // `apply_reflection` before saving, so the only way a
641                    // proposed record keeps the default `Absent` is a run.json
642                    // predating the `reflection` field. Fold it into `faint`
643                    // rather than dropping it from the breakdown entirely:
644                    // that is what `classify()` itself falls back to when
645                    // there is nothing to score against.
646                    crate::advise::Reflection::Absent => entry.faint += 1,
647                }
648            }
649        }
650
651        for round in &state.reviews {
652            totals.review_rounds += 1;
653
654            // A round whose fixer never reported back (crashed, timed out, or
655            // replied with something magi could not parse) leaves adoption
656            // unknown, not zero. Counting it would score every reviewer in
657            // that round as having been ignored, when the truth is simply
658            // unrecorded — so it stays out of the adoption-rate denominator
659            // entirely rather than silently becoming a round of 0 adoptions.
660            let report_lost = round.fix.as_ref().is_some_and(|f| f.failed.is_some());
661            let adopted: Vec<&String> = round
662                .fix
663                .as_ref()
664                .map(|f| f.addressed.iter().collect())
665                .unwrap_or_default();
666
667            for rec in &round.reviews {
668                let entry = reviewers
669                    .entry(rec.agent.clone())
670                    .or_insert_with(|| ReviewerStats {
671                        agent: rec.agent.clone(),
672                        ..ReviewerStats::default()
673                    });
674                // Seating and answering are facts about the seat itself: they
675                // hold whether or not this round's adoption is scoreable, so
676                // they are counted before the lost-report guard. A seat that
677                // never answered stays out of every scoring denominator —
678                // silence is not a review that found nothing.
679                entry.seated += 1;
680                if rec.failed.is_some() {
681                    entry.timeouts += 1;
682                    continue;
683                }
684                if report_lost {
685                    continue;
686                }
687                entry.rounds += 1;
688                entry.submitted += rec.findings.len();
689                for f in &rec.findings {
690                    if adopted.iter().any(|a| **a == f.id) {
691                        entry.adopted += 1;
692                    }
693                    let overlapped = round
694                        .reviews
695                        .iter()
696                        .filter(|other| other.reviewer != rec.reviewer)
697                        .flat_map(|other| other.findings.iter())
698                        .any(|g| same_defect(f, g));
699                    if !overlapped {
700                        entry.unique += 1;
701                    }
702                }
703            }
704
705            if round.e2e_deferred {
706                e2e.deferred += 1;
707            } else if !round.e2e.is_empty() {
708                e2e.rounds += 1;
709                if round.e2e.iter().any(|o| !o.ok()) {
710                    e2e.failures += 1;
711                    if round.blocking == 0 {
712                        e2e.sole_detections += 1;
713                    }
714                }
715            }
716        }
717    }
718
719    let mut agents: Vec<AgentStats> = agents.into_values().collect();
720    agents.sort_by(|a, b| {
721        b.win_rate()
722            .total_cmp(&a.win_rate())
723            .then(b.entered.cmp(&a.entered))
724    });
725    let mut reviewers: Vec<ReviewerStats> = reviewers.into_values().collect();
726    // A seat only sighted in rounds whose adoption could not be scored has
727    // nothing to report: no scoreable round, no silence to flag. It stays out
728    // of the table entirely rather than appearing as a row of zeroes, which
729    // would read as a reviewer that produced nothing.
730    reviewers.retain(|r| r.rounds > 0 || r.timeouts > 0);
731    reviewers.sort_by(|a, b| {
732        b.adopted_per_round()
733            .total_cmp(&a.adopted_per_round())
734            .then(b.rounds.cmp(&a.rounds))
735    });
736
737    let mut advisors: Vec<AdvisorStats> = advisors.into_values().collect();
738    advisors.sort_by(|a, b| {
739        b.reflection_rate()
740            .total_cmp(&a.reflection_rate())
741            .then(b.proposed.cmp(&a.proposed))
742    });
743
744    let nodes = node_durations(states);
745
746    Stats {
747        totals,
748        agents,
749        reviewers,
750        advisors,
751        e2e,
752        nodes,
753        release_bumps,
754    }
755}
756
757/// Do two findings describe the same defect?
758///
759/// A deliberate heuristic: same normalised title, or the same file within five
760/// lines. Two reviewers rarely word a finding identically, and exact matching
761/// would report every overlap as a unique find.
762fn same_defect(a: &crate::verdict::Finding, b: &crate::verdict::Finding) -> bool {
763    if normalize(&a.title) == normalize(&b.title) {
764        return true;
765    }
766    match (&a.file, &b.file) {
767        (Some(fa), Some(fb)) if fa == fb => match (a.line, b.line) {
768            (Some(la), Some(lb)) => la.abs_diff(lb) <= 5,
769            _ => false,
770        },
771        _ => false,
772    }
773}
774
775fn normalize(title: &str) -> String {
776    title
777        .chars()
778        .filter(|c| c.is_alphanumeric())
779        .map(|c| c.to_ascii_lowercase())
780        .collect()
781}
782
783#[cfg(test)]
784mod tests {
785    use super::*;
786    use crate::config::Config;
787    use crate::run::{Candidate, CommandOutcome, FixRecord, ReviewRecord, ReviewRound, Tally};
788    use crate::verdict::{Finding, Severity};
789    use std::path::PathBuf;
790
791    fn finding(id: &str, file: &str, line: u32, title: &str, sev: Severity) -> Finding {
792        Finding {
793            id: id.to_owned(),
794            severity: sev,
795            file: Some(file.to_owned()),
796            line: Some(line),
797            title: title.to_owned(),
798            detail: String::new(),
799        }
800    }
801
802    fn candidate(label: char, agent: &str) -> Candidate {
803        Candidate {
804            index: 0,
805            label,
806            agent: agent.to_owned(),
807            branch: format!("magi/x/{label}"),
808            worktree: PathBuf::from("/w"),
809            summary: String::new(),
810            stat: String::new(),
811            files: 1,
812            commits: 1,
813            empty: false,
814            failed: None,
815            verified_noop: None,
816            duration_ms: 0,
817            folded: false,
818        }
819    }
820
821    fn state_with(reviews: Vec<ReviewRound>, winner: char, status: RunStatus) -> RunState {
822        let mut s = RunState::new(
823            PathBuf::from("/repo"),
824            "main".to_owned(),
825            "abcdef".to_owned(),
826            "task".to_owned(),
827            Config::default(),
828        );
829        s.candidates = vec![candidate('A', "alpha"), candidate('B', "beta")];
830        s.tally = Some(Tally {
831            first_choice: BTreeMap::from([('A', 1), ('B', 2)]),
832            borda: BTreeMap::new(),
833            winner,
834            rankings: 3,
835            unanimous_initial: false,
836            deliberated: true,
837            changed_votes: 1,
838            unanimous_final: true,
839            tie_break: None,
840            judges: 3,
841            present: 3,
842            quorum: 2,
843            met_quorum: true,
844            uncontested: None,
845        });
846        s.reviews = reviews;
847        s.status = status;
848        s
849    }
850
851    fn state_with_repo(
852        reviews: Vec<ReviewRound>,
853        winner: char,
854        status: RunStatus,
855        repo: &str,
856    ) -> RunState {
857        let mut s = state_with(reviews, winner, status);
858        s.repo = PathBuf::from(repo);
859        s
860    }
861
862    #[test]
863    fn win_rates_and_completion_are_counted_per_agent() {
864        let states = vec![
865            state_with(Vec::new(), 'B', RunStatus::Merged),
866            state_with(Vec::new(), 'A', RunStatus::Blocked),
867        ];
868        let stats = collect(&states);
869        assert_eq!(stats.totals.runs, 2);
870        assert_eq!(stats.totals.merged, 1);
871        assert_eq!(stats.totals.blocked, 1);
872        assert_eq!(stats.totals.completion_rate(), 50.0);
873        assert_eq!(stats.totals.split, 2);
874        assert_eq!(stats.totals.minds_changed, 2);
875        assert_eq!(stats.totals.converged, 2);
876
877        let beta = stats.agents.iter().find(|a| a.agent == "beta").unwrap();
878        assert_eq!(beta.entered, 2);
879        assert_eq!(beta.wins, 1);
880        assert_eq!(beta.win_rate(), 50.0);
881    }
882
883    #[test]
884    fn reviewer_precision_and_uniqueness() {
885        let round = ReviewRound {
886            round: 1,
887            head: "h".to_owned(),
888            verified_head: None,
889            verified_at: None,
890            reviews: vec![
891                ReviewRecord {
892                    attempts: 0,
893                    reviewer: 1,
894                    agent: "alpha".to_owned(),
895                    summary: String::new(),
896                    findings: vec![
897                        finding(
898                            "R1-1-1",
899                            "src/a.rs",
900                            10,
901                            "panics on empty",
902                            Severity::Blocker,
903                        ),
904                        finding("R1-1-2", "src/b.rs", 40, "leaks a handle", Severity::Major),
905                    ],
906                    vote: None,
907                    failed: None,
908                    duration_ms: 0,
909                },
910                ReviewRecord {
911                    attempts: 0,
912                    reviewer: 2,
913                    agent: "beta".to_owned(),
914                    summary: String::new(),
915                    // Same defect as R1-1-1, three lines off: an overlap.
916                    findings: vec![finding(
917                        "R1-2-1",
918                        "src/a.rs",
919                        13,
920                        "empty input panic",
921                        Severity::Blocker,
922                    )],
923                    vote: None,
924                    failed: None,
925                    duration_ms: 0,
926                },
927            ],
928            e2e: Vec::new(),
929            verify_retried: false,
930            e2e_deferred: false,
931            e2e_defer_reason: None,
932            fix: Some(FixRecord {
933                agent: "alpha".to_owned(),
934                addressed: vec!["R1-1-1".to_owned()],
935                rejected: Vec::new(),
936                notes: String::new(),
937                committed: true,
938                failed: None,
939                duration_ms: 0,
940                continuation: None,
941            }),
942            blocking: 3,
943            answered: 2,
944            expected: 2,
945            clean: false,
946            progressed: true,
947            vote_split: false,
948            reconsideration: Vec::new(),
949            verdict: None,
950        };
951        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Ready)]);
952        let alpha = stats.reviewers.iter().find(|r| r.agent == "alpha").unwrap();
953        assert_eq!(alpha.submitted, 2);
954        assert_eq!(alpha.adopted, 1);
955        assert_eq!(alpha.precision(), 50.0);
956        assert_eq!(alpha.adopted_per_round(), 1.0);
957        // The src/a.rs finding overlaps beta's; src/b.rs does not.
958        assert_eq!(alpha.unique, 1);
959
960        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
961        assert_eq!(beta.submitted, 1);
962        assert_eq!(beta.adopted, 0);
963        assert_eq!(beta.unique, 0);
964    }
965
966    #[test]
967    fn a_lost_fix_report_does_not_count_as_zero_adoption() {
968        let submitted = ReviewRound {
969            round: 1,
970            head: "h".to_owned(),
971            verified_head: None,
972            verified_at: None,
973            reviews: vec![ReviewRecord {
974                attempts: 0,
975                reviewer: 1,
976                agent: "alpha".to_owned(),
977                summary: String::new(),
978                findings: vec![finding(
979                    "R1-1-1",
980                    "src/a.rs",
981                    10,
982                    "panics on empty",
983                    Severity::Blocker,
984                )],
985                vote: None,
986                failed: None,
987                duration_ms: 0,
988            }],
989            e2e: Vec::new(),
990            verify_retried: false,
991            e2e_deferred: false,
992            e2e_defer_reason: None,
993            // The fixer's diff may well have landed (blocking counts do fall
994            // round over round) — only its adoption report never came back.
995            fix: Some(FixRecord {
996                agent: "alpha".to_owned(),
997                addressed: Vec::new(),
998                rejected: Vec::new(),
999                notes: String::new(),
1000                committed: true,
1001                failed: Some("unparsable fix report".to_owned()),
1002                duration_ms: 0,
1003                continuation: None,
1004            }),
1005            blocking: 4,
1006            answered: 1,
1007            expected: 1,
1008            clean: false,
1009            progressed: false,
1010            vote_split: false,
1011            reconsideration: Vec::new(),
1012            verdict: None,
1013        };
1014        let stats = collect(&[state_with(vec![submitted], 'A', RunStatus::Ready)]);
1015        assert!(
1016            stats.reviewers.is_empty(),
1017            "a round with no adoption signal must not enter any reviewer's \
1018             denominator: {:?}",
1019            stats.reviewers
1020        );
1021    }
1022
1023    #[test]
1024    fn timed_out_seat_counts_as_a_timeout_not_a_clean_submission() {
1025        let round = ReviewRound {
1026            round: 1,
1027            head: "h".to_owned(),
1028            verified_head: None,
1029            verified_at: None,
1030            reviews: vec![
1031                ReviewRecord {
1032                    attempts: 0,
1033                    reviewer: 1,
1034                    agent: "alpha".to_owned(),
1035                    summary: String::new(),
1036                    findings: Vec::new(),
1037                    vote: None,
1038                    failed: None,
1039                    duration_ms: 0,
1040                },
1041                ReviewRecord {
1042                    attempts: 0,
1043                    reviewer: 2,
1044                    agent: "beta".to_owned(),
1045                    summary: String::new(),
1046                    findings: Vec::new(),
1047                    vote: None,
1048                    failed: Some("agent timed out".to_owned()),
1049                    duration_ms: 0,
1050                },
1051            ],
1052            e2e: Vec::new(),
1053            verify_retried: false,
1054            e2e_deferred: false,
1055            e2e_defer_reason: None,
1056            fix: None,
1057            blocking: 0,
1058            answered: 1,
1059            expected: 2,
1060            clean: false,
1061            progressed: false,
1062            vote_split: false,
1063            reconsideration: Vec::new(),
1064            verdict: None,
1065        };
1066        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Blocked)]);
1067
1068        let alpha = stats.reviewers.iter().find(|r| r.agent == "alpha").unwrap();
1069        assert_eq!(alpha.seated, 1);
1070        assert_eq!(alpha.rounds, 1);
1071        assert_eq!(alpha.timeouts, 0);
1072        assert_eq!(alpha.submitted, 0);
1073
1074        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
1075        assert_eq!(beta.seated, 1);
1076        assert_eq!(beta.timeouts, 1);
1077        assert_eq!(beta.submitted, 0);
1078        // A timeout must never read as a submission with nothing found: it
1079        // stays out of the scoring denominators entirely rather than becoming
1080        // a 0/0 that looks identical to a reviewer who answered and passed.
1081        assert_eq!(beta.rounds, 0);
1082        assert_eq!(beta.timeout_rate(), 100.0);
1083    }
1084
1085    #[test]
1086    fn a_timeout_is_still_recorded_when_the_round_also_lost_its_fix_report() {
1087        // Two independent gaps in one round: `beta` never answered, and the
1088        // fixer's adoption report never came back. The lost report suppresses
1089        // adoption scoring (see `a_lost_fix_report_does_not_count_as_zero_
1090        // adoption`) — it must not also swallow the fact that a seat was
1091        // silent, which is a property of the seat and not of the fixer.
1092        let round = ReviewRound {
1093            round: 1,
1094            head: "h".to_owned(),
1095            verified_head: None,
1096            verified_at: None,
1097            reviews: vec![
1098                ReviewRecord {
1099                    attempts: 0,
1100                    reviewer: 1,
1101                    agent: "alpha".to_owned(),
1102                    summary: String::new(),
1103                    findings: vec![finding(
1104                        "R1-1-1",
1105                        "src/a.rs",
1106                        10,
1107                        "panics on empty",
1108                        Severity::Blocker,
1109                    )],
1110                    vote: None,
1111                    failed: None,
1112                    duration_ms: 0,
1113                },
1114                ReviewRecord {
1115                    attempts: 0,
1116                    reviewer: 2,
1117                    agent: "beta".to_owned(),
1118                    summary: String::new(),
1119                    findings: Vec::new(),
1120                    vote: None,
1121                    failed: Some("agent timed out".to_owned()),
1122                    duration_ms: 0,
1123                },
1124            ],
1125            e2e: Vec::new(),
1126            verify_retried: false,
1127            e2e_deferred: false,
1128            e2e_defer_reason: None,
1129            fix: Some(FixRecord {
1130                agent: "alpha".to_owned(),
1131                addressed: Vec::new(),
1132                rejected: Vec::new(),
1133                notes: String::new(),
1134                committed: true,
1135                failed: Some("unparsable fix report".to_owned()),
1136                duration_ms: 0,
1137                continuation: None,
1138            }),
1139            blocking: 1,
1140            answered: 1,
1141            expected: 2,
1142            clean: false,
1143            progressed: false,
1144            vote_split: false,
1145            reconsideration: Vec::new(),
1146            verdict: None,
1147        };
1148        let stats = collect(&[state_with(vec![round], 'A', RunStatus::Blocked)]);
1149
1150        let beta = stats.reviewers.iter().find(|r| r.agent == "beta").unwrap();
1151        assert_eq!(beta.timeouts, 1);
1152        assert_eq!(beta.timeout_rate(), 100.0);
1153        // `alpha` answered, so the lost report keeps it out of the table
1154        // altogether — nothing about its findings can be scored.
1155        assert!(
1156            !stats.reviewers.iter().any(|r| r.agent == "alpha"),
1157            "{:?}",
1158            stats.reviewers
1159        );
1160    }
1161
1162    #[test]
1163    fn e2e_sole_detection_needs_a_clean_static_review() {
1164        let fail = CommandOutcome {
1165            command: "cargo test".to_owned(),
1166            code: Some(101),
1167            output_tail: "boom".to_owned(),
1168            duration_ms: 1,
1169            resource_blocked: false,
1170        };
1171        let sole = ReviewRound {
1172            round: 1,
1173            head: "h".to_owned(),
1174            verified_head: None,
1175            verified_at: None,
1176            reviews: Vec::new(),
1177            e2e: vec![fail.clone()],
1178            verify_retried: false,
1179            e2e_deferred: false,
1180            e2e_defer_reason: None,
1181            fix: None,
1182            blocking: 0,
1183            answered: 0,
1184            expected: 0,
1185            clean: false,
1186            progressed: false,
1187            vote_split: false,
1188            reconsideration: Vec::new(),
1189            verdict: None,
1190        };
1191        let alongside = ReviewRound {
1192            round: 2,
1193            head: "h".to_owned(),
1194            verified_head: None,
1195            verified_at: None,
1196            reviews: Vec::new(),
1197            e2e: vec![fail],
1198            verify_retried: false,
1199            e2e_deferred: false,
1200            e2e_defer_reason: None,
1201            fix: None,
1202            blocking: 2,
1203            answered: 0,
1204            expected: 0,
1205            clean: false,
1206            progressed: false,
1207            vote_split: false,
1208            reconsideration: Vec::new(),
1209            verdict: None,
1210        };
1211        let stats = collect(&[state_with(vec![sole, alongside], 'A', RunStatus::Ready)]);
1212        assert_eq!(stats.e2e.rounds, 2);
1213        assert_eq!(stats.e2e.failures, 2);
1214        assert_eq!(stats.e2e.sole_detections, 1);
1215        assert_eq!(stats.e2e.sole_rate(), 50.0);
1216    }
1217
1218    #[test]
1219    fn every_run_status_lands_in_exactly_one_breakdown_bucket() {
1220        let states = vec![
1221            state_with(Vec::new(), 'A', RunStatus::Merged),
1222            state_with(Vec::new(), 'A', RunStatus::Ready),
1223            state_with(Vec::new(), 'A', RunStatus::Blocked),
1224            state_with(Vec::new(), 'A', RunStatus::Failed),
1225            state_with(Vec::new(), 'A', RunStatus::Stalled),
1226            state_with(Vec::new(), 'A', RunStatus::VerifiedNoop),
1227            state_with(Vec::new(), 'A', RunStatus::Superseded),
1228            state_with(Vec::new(), 'A', RunStatus::Implementing),
1229            state_with(Vec::new(), 'A', RunStatus::Landing),
1230        ];
1231        let stats = collect(&states);
1232        let t = &stats.totals;
1233        assert_eq!(t.runs, 9);
1234        assert_eq!(t.merged, 1);
1235        assert_eq!(t.ready, 1);
1236        assert_eq!(t.blocked, 1);
1237        assert_eq!(t.failed, 1);
1238        assert_eq!(t.stalled, 1);
1239        assert_eq!(t.verified_noop, 1);
1240        assert_eq!(t.superseded, 1);
1241        // `Implementing` and `Landing` both fall into the one non-terminal
1242        // bucket.
1243        assert_eq!(t.in_progress, 2);
1244        assert_eq!(
1245            t.merged
1246                + t.ready
1247                + t.blocked
1248                + t.failed
1249                + t.stalled
1250                + t.verified_noop
1251                + t.superseded
1252                + t.in_progress,
1253            t.runs,
1254            "every run must land in exactly one bucket of the breakdown"
1255        );
1256    }
1257
1258    #[test]
1259    fn empty_input_yields_zeroed_rates_not_nan() {
1260        let stats = collect(&[]);
1261        assert_eq!(stats.totals.completion_rate(), 0.0);
1262        assert_eq!(stats.totals.split_rate(), 0.0);
1263        assert_eq!(stats.e2e.sole_rate(), 0.0);
1264        assert!(stats.agents.is_empty());
1265        assert!(stats.advisors.is_empty());
1266        assert_eq!(AdvisorStats::default().reflection_rate(), 0.0);
1267    }
1268
1269    fn advisor_record(
1270        seat: &str,
1271        agent: &str,
1272        proposal: Option<crate::verdict::Proposal>,
1273        reflection: crate::advise::Reflection,
1274    ) -> crate::advise::AdvisorRecord {
1275        crate::advise::AdvisorRecord {
1276            seat: seat.to_owned(),
1277            agent: agent.to_owned(),
1278            proposal,
1279            error: None,
1280            duration_ms: 0,
1281            reflection,
1282        }
1283    }
1284
1285    fn a_proposal() -> crate::verdict::Proposal {
1286        crate::verdict::Proposal {
1287            approach: "do the thing".to_owned(),
1288            key_tradeoff: "speed over memory".to_owned(),
1289            risks: Vec::new(),
1290            touches: Vec::new(),
1291            why_not_naive: "the naive version breaks under load".to_owned(),
1292        }
1293    }
1294
1295    #[test]
1296    fn advisor_stats_count_proposed_absent_and_reflection_split() {
1297        use crate::advise::{Advice, Reflection};
1298
1299        let mut s = state_with(Vec::new(), 'A', RunStatus::Merged);
1300        s.advice = Some(Advice {
1301            records: vec![
1302                advisor_record("advisor-1", "alpha", Some(a_proposal()), Reflection::Strong),
1303                advisor_record("advisor-2", "alpha", Some(a_proposal()), Reflection::Faint),
1304                advisor_record("advisor-3", "alpha", None, Reflection::Absent),
1305            ],
1306            synthesis: Some("blended brief".to_owned()),
1307        });
1308
1309        let stats = collect(&[s]);
1310        let alpha = stats.advisors.iter().find(|a| a.agent == "alpha").unwrap();
1311        assert_eq!(alpha.seated, 3);
1312        assert_eq!(alpha.proposed, 2);
1313        assert_eq!(alpha.absent, 1);
1314        assert_eq!(alpha.strong, 1);
1315        assert_eq!(alpha.faint, 1);
1316        assert_eq!(alpha.reflection_rate(), 50.0);
1317    }
1318
1319    #[test]
1320    fn advisor_stats_count_absent_from_the_proposal_not_the_reflection_default() {
1321        // A record whose `proposal` is `None` but whose `reflection` was
1322        // never classified (predates `apply_reflection`, or the field's own
1323        // serde default) must still count as `absent` — and a run whose
1324        // synthesis never ran leaves every *proposed* record `Faint` by the
1325        // same default, which must not spill into `absent` either.
1326        //
1327        // A third case: a proposed record whose `reflection` was *never*
1328        // classified at all (a run.json predating the `reflection` field)
1329        // must not vanish from the breakdown either — it has to land
1330        // somewhere in faint/strong, not be silently dropped from all three
1331        // counters while still counting toward `proposed`.
1332        use crate::advise::{Advice, Reflection};
1333
1334        let mut s = state_with(Vec::new(), 'A', RunStatus::Merged);
1335        s.advice = Some(Advice {
1336            records: vec![
1337                advisor_record("advisor-1", "alpha", None, Reflection::Absent),
1338                advisor_record("advisor-2", "alpha", Some(a_proposal()), Reflection::Faint),
1339                advisor_record("advisor-3", "alpha", Some(a_proposal()), Reflection::Absent),
1340            ],
1341            synthesis: None,
1342        });
1343
1344        let stats = collect(&[s]);
1345        let alpha = stats.advisors.iter().find(|a| a.agent == "alpha").unwrap();
1346        assert_eq!(alpha.seated, 3);
1347        assert_eq!(alpha.proposed, 2);
1348        assert_eq!(alpha.absent, 1);
1349        assert_eq!(alpha.faint, 2);
1350        assert_eq!(alpha.strong, 0);
1351    }
1352
1353    #[test]
1354    fn advisor_stats_ignore_runs_with_advise_off() {
1355        let s = state_with(Vec::new(), 'A', RunStatus::Merged);
1356        assert!(s.advice.is_none());
1357        let stats = collect(&[s]);
1358        assert!(stats.advisors.is_empty());
1359    }
1360
1361    #[test]
1362    fn same_defect_matches_titles_across_files() {
1363        let a = finding("1", "src/a.rs", 1, "Panics On Empty!", Severity::Major);
1364        let b = finding("2", "src/z.rs", 900, "panics on empty", Severity::Nit);
1365        assert!(same_defect(&a, &b));
1366        let c = finding("3", "src/z.rs", 900, "totally different", Severity::Nit);
1367        assert!(!same_defect(&a, &c));
1368    }
1369
1370    #[test]
1371    fn release_bump_stats_split_clean_from_attention_and_track_coverage() {
1372        use crate::run::ReleaseBump;
1373
1374        // Not recorded at all: uses the release-bump step? unknown.
1375        let unrecorded = state_with(Vec::new(), 'A', RunStatus::Merged);
1376
1377        // Clean: automerge worked, nobody had to look at it.
1378        let mut automerged = state_with(Vec::new(), 'A', RunStatus::Merged);
1379        automerged.release_bump = Some(ReleaseBump {
1380            pr_url: Some("https://github.com/o/r/pull/1".to_owned()),
1381            version: Some("1.2.3".to_owned()),
1382            automerge_enabled: true,
1383            merged_directly: false,
1384            problem: None,
1385            action_required: None,
1386        });
1387
1388        // Also clean, but automerge itself was rejected by GitHub and magi
1389        // merged the already-green PR directly - `automerge_enabled` reads
1390        // false here, and that must not make this count as needing a human.
1391        let mut merged_directly = state_with(Vec::new(), 'A', RunStatus::Merged);
1392        merged_directly.release_bump = Some(ReleaseBump {
1393            pr_url: Some("https://github.com/o/r/pull/2".to_owned()),
1394            version: Some("1.2.4".to_owned()),
1395            automerge_enabled: false,
1396            merged_directly: true,
1397            problem: None,
1398            action_required: None,
1399        });
1400
1401        // Needs a human, PR opened.
1402        let mut blocked_with_pr = state_with(Vec::new(), 'A', RunStatus::Merged);
1403        blocked_with_pr.release_bump = Some(ReleaseBump {
1404            pr_url: Some("https://github.com/o/r/pull/3".to_owned()),
1405            version: Some("1.2.5".to_owned()),
1406            automerge_enabled: false,
1407            merged_directly: false,
1408            problem: Some("checks red".to_owned()),
1409            action_required: Some("look at the PR".to_owned()),
1410        });
1411
1412        // Needs a human, no PR ever opened.
1413        let mut blocked_without_pr = state_with(Vec::new(), 'A', RunStatus::Merged);
1414        blocked_without_pr.release_bump = Some(ReleaseBump {
1415            pr_url: None,
1416            version: Some("1.2.6".to_owned()),
1417            automerge_enabled: false,
1418            merged_directly: false,
1419            problem: Some("gh pr create failed".to_owned()),
1420            action_required: Some("open the PR by hand".to_owned()),
1421        });
1422
1423        let stats = collect(&[
1424            unrecorded,
1425            automerged,
1426            merged_directly,
1427            blocked_with_pr,
1428            blocked_without_pr,
1429        ]);
1430        let b = &stats.release_bumps;
1431        assert_eq!(b.merged, 5);
1432        assert_eq!(b.recorded, 4);
1433        assert_eq!(b.pr_opened, 3);
1434        assert_eq!(b.automerge_enabled, 1);
1435        assert_eq!(b.merged_directly, 1);
1436        assert_eq!(b.needs_attention, 2);
1437        assert_eq!(b.clean(), 2);
1438        // Clean and attention must always split `recorded` exactly.
1439        assert_eq!(b.clean() + b.needs_attention, b.recorded);
1440        assert_eq!(b.coverage_rate(), 80.0);
1441        assert!((b.automerge_rate() - 33.333_333_333_333_336).abs() < 1e-9);
1442        assert_eq!(b.attention_rate(), 50.0);
1443    }
1444
1445    #[test]
1446    fn release_bump_ignores_runs_that_are_not_merged() {
1447        use crate::run::ReleaseBump;
1448
1449        let mut blocked = state_with(Vec::new(), 'A', RunStatus::Blocked);
1450        blocked.release_bump = Some(ReleaseBump {
1451            pr_url: Some("https://github.com/o/r/pull/9".to_owned()),
1452            version: Some("9.9.9".to_owned()),
1453            automerge_enabled: true,
1454            merged_directly: false,
1455            problem: None,
1456            action_required: None,
1457        });
1458
1459        let stats = collect(&[blocked]);
1460        let b = &stats.release_bumps;
1461        assert_eq!(b.merged, 0);
1462        assert_eq!(b.recorded, 0);
1463        assert_eq!(b.pr_opened, 0);
1464    }
1465
1466    #[test]
1467    fn release_bump_stats_are_zero_on_merged_runs_with_no_bump_or_no_runs() {
1468        let stats = collect(&[state_with(Vec::new(), 'A', RunStatus::Merged)]);
1469        let b = &stats.release_bumps;
1470        assert_eq!(b.merged, 1);
1471        assert_eq!(b.recorded, 0);
1472        assert_eq!(b.coverage_rate(), 0.0);
1473        assert_eq!(b.automerge_rate(), 0.0);
1474        assert_eq!(b.attention_rate(), 0.0);
1475        assert_eq!(b.clean(), 0);
1476
1477        let empty = collect(&[]);
1478        let b = &empty.release_bumps;
1479        assert_eq!(b.merged, 0);
1480        assert_eq!(b.coverage_rate(), 0.0);
1481        assert_eq!(b.automerge_rate(), 0.0);
1482        assert_eq!(b.attention_rate(), 0.0);
1483    }
1484
1485    fn event(node: &str, at_secs: i64, message: &str) -> crate::run::Event {
1486        crate::run::Event {
1487            at: jiff::Timestamp::from_second(at_secs).unwrap(),
1488            node: node.to_owned(),
1489            message: message.to_owned(),
1490        }
1491    }
1492
1493    fn state_with_events(events: Vec<crate::run::Event>) -> RunState {
1494        let mut s = RunState::new(
1495            PathBuf::from("/repo"),
1496            "main".to_owned(),
1497            "abcdef".to_owned(),
1498            "task".to_owned(),
1499            Config::default(),
1500        );
1501        s.events = events;
1502        s
1503    }
1504
1505    #[test]
1506    fn a_node_with_multiple_events_spans_first_to_last() {
1507        let s = state_with_events(vec![
1508            event("implement", 1_000, "start"),
1509            event("implement", 1_030, "still running"),
1510            event("implement", 1_090, "done"),
1511        ]);
1512        let nodes = node_durations(&[s]);
1513        assert_eq!(nodes.len(), 1);
1514        assert_eq!(nodes[0].node, "implement");
1515        assert_eq!(nodes[0].runs, 1);
1516        assert_eq!(nodes[0].total_secs, 90);
1517        assert_eq!(nodes[0].max_secs, 90);
1518        assert_eq!(nodes[0].single, 0);
1519        assert_eq!(nodes[0].mean_secs(), 90.0);
1520    }
1521
1522    #[test]
1523    fn a_node_with_a_single_event_is_unmeasured_not_zero() {
1524        let s = state_with_events(vec![event("gate", 2_000, "ran once")]);
1525        let nodes = node_durations(&[s]);
1526        assert_eq!(nodes.len(), 1);
1527        assert_eq!(nodes[0].node, "gate");
1528        assert_eq!(nodes[0].runs, 0);
1529        assert_eq!(nodes[0].total_secs, 0);
1530        assert_eq!(nodes[0].single, 1);
1531        // No measured runs: the mean must read as zero, never NaN or a
1532        // fabricated span from the lone event.
1533        assert_eq!(nodes[0].mean_secs(), 0.0);
1534    }
1535
1536    #[test]
1537    fn a_run_with_no_events_produces_no_node_rows() {
1538        let s = state_with_events(vec![]);
1539        let nodes = node_durations(&[s]);
1540        assert!(nodes.is_empty());
1541    }
1542
1543    #[test]
1544    fn multiple_runs_aggregate_the_same_node() {
1545        let a = state_with_events(vec![event("judge", 0, "start"), event("judge", 60, "done")]);
1546        let b = state_with_events(vec![
1547            event("judge", 0, "start"),
1548            event("judge", 200, "done"),
1549        ]);
1550        // A single-event run for the same node must add to `single` without
1551        // disturbing the measured runs' total or max.
1552        let c = state_with_events(vec![event("judge", 5, "start")]);
1553        let nodes = node_durations(&[a, b, c]);
1554        assert_eq!(nodes.len(), 1);
1555        let judge = &nodes[0];
1556        assert_eq!(judge.node, "judge");
1557        assert_eq!(judge.runs, 2);
1558        assert_eq!(judge.total_secs, 260);
1559        assert_eq!(judge.max_secs, 200);
1560        assert_eq!(judge.single, 1);
1561        assert_eq!(judge.mean_secs(), 130.0);
1562    }
1563
1564    #[test]
1565    fn by_repo_splits_states_and_group_totals_sum_to_the_whole() {
1566        let states = vec![
1567            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1568            state_with_repo(Vec::new(), 'A', RunStatus::Blocked, "/repos/a"),
1569            state_with_repo(Vec::new(), 'B', RunStatus::Merged, "/repos/b"),
1570        ];
1571        let groups = by_repo(&states);
1572        assert_eq!(groups.len(), 2);
1573
1574        let total_runs: usize = groups.iter().map(|g| g.stats.totals.runs).sum();
1575        assert_eq!(total_runs, collect(&states).totals.runs);
1576        let total_merged: usize = groups.iter().map(|g| g.stats.totals.merged).sum();
1577        assert_eq!(total_merged, collect(&states).totals.merged);
1578
1579        // Busiest repository (2 runs) sorts first.
1580        assert_eq!(groups[0].repo, PathBuf::from("/repos/a"));
1581        assert_eq!(groups[0].name, "a");
1582        assert_eq!(groups[0].stats.totals.runs, 2);
1583        assert_eq!(groups[1].repo, PathBuf::from("/repos/b"));
1584        assert_eq!(groups[1].name, "b");
1585        assert_eq!(groups[1].stats.totals.runs, 1);
1586    }
1587
1588    #[test]
1589    fn by_repo_breaks_a_run_count_tie_by_path() {
1590        let states = vec![
1591            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/z"),
1592            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1593        ];
1594        let groups = by_repo(&states);
1595        assert_eq!(groups.len(), 2);
1596        assert_eq!(groups[0].repo, PathBuf::from("/repos/a"));
1597        assert_eq!(groups[1].repo, PathBuf::from("/repos/z"));
1598    }
1599
1600    #[test]
1601    fn by_repo_on_empty_input_yields_no_groups() {
1602        assert!(by_repo(&[]).is_empty());
1603    }
1604
1605    #[test]
1606    fn filter_repo_matches_the_full_path_exactly() {
1607        let states = vec![
1608            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/a"),
1609            state_with_repo(Vec::new(), 'A', RunStatus::Merged, "/repos/ab"),
1610        ];
1611        let hits = filter_repo(&states, Path::new("/repos/a"));
1612        assert_eq!(hits.len(), 1);
1613        assert_eq!(hits[0].repo, PathBuf::from("/repos/a"));
1614    }
1615
1616    #[test]
1617    fn filter_repo_returns_nothing_for_an_unknown_repo_or_empty_input() {
1618        let states = vec![state_with_repo(
1619            Vec::new(),
1620            'A',
1621            RunStatus::Merged,
1622            "/repos/a",
1623        )];
1624        assert!(filter_repo(&states, Path::new("/repos/nope")).is_empty());
1625        assert!(filter_repo(&[], Path::new("/repos/a")).is_empty());
1626    }
1627}