magi/
recovery.rs

1//! Carrying an earlier attempt's evidence into the run that takes its branch over.
2//!
3//! A task whose previous run stopped on a failed verification used to reopen
4//! its branch as a context-free review-only run: the new reviewers and fixer
5//! never saw the failing command's output, the review decisions or the
6//! unresolved findings, and round 1 started from scratch.
7//!
8//! [`assess`] is pure. From the earlier run's record and the relation of the
9//! branch tip to the commit that run reviewed it decides between
10//!
11//! * [`Mode::VerifyRepair`]: the earlier panel found nothing blocking and only
12//!   verification failed, on exactly the commit that is about to be reviewed.
13//!   The fixer gets the failure first, then the review is scoped to what
14//!   changed since the reviewed commit.
15//! * [`Mode::FullReview`]: anything doubtful. The reviewers still see the
16//!   earlier context, but nothing is scoped and no approval is carried; each
17//!   [`RecoveryRecord::reasons`] entry says why.
18//!
19//! Inherited verification is evidence for a prompt, never a result: it is not
20//! copied into [`crate::run::ReviewRound`], so verification always runs again
21//! on the commit that is going to land.
22
23use serde::{Deserialize, Serialize};
24
25use crate::run::{CommandOutcome, E2eStatus, ReviewRound, RunState};
26
27/// Restore an interrupted repair and confirm its original commit before any review.
28pub(crate) async fn restore_repair(wt: &std::path::Path, before: &str) -> anyhow::Result<()> {
29    crate::git::restore_to(wt, before).await?;
30    let tip = crate::git::rev_parse(wt, "HEAD").await?;
31    anyhow::ensure!(tip == before, "the winner is at {tip}, not {before}");
32    Ok(())
33}
34
35fn short(sha: &str) -> &str {
36    sha.get(..8).unwrap_or(sha)
37}
38use crate::verdict::{Finding, ReviewVote};
39
40/// How many times evidence may be handed from one run to the next. Past this
41/// the successor starts as a plain full review.
42pub const MAX_GENERATION: u32 = 2;
43
44/// Longest output tail carried into a prompt, in bytes.
45pub const TAIL_MAX: usize = 6_000;
46
47/// Longest scoped diff shown to a reviewer, in bytes.
48pub const SINCE_DIFF_MAX: usize = 60_000;
49
50/// What the review after a takeover does with the earlier run.
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "kebab-case")]
53pub enum Mode {
54    /// One repair fix with the failure in hand, then a review scoped to the
55    /// changes since the reviewed commit.
56    VerifyRepair,
57    /// The whole patch is reviewed again; the earlier run is context only.
58    FullReview,
59}
60
61/// Which check failed.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "kebab-case")]
64pub enum Source {
65    /// The round's `verify.e2e`.
66    E2e,
67    /// The final `verify.gate`.
68    Gate,
69}
70
71/// The failed verification the earlier run stopped on.
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73pub struct Evidence {
74    /// Which check.
75    pub source: Source,
76    /// The command, as configured.
77    pub command: String,
78    /// Exit code, `None` on timeout or signal.
79    pub code: Option<i32>,
80    /// Scrubbed, clipped tail of its output.
81    pub output_tail: String,
82    /// The commit it was run against.
83    pub head: String,
84}
85
86/// One earlier review round, reduced to its decision.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct PriorRound {
89    /// Round number in the earlier run.
90    pub round: usize,
91    /// The commit it reviewed.
92    pub head: String,
93    /// Findings that held the merge.
94    pub blocking: usize,
95    /// Seats that answered, of `expected`.
96    pub answered: usize,
97    /// Seats the round expected.
98    pub expected: usize,
99    /// The round's verdict, when a vote was usable.
100    pub verdict: Option<ReviewVote>,
101}
102
103/// Where the one repair fix stands. Saved before the fixer is called, so a
104/// resume never repairs twice.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107pub enum Repair {
108    /// Not attempted (yet, or not applicable).
109    NotStarted,
110    /// The fixer was called from this commit and has not been settled.
111    Started {
112        /// HEAD before the call.
113        before: String,
114    },
115    /// Settled.
116    Done {
117        /// HEAD before the call.
118        before: String,
119        /// HEAD after it.
120        after: String,
121        /// The fixer's own commit or rescue commit moved HEAD.
122        committed: bool,
123        /// Why nothing was kept (the worktree was restored to `before`).
124        failed: Option<String>,
125    },
126}
127
128/// What a run inherited from the attempt it took over, and why.
129#[derive(Debug, Clone, Serialize, Deserialize)]
130pub struct RecoveryRecord {
131    /// The earlier run.
132    pub from_run: String,
133    /// How many hand-overs of evidence lie behind this one.
134    pub generation: u32,
135    /// The decision.
136    pub mode: Mode,
137    /// For [`Mode::FullReview`]: every reason a narrower recovery was not
138    /// justified. Empty for [`Mode::VerifyRepair`].
139    #[serde(default)]
140    pub reasons: Vec<String>,
141    /// The earlier run's last reviewed commit, when it had one.
142    #[serde(default)]
143    pub reviewed_head: Option<String>,
144    /// The earlier rounds' decisions.
145    #[serde(default)]
146    pub rounds: Vec<PriorRound>,
147    /// Findings the last earlier round left open.
148    #[serde(default)]
149    pub open_findings: Vec<Finding>,
150    /// The failed verification, when one was recorded.
151    #[serde(default)]
152    pub evidence: Option<Evidence>,
153    /// The task's own statement replaced the commit-log stand-in.
154    #[serde(default)]
155    pub criteria_inherited: bool,
156    /// The repair fix.
157    #[serde(default = "not_started")]
158    pub repair: Repair,
159}
160
161fn not_started() -> Repair {
162    Repair::NotStarted
163}
164
165/// How the branch tip relates to the commit the earlier run reviewed.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum Relation {
168    /// The same commit.
169    Same,
170    /// The reviewed commit is a proper ancestor: commits were added after it.
171    Descendant,
172    /// Not an ancestor: rebased, rewritten or unknown to this repository.
173    Diverged,
174}
175
176/// Clip `text` to its last `max` bytes on a char boundary.
177pub fn tail_of(text: &str, max: usize) -> String {
178    if text.len() <= max {
179        return text.to_owned();
180    }
181    let mut start = text.len() - max;
182    while !text.is_char_boundary(start) {
183        start += 1;
184    }
185    format!("[...]\n{}", &text[start..])
186}
187
188/// Clip `text` to its first `max` bytes on a char boundary.
189pub fn head_of(text: &str, max: usize) -> String {
190    if text.len() <= max {
191        return text.to_owned();
192    }
193    let mut end = max;
194    while !text.is_char_boundary(end) {
195        end -= 1;
196    }
197    format!("{}\n[... truncated]", &text[..end])
198}
199
200fn evidence_from(round: &ReviewRound, prior: &RunState) -> Option<Evidence> {
201    let scrub =
202        |s: &str| crate::scrub::scrub(&tail_of(s, TAIL_MAX), &crate::scrub::Identity::current());
203    let failed = |o: &&CommandOutcome| !o.ok() && !o.resource_blocked;
204    if round.e2e_status() == E2eStatus::Failed
205        && let (Some(o), Some(head)) = (round.e2e.iter().find(failed), &round.verified_head)
206    {
207        return Some(Evidence {
208            source: Source::E2e,
209            command: o.command.clone(),
210            code: o.code,
211            output_tail: scrub(&o.output_tail),
212            head: head.clone(),
213        });
214    }
215    // The gate runs on the tree the last round left; it is only evidence about
216    // that round's head while no gate fix moved the tree afterwards.
217    if prior.gate_ran
218        && prior.gate_fixes.is_empty()
219        && let Some(o) = prior.gate.iter().find(failed)
220    {
221        return Some(Evidence {
222            source: Source::Gate,
223            command: o.command.clone(),
224            code: o.code,
225            output_tail: scrub(&o.output_tail),
226            head: round.head.clone(),
227        });
228    }
229    None
230}
231
232/// The record for a run that takes `prior` over. Pure.
233///
234/// `prior` is `None` when the earlier attempt's record could not be read.
235/// `base_now` is the base commit the new run reviews against and `relation`
236/// how the branch tip stands to the earlier run's last reviewed commit.
237pub fn assess(
238    from_run: &str,
239    prior: Option<&RunState>,
240    base_now: &str,
241    relation: Relation,
242) -> RecoveryRecord {
243    let mut rec = RecoveryRecord {
244        from_run: from_run.to_owned(),
245        generation: 0,
246        mode: Mode::FullReview,
247        reasons: Vec::new(),
248        reviewed_head: None,
249        rounds: Vec::new(),
250        open_findings: Vec::new(),
251        evidence: None,
252        criteria_inherited: false,
253        repair: Repair::NotStarted,
254    };
255    let Some(prior) = prior else {
256        rec.reasons
257            .push("the earlier run's record could not be read".to_owned());
258        return rec;
259    };
260    rec.generation = prior.recovery.as_ref().map_or(0, |r| r.generation) + 1;
261    rec.rounds = prior
262        .reviews
263        .iter()
264        .map(|r| PriorRound {
265            round: r.round,
266            head: r.head.clone(),
267            blocking: r.blocking,
268            answered: r.answered,
269            expected: r.expected,
270            verdict: r.verdict,
271        })
272        .collect();
273    let Some(last) = prior.reviews.last() else {
274        rec.reasons
275            .push("the earlier run never completed a review round".to_owned());
276        return rec;
277    };
278    rec.reviewed_head = Some(last.head.clone());
279    rec.open_findings = last
280        .reviews
281        .iter()
282        .flat_map(|r| r.findings.iter().cloned())
283        .collect();
284    rec.evidence = evidence_from(last, prior);
285
286    if rec.generation > MAX_GENERATION {
287        rec.reasons.push(format!(
288            "evidence has already been handed over {MAX_GENERATION} times"
289        ));
290    }
291    if last.answered < last.expected || last.reviews.iter().any(|r| r.failed.is_some()) {
292        rec.reasons.push(format!(
293            "only {} of {} reviewer seat(s) answered the last round; a panel that did not \
294             answer is not a clean one",
295            last.answered, last.expected
296        ));
297    }
298    if last.blocking > 0 {
299        rec.reasons.push(format!(
300            "the last round still had {} blocking finding(s)",
301            last.blocking
302        ));
303    }
304    match &rec.evidence {
305        None => rec
306            .reasons
307            .push("no failed verification with a known command was recorded".to_owned()),
308        Some(e) if e.output_tail.trim().is_empty() => rec
309            .reasons
310            .push("the failed verification kept no output to act on".to_owned()),
311        Some(e) if e.head != last.head => rec.reasons.push(
312            "the failed verification was about a different commit than the one reviewed".to_owned(),
313        ),
314        Some(_) => {}
315    }
316    match relation {
317        Relation::Same => {}
318        Relation::Descendant => rec.reasons.push(
319            "commits were added after the reviewed commit; they were never reviewed".to_owned(),
320        ),
321        Relation::Diverged => rec.reasons.push(
322            "the reviewed commit is not an ancestor of the branch: it was rebased or rewritten"
323                .to_owned(),
324        ),
325    }
326    if prior.base_commit != base_now {
327        rec.reasons.push(format!(
328            "the base moved from {} to {}",
329            short(&prior.base_commit),
330            short(base_now)
331        ));
332    }
333    if rec.reasons.is_empty() {
334        rec.mode = Mode::VerifyRepair;
335    }
336    rec
337}
338
339/// Re-check a [`Mode::VerifyRepair`] record against the tree the review is
340/// about to look at (a base sync may have rebased it after the takeover was
341/// assessed). Returns true when it was downgraded. Pure.
342pub fn recheck(rec: &mut RecoveryRecord, head: &str) -> bool {
343    if rec.mode != Mode::VerifyRepair {
344        return false;
345    }
346    let Some(reviewed) = rec.reviewed_head.as_deref() else {
347        return false;
348    };
349    if reviewed == head {
350        return false;
351    }
352    rec.mode = Mode::FullReview;
353    rec.reasons.push(format!(
354        "the branch moved from {} to {} before the review started (rebased onto the base or \
355         changed); the earlier approval does not carry",
356        short(reviewed),
357        short(head)
358    ));
359    true
360}
361
362impl RecoveryRecord {
363    /// One line for the event log, `magi show` and the web.
364    pub fn summary(&self) -> String {
365        let from = crate::run::short_of(&self.from_run);
366        match self.mode {
367            Mode::VerifyRepair => format!(
368                "inherited from run {from}: reviewed commit, {} round(s), failed verification \
369                 `{}`; repair first, then a review scoped to the changes since",
370                self.rounds.len(),
371                self.evidence.as_ref().map_or("?", |e| e.command.as_str())
372            ),
373            Mode::FullReview => format!(
374                "full review after run {from}: {}",
375                if self.reasons.is_empty() {
376                    "no narrower recovery applied".to_owned()
377                } else {
378                    self.reasons.join("; ")
379                }
380            ),
381        }
382    }
383}
384
385#[cfg(test)]
386mod tests {
387    use super::*;
388    use crate::run::ReviewRecord;
389    use crate::verdict::Severity;
390
391    fn outcome(code: i32, tail: &str) -> CommandOutcome {
392        CommandOutcome {
393            command: "cargo test".to_owned(),
394            code: Some(code),
395            output_tail: tail.to_owned(),
396            duration_ms: 1,
397            resource_blocked: false,
398        }
399    }
400
401    fn seat(failed: bool, findings: Vec<Finding>) -> ReviewRecord {
402        ReviewRecord {
403            reviewer: 1,
404            agent: "a".to_owned(),
405            summary: String::new(),
406            findings,
407            vote: Some(ReviewVote::Approve),
408            failed: failed.then(|| "timed out".to_owned()),
409            duration_ms: 0,
410            attempts: 0,
411        }
412    }
413
414    fn round(head: &str, e2e: Vec<CommandOutcome>) -> ReviewRound {
415        ReviewRound {
416            round: 3,
417            head: head.to_owned(),
418            verified_head: Some(head.to_owned()),
419            verified_at: None,
420            reviews: vec![seat(false, Vec::new())],
421            e2e,
422            verify_retried: false,
423            e2e_deferred: false,
424            e2e_defer_reason: None,
425            fix: None,
426            blocking: 0,
427            answered: 1,
428            expected: 1,
429            clean: false,
430            progressed: false,
431            vote_split: false,
432            reconsideration: Vec::new(),
433            verdict: Some(ReviewVote::Approve),
434        }
435    }
436
437    fn prior(round: ReviewRound) -> RunState {
438        let mut s = RunState::new(
439            std::path::PathBuf::from("/tmp/x"),
440            "main".to_owned(),
441            "base1".to_owned(),
442            "task".to_owned(),
443            crate::config::Config::default(),
444        );
445        s.reviews.push(round);
446        s
447    }
448
449    fn failing() -> RunState {
450        prior(round("aaa111", vec![outcome(101, "1 failed: quiet_spawn")]))
451    }
452
453    #[test]
454    fn unchanged_commit_with_a_nonblocking_failed_verification_is_a_verify_repair() {
455        let r = assess("r1", Some(&failing()), "base1", Relation::Same);
456        assert_eq!(r.mode, Mode::VerifyRepair, "{:?}", r.reasons);
457        assert!(r.reasons.is_empty());
458        assert_eq!(r.generation, 1);
459        let e = r.evidence.unwrap();
460        assert_eq!(e.command, "cargo test");
461        assert_eq!(e.head, "aaa111");
462        assert!(e.output_tail.contains("quiet_spawn"));
463    }
464
465    #[test]
466    fn descendant_rebased_and_moved_base_widen_the_review() {
467        let p = failing();
468        let d = assess("r1", Some(&p), "base1", Relation::Descendant);
469        assert_eq!(d.mode, Mode::FullReview);
470        assert!(d.reasons[0].contains("never reviewed"));
471        let x = assess("r1", Some(&p), "base1", Relation::Diverged);
472        assert_eq!(x.mode, Mode::FullReview);
473        assert!(x.reasons[0].contains("rebased"));
474        let b = assess("r1", Some(&p), "base2", Relation::Same);
475        assert_eq!(b.mode, Mode::FullReview);
476        assert!(b.reasons[0].contains("base moved"));
477        // The evidence still travels as context.
478        assert!(b.evidence.is_some());
479    }
480
481    #[test]
482    fn missing_or_empty_evidence_falls_back_to_a_full_review() {
483        let none = prior(round("aaa111", Vec::new()));
484        let r = assess("r1", Some(&none), "base1", Relation::Same);
485        assert_eq!(r.mode, Mode::FullReview);
486        assert!(r.reasons[0].contains("no failed verification"));
487        let empty = prior(round("aaa111", vec![outcome(1, "  \n")]));
488        let r = assess("r1", Some(&empty), "base1", Relation::Same);
489        assert_eq!(r.mode, Mode::FullReview);
490        assert!(r.reasons[0].contains("no output"));
491        let r = assess("r1", None, "base1", Relation::Same);
492        assert_eq!(r.mode, Mode::FullReview);
493        assert!(r.reasons[0].contains("could not be read"));
494    }
495
496    #[test]
497    fn evidence_about_another_commit_is_not_reused() {
498        let mut p = failing();
499        p.reviews[0].verified_head = Some("bbb222".to_owned());
500        let r = assess("r1", Some(&p), "base1", Relation::Same);
501        assert_eq!(r.mode, Mode::FullReview);
502        assert!(r.reasons[0].contains("different commit"));
503    }
504
505    #[test]
506    fn a_timed_out_seat_or_a_blocking_finding_is_never_a_clean_panel() {
507        let mut p = failing();
508        p.reviews[0].answered = 0;
509        p.reviews[0].reviews[0] = seat(true, Vec::new());
510        let r = assess("r1", Some(&p), "base1", Relation::Same);
511        assert_eq!(r.mode, Mode::FullReview);
512        assert!(r.reasons[0].contains("did not answer"));
513
514        let mut p = failing();
515        p.reviews[0].blocking = 1;
516        p.reviews[0].reviews[0].findings = vec![Finding {
517            id: "R3-1-1".to_owned(),
518            severity: Severity::Major,
519            file: None,
520            line: None,
521            title: "t".to_owned(),
522            detail: String::new(),
523        }];
524        let r = assess("r1", Some(&p), "base1", Relation::Same);
525        assert_eq!(r.mode, Mode::FullReview);
526        assert_eq!(r.open_findings.len(), 1);
527    }
528
529    #[test]
530    fn generations_are_bounded() {
531        let mut p = failing();
532        let mut inherited = assess("r0", Some(&failing()), "base1", Relation::Same);
533        inherited.generation = MAX_GENERATION;
534        p.recovery = Some(inherited);
535        let r = assess("r1", Some(&p), "base1", Relation::Same);
536        assert_eq!(r.generation, MAX_GENERATION + 1);
537        assert_eq!(r.mode, Mode::FullReview);
538        assert!(r.reasons[0].contains("handed over"));
539    }
540
541    #[test]
542    fn a_gate_failure_counts_only_while_no_gate_fix_moved_the_tree() {
543        let mut p = prior(round("aaa111", Vec::new()));
544        p.gate_ran = true;
545        p.gate = vec![outcome(2, "gate red")];
546        let r = assess("r1", Some(&p), "base1", Relation::Same);
547        assert_eq!(r.mode, Mode::VerifyRepair);
548        assert_eq!(r.evidence.unwrap().source, Source::Gate);
549        p.gate_fixes.push(crate::run::GateFixRecord {
550            agent: "a".to_owned(),
551            failed: Vec::new(),
552            notes: String::new(),
553            committed: true,
554            error: None,
555        });
556        let r = assess("r1", Some(&p), "base1", Relation::Same);
557        assert_eq!(r.mode, Mode::FullReview);
558    }
559
560    #[test]
561    fn recheck_downgrades_a_moved_branch() {
562        let mut r = assess("r1", Some(&failing()), "base1", Relation::Same);
563        assert!(!recheck(&mut r, "aaa111"));
564        assert_eq!(r.mode, Mode::VerifyRepair);
565        assert!(recheck(&mut r, "ccc333"));
566        assert_eq!(r.mode, Mode::FullReview);
567        assert!(r.reasons[0].contains("moved"));
568    }
569
570    #[test]
571    fn tails_are_clipped_on_char_boundaries() {
572        let s = "あ".repeat(100);
573        assert!(tail_of(&s, 10).len() < 30);
574        assert!(head_of(&s, 10).starts_with('あ'));
575        assert_eq!(tail_of("ok", 10), "ok");
576    }
577}
magi/recovery.rs

magi/
recovery.rs
