magi/
recovery.rs

1//! Carrying an earlier attempt's evidence into the run that takes its branch over.
2//!
3//! A task whose previous run stopped on a failed verification used to reopen
4//! its branch as a context-free review-only run: the new reviewers and fixer
5//! never saw the failing command's output, the review decisions or the
6//! unresolved findings, and round 1 started from scratch.
7//!
8//! [`assess`] is pure. From the earlier run's record and the relation of the
9//! branch tip to the commit that run reviewed it decides between
10//!
11//! * [`Mode::VerifyRepair`]: the earlier panel found nothing blocking and only
12//!   verification failed, on exactly the commit that is about to be reviewed.
13//!   The fixer gets the failure first, then the review is scoped to what
14//!   changed since the reviewed commit.
15//! * [`Mode::FullReview`]: anything doubtful. The reviewers still see the
16//!   earlier context, but nothing is scoped and no approval is carried; each
17//!   [`RecoveryRecord::reasons`] entry says why.
18//!
19//! Inherited verification is evidence for a prompt, never a result: it is not
20//! copied into [`crate::run::ReviewRound`], so verification always runs again
21//! on the commit that is going to land.
22
23use serde::{Deserialize, Serialize};
24
25use crate::run::{CommandOutcome, E2eStatus, ReviewRound, RunState};
26
27/// Restore an interrupted repair and confirm its original commit before any review.
28pub(crate) async fn restore_repair(wt: &std::path::Path, before: &str) -> anyhow::Result<()> {
29    crate::git::restore_to(wt, before).await?;
30    let tip = crate::git::rev_parse(wt, "HEAD").await?;
31    anyhow::ensure!(tip == before, "the winner is at {tip}, not {before}");
32    Ok(())
33}
34
35fn short(sha: &str) -> &str {
36    sha.get(..8).unwrap_or(sha)
37}
38use crate::verdict::{Finding, ReviewVote};
39
40/// How many times evidence may be handed from one run to the next. Past this
41/// the successor starts as a plain full review.
42pub const MAX_GENERATION: u32 = 2;
43
44/// Longest output tail carried into a prompt, in bytes.
45pub const TAIL_MAX: usize = 6_000;
46
47/// Longest scoped diff shown to a reviewer, in bytes.
48pub const SINCE_DIFF_MAX: usize = 60_000;
49
50/// What the review after a takeover does with the earlier run.
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "kebab-case")]
53pub enum Mode {
54    /// One repair fix with the failure in hand, then a review scoped to the
55    /// changes since the reviewed commit.
56    VerifyRepair,
57    /// The whole patch is reviewed again; the earlier run is context only.
58    FullReview,
59}
60
61/// Which check failed.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "kebab-case")]
64pub enum Source {
65    /// The round's `verify.e2e`.
66    E2e,
67    /// The final `verify.gate`.
68    Gate,
69}
70
71/// The failed verification the earlier run stopped on.
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73pub struct Evidence {
74    /// Which check.
75    pub source: Source,
76    /// The command, as configured.
77    pub command: String,
78    /// Exit code, `None` on timeout or signal.
79    pub code: Option<i32>,
80    /// Scrubbed, clipped tail of its output.
81    pub output_tail: String,
82    /// The commit it was run against.
83    pub head: String,
84}
85
86/// One earlier review round, reduced to its decision.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct PriorRound {
89    /// Round number in the earlier run.
90    pub round: usize,
91    /// The commit it reviewed.
92    pub head: String,
93    /// Findings that held the merge.
94    pub blocking: usize,
95    /// Seats that answered, of `expected`.
96    pub answered: usize,
97    /// Seats the round expected.
98    pub expected: usize,
99    /// The round's verdict, when a vote was usable.
100    pub verdict: Option<ReviewVote>,
101}
102
103/// Where the one repair fix stands. Saved before the fixer is called, so a
104/// resume never repairs twice.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107pub enum Repair {
108    /// Not attempted (yet, or not applicable).
109    NotStarted,
110    /// The fixer was called from this commit and has not been settled.
111    Started {
112        /// HEAD before the call.
113        before: String,
114    },
115    /// The seat could not start. Restore under coordination before retrying.
116    WaitingForRestore {
117        /// HEAD before the blocked attempt (or its earlier fallback).
118        before: String,
119    },
120    /// Settled.
121    Done {
122        /// HEAD before the call.
123        before: String,
124        /// HEAD after it.
125        after: String,
126        /// The fixer's own commit or rescue commit moved HEAD.
127        committed: bool,
128        /// Why nothing was kept (the worktree was restored to `before`).
129        failed: Option<String>,
130    },
131}
132
133/// What a run inherited from the attempt it took over, and why.
134#[derive(Debug, Clone, Serialize, Deserialize)]
135pub struct RecoveryRecord {
136    /// The earlier run.
137    pub from_run: String,
138    /// How many hand-overs of evidence lie behind this one.
139    pub generation: u32,
140    /// The decision.
141    pub mode: Mode,
142    /// For [`Mode::FullReview`]: every reason a narrower recovery was not
143    /// justified. Empty for [`Mode::VerifyRepair`].
144    #[serde(default)]
145    pub reasons: Vec<String>,
146    /// The earlier run's last reviewed commit, when it had one.
147    #[serde(default)]
148    pub reviewed_head: Option<String>,
149    /// The earlier rounds' decisions.
150    #[serde(default)]
151    pub rounds: Vec<PriorRound>,
152    /// Findings the last earlier round left open.
153    #[serde(default)]
154    pub open_findings: Vec<Finding>,
155    /// The failed verification, when one was recorded.
156    #[serde(default)]
157    pub evidence: Option<Evidence>,
158    /// The task's own statement replaced the commit-log stand-in.
159    #[serde(default)]
160    pub criteria_inherited: bool,
161    /// The repair fix.
162    #[serde(default = "not_started")]
163    pub repair: Repair,
164}
165
166fn not_started() -> Repair {
167    Repair::NotStarted
168}
169
170/// How the branch tip relates to the commit the earlier run reviewed.
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Relation {
173    /// The same commit.
174    Same,
175    /// The reviewed commit is a proper ancestor: commits were added after it.
176    Descendant,
177    /// Not an ancestor: rebased, rewritten or unknown to this repository.
178    Diverged,
179}
180
181/// Clip `text` to its last `max` bytes on a char boundary.
182pub fn tail_of(text: &str, max: usize) -> String {
183    if text.len() <= max {
184        return text.to_owned();
185    }
186    let mut start = text.len() - max;
187    while !text.is_char_boundary(start) {
188        start += 1;
189    }
190    format!("[...]\n{}", &text[start..])
191}
192
193/// Clip `text` to its first `max` bytes on a char boundary.
194pub fn head_of(text: &str, max: usize) -> String {
195    if text.len() <= max {
196        return text.to_owned();
197    }
198    let mut end = max;
199    while !text.is_char_boundary(end) {
200        end -= 1;
201    }
202    format!("{}\n[... truncated]", &text[..end])
203}
204
205fn evidence_from(round: &ReviewRound, prior: &RunState) -> Option<Evidence> {
206    let scrub = |s: &str| {
207        crate::scrub::scrub(
208            &crate::run::summarize_verify(s, TAIL_MAX),
209            &crate::scrub::Identity::current(),
210        )
211    };
212    let failed = |o: &&CommandOutcome| !o.ok() && !o.resource_blocked;
213    if round.e2e_status() == E2eStatus::Failed
214        && let (Some(o), Some(head)) = (round.e2e.iter().find(failed), &round.verified_head)
215    {
216        return Some(Evidence {
217            source: Source::E2e,
218            command: o.command.clone(),
219            code: o.code,
220            output_tail: scrub(&o.output_tail),
221            head: head.clone(),
222        });
223    }
224    // The gate runs on the tree the last round left; it is only evidence about
225    // that round's head while no gate fix moved the tree afterwards.
226    if prior.gate_ran
227        && prior.gate_fixes.is_empty()
228        && let Some(o) = prior.gate.iter().find(failed)
229    {
230        return Some(Evidence {
231            source: Source::Gate,
232            command: o.command.clone(),
233            code: o.code,
234            output_tail: scrub(&o.output_tail),
235            head: round.head.clone(),
236        });
237    }
238    None
239}
240
241/// The record for a run that takes `prior` over. Pure.
242///
243/// `prior` is `None` when the earlier attempt's record could not be read.
244/// `base_now` is the base commit the new run reviews against and `relation`
245/// how the branch tip stands to the earlier run's last reviewed commit.
246pub fn assess(
247    from_run: &str,
248    prior: Option<&RunState>,
249    base_now: &str,
250    relation: Relation,
251) -> RecoveryRecord {
252    let mut rec = RecoveryRecord {
253        from_run: from_run.to_owned(),
254        generation: 0,
255        mode: Mode::FullReview,
256        reasons: Vec::new(),
257        reviewed_head: None,
258        rounds: Vec::new(),
259        open_findings: Vec::new(),
260        evidence: None,
261        criteria_inherited: false,
262        repair: Repair::NotStarted,
263    };
264    let Some(prior) = prior else {
265        rec.reasons
266            .push("the earlier run's record could not be read".to_owned());
267        return rec;
268    };
269    rec.generation = prior.recovery.as_ref().map_or(0, |r| r.generation) + 1;
270    rec.rounds = prior
271        .reviews
272        .iter()
273        .map(|r| PriorRound {
274            round: r.round,
275            head: r.head.clone(),
276            blocking: r.blocking,
277            answered: r.answered,
278            expected: r.expected,
279            verdict: r.verdict,
280        })
281        .collect();
282    let Some(last) = prior.reviews.last() else {
283        rec.reasons
284            .push("the earlier run never completed a review round".to_owned());
285        return rec;
286    };
287    rec.reviewed_head = Some(last.head.clone());
288    rec.open_findings = last
289        .reviews
290        .iter()
291        .flat_map(|r| r.findings.iter().cloned())
292        .collect();
293    rec.evidence = evidence_from(last, prior);
294
295    if rec.generation > MAX_GENERATION {
296        rec.reasons.push(format!(
297            "evidence has already been handed over {MAX_GENERATION} times"
298        ));
299    }
300    if last.answered < last.expected || last.reviews.iter().any(|r| r.failed.is_some()) {
301        rec.reasons.push(format!(
302            "only {} of {} reviewer seat(s) answered the last round; a panel that did not \
303             answer is not a clean one",
304            last.answered, last.expected
305        ));
306    }
307    if last.blocking > 0 {
308        rec.reasons.push(format!(
309            "the last round still had {} blocking finding(s)",
310            last.blocking
311        ));
312    }
313    match &rec.evidence {
314        None => rec
315            .reasons
316            .push("no failed verification with a known command was recorded".to_owned()),
317        Some(e) if e.output_tail.trim().is_empty() => rec
318            .reasons
319            .push("the failed verification kept no output to act on".to_owned()),
320        Some(e) if e.head != last.head => {
321            let sync = prior.base_sync.as_ref().map_or_else(
322                || "no base sync recorded".to_owned(),
323                |sync| {
324                    format!(
325                        "base sync to {}, {} rebase attempt(s), conflict: {}",
326                        sync.tip,
327                        sync.attempts,
328                        sync.conflict.as_deref().unwrap_or("none")
329                    )
330                },
331            );
332            rec.reasons.push(format!(
333                "the failed verification was about a different commit {} than the one reviewed {}; \
334                 {sync}; additional review is required before approval can carry forward",
335                e.head, last.head,
336            ));
337        }
338        Some(_) => {}
339    }
340    match relation {
341        Relation::Same => {}
342        Relation::Descendant => rec.reasons.push(
343            "commits were added after the reviewed commit; they were never reviewed".to_owned(),
344        ),
345        Relation::Diverged => rec.reasons.push(
346            "the reviewed commit is not an ancestor of the branch: it was rebased or rewritten"
347                .to_owned(),
348        ),
349    }
350    if prior.base_commit != base_now {
351        rec.reasons.push(format!(
352            "the base moved from {} to {}",
353            short(&prior.base_commit),
354            short(base_now)
355        ));
356    }
357    if rec.reasons.is_empty() {
358        rec.mode = Mode::VerifyRepair;
359    }
360    rec
361}
362
363/// Re-check a [`Mode::VerifyRepair`] record against the tree the review is
364/// about to look at (a base sync may have rebased it after the takeover was
365/// assessed). Returns true when it was downgraded. Pure.
366pub fn recheck(rec: &mut RecoveryRecord, head: &str) -> bool {
367    if rec.mode != Mode::VerifyRepair {
368        return false;
369    }
370    let Some(reviewed) = rec.reviewed_head.as_deref() else {
371        return false;
372    };
373    if reviewed == head {
374        return false;
375    }
376    rec.mode = Mode::FullReview;
377    rec.reasons.push(format!(
378        "the branch moved from {} to {} before the review started (rebased onto the base or \
379         changed); the earlier approval does not carry",
380        short(reviewed),
381        short(head)
382    ));
383    true
384}
385
386impl RecoveryRecord {
387    /// One line for the event log, `magi show` and the web.
388    pub fn summary(&self) -> String {
389        let from = crate::run::short_of(&self.from_run);
390        match self.mode {
391            Mode::VerifyRepair => format!(
392                "inherited from run {from}: reviewed commit, {} round(s), failed verification \
393                 `{}`; repair first, then a review scoped to the changes since",
394                self.rounds.len(),
395                self.evidence.as_ref().map_or("?", |e| e.command.as_str())
396            ),
397            Mode::FullReview => format!(
398                "full review after run {from}: {}",
399                if self.reasons.is_empty() {
400                    "no narrower recovery applied".to_owned()
401                } else {
402                    self.reasons.join("; ")
403                }
404            ),
405        }
406    }
407}
408
409#[cfg(test)]
410mod tests {
411    use super::*;
412    use crate::run::ReviewRecord;
413    use crate::verdict::Severity;
414
415    fn outcome(code: i32, tail: &str) -> CommandOutcome {
416        CommandOutcome {
417            command: "cargo test".to_owned(),
418            code: Some(code),
419            output_tail: tail.to_owned(),
420            duration_ms: 1,
421            resource_blocked: false,
422        }
423    }
424
425    fn seat(failed: bool, findings: Vec<Finding>) -> ReviewRecord {
426        ReviewRecord {
427            reviewer: 1,
428            agent: "a".to_owned(),
429            summary: String::new(),
430            findings,
431            vote: Some(ReviewVote::Approve),
432            failed: failed.then(|| "timed out".to_owned()),
433            duration_ms: 0,
434            attempts: 0,
435        }
436    }
437
438    fn round(head: &str, e2e: Vec<CommandOutcome>) -> ReviewRound {
439        ReviewRound {
440            round: 3,
441            head: head.to_owned(),
442            verified_head: Some(head.to_owned()),
443            verified_at: None,
444            reviews: vec![seat(false, Vec::new())],
445            e2e,
446            verify_retried: false,
447            e2e_deferred: false,
448            e2e_defer_reason: None,
449            fix: None,
450            blocking: 0,
451            answered: 1,
452            expected: 1,
453            clean: false,
454            progressed: false,
455            vote_split: false,
456            reconsideration: Vec::new(),
457            verdict: Some(ReviewVote::Approve),
458        }
459    }
460
461    fn prior(round: ReviewRound) -> RunState {
462        let mut s = RunState::new(
463            std::path::PathBuf::from("/tmp/x"),
464            "main".to_owned(),
465            "base1".to_owned(),
466            "task".to_owned(),
467            crate::config::Config::default(),
468        );
469        s.reviews.push(round);
470        s
471    }
472
473    fn failing() -> RunState {
474        prior(round("aaa111", vec![outcome(101, "1 failed: quiet_spawn")]))
475    }
476
477    #[test]
478    fn unchanged_commit_with_a_nonblocking_failed_verification_is_a_verify_repair() {
479        let r = assess("r1", Some(&failing()), "base1", Relation::Same);
480        assert_eq!(r.mode, Mode::VerifyRepair, "{:?}", r.reasons);
481        assert!(r.reasons.is_empty());
482        assert_eq!(r.generation, 1);
483        let e = r.evidence.unwrap();
484        assert_eq!(e.command, "cargo test");
485        assert_eq!(e.head, "aaa111");
486        assert!(e.output_tail.contains("quiet_spawn"));
487    }
488
489    #[test]
490    fn verification_recovery_preserves_panic_before_large_backtrace() {
491        let output = format!(
492            "[full output: artifacts/e2e-1.out]\n---- widget stdout ----\nwidget log\nthread 'widget' panicked at widget.rs:12:\nassertion failed: widget missing\n{}test result: FAILED\n",
493            "  41: 0x1234 - backtrace frame\n".repeat(10000)
494        );
495        let p = prior(round("aaa111", vec![outcome(101, &output)]));
496        let r = assess("r1", Some(&p), "base1", Relation::Same);
497        let prompt = crate::prompt::recovery_fix("repair widget", &r, 2, "en");
498        assert!(prompt.contains("panicked at widget.rs:12"));
499        assert!(prompt.contains("assertion failed: widget missing"));
500        let evidence = r.evidence.unwrap();
501        assert!(evidence.output_tail.len() <= TAIL_MAX);
502        assert!(evidence.output_tail.contains("panicked at widget.rs:12"));
503        assert!(
504            evidence
505                .output_tail
506                .contains("assertion failed: widget missing")
507        );
508        assert!(evidence.output_tail.contains("artifacts/e2e-1.out"));
509    }
510
511    #[test]
512    fn descendant_rebased_and_moved_base_widen_the_review() {
513        let p = failing();
514        let d = assess("r1", Some(&p), "base1", Relation::Descendant);
515        assert_eq!(d.mode, Mode::FullReview);
516        assert!(d.reasons[0].contains("never reviewed"));
517        let x = assess("r1", Some(&p), "base1", Relation::Diverged);
518        assert_eq!(x.mode, Mode::FullReview);
519        assert!(x.reasons[0].contains("rebased"));
520        let b = assess("r1", Some(&p), "base2", Relation::Same);
521        assert_eq!(b.mode, Mode::FullReview);
522        assert!(b.reasons[0].contains("base moved"));
523        // The evidence still travels as context.
524        assert!(b.evidence.is_some());
525    }
526
527    #[test]
528    fn missing_or_empty_evidence_falls_back_to_a_full_review() {
529        let none = prior(round("aaa111", Vec::new()));
530        let r = assess("r1", Some(&none), "base1", Relation::Same);
531        assert_eq!(r.mode, Mode::FullReview);
532        assert!(r.reasons[0].contains("no failed verification"));
533        let empty = prior(round("aaa111", vec![outcome(1, "  \n")]));
534        let r = assess("r1", Some(&empty), "base1", Relation::Same);
535        assert_eq!(r.mode, Mode::FullReview);
536        assert!(r.reasons[0].contains("no output"));
537        let r = assess("r1", None, "base1", Relation::Same);
538        assert_eq!(r.mode, Mode::FullReview);
539        assert!(r.reasons[0].contains("could not be read"));
540    }
541
542    #[test]
543    fn evidence_about_another_commit_is_not_reused() {
544        let mut p = failing();
545        p.reviews[0].verified_head = Some("bbb222".to_owned());
546        let r = assess("r1", Some(&p), "base1", Relation::Same);
547        assert_eq!(r.mode, Mode::FullReview);
548        assert!(r.reasons[0].contains("different commit"));
549    }
550
551    #[test]
552    fn a_timed_out_seat_or_a_blocking_finding_is_never_a_clean_panel() {
553        let mut p = failing();
554        p.reviews[0].answered = 0;
555        p.reviews[0].reviews[0] = seat(true, Vec::new());
556        let r = assess("r1", Some(&p), "base1", Relation::Same);
557        assert_eq!(r.mode, Mode::FullReview);
558        assert!(r.reasons[0].contains("did not answer"));
559
560        let mut p = failing();
561        p.reviews[0].blocking = 1;
562        p.reviews[0].reviews[0].findings = vec![Finding {
563            id: "R3-1-1".to_owned(),
564            severity: Severity::Major,
565            file: None,
566            line: None,
567            title: "t".to_owned(),
568            detail: String::new(),
569        }];
570        let r = assess("r1", Some(&p), "base1", Relation::Same);
571        assert_eq!(r.mode, Mode::FullReview);
572        assert_eq!(r.open_findings.len(), 1);
573    }
574
575    #[test]
576    fn generations_are_bounded() {
577        let mut p = failing();
578        let mut inherited = assess("r0", Some(&failing()), "base1", Relation::Same);
579        inherited.generation = MAX_GENERATION;
580        p.recovery = Some(inherited);
581        let r = assess("r1", Some(&p), "base1", Relation::Same);
582        assert_eq!(r.generation, MAX_GENERATION + 1);
583        assert_eq!(r.mode, Mode::FullReview);
584        assert!(r.reasons[0].contains("handed over"));
585    }
586
587    #[test]
588    fn a_gate_failure_counts_only_while_no_gate_fix_moved_the_tree() {
589        let mut p = prior(round("aaa111", Vec::new()));
590        p.gate_ran = true;
591        p.gate = vec![outcome(2, "gate red")];
592        let r = assess("r1", Some(&p), "base1", Relation::Same);
593        assert_eq!(r.mode, Mode::VerifyRepair);
594        assert_eq!(r.evidence.unwrap().source, Source::Gate);
595        p.gate_fixes.push(crate::run::GateFixRecord {
596            agent: "a".to_owned(),
597            failed: Vec::new(),
598            notes: String::new(),
599            committed: true,
600            error: None,
601        });
602        let r = assess("r1", Some(&p), "base1", Relation::Same);
603        assert_eq!(r.mode, Mode::FullReview);
604    }
605
606    #[test]
607    fn recheck_downgrades_a_moved_branch() {
608        let mut r = assess("r1", Some(&failing()), "base1", Relation::Same);
609        assert!(!recheck(&mut r, "aaa111"));
610        assert_eq!(r.mode, Mode::VerifyRepair);
611        assert!(recheck(&mut r, "ccc333"));
612        assert_eq!(r.mode, Mode::FullReview);
613        assert!(r.reasons[0].contains("moved"));
614    }
615
616    #[test]
617    fn tails_are_clipped_on_char_boundaries() {
618        let s = "あ".repeat(100);
619        assert!(tail_of(&s, 10).len() < 30);
620        assert!(head_of(&s, 10).starts_with('あ'));
621        assert_eq!(tail_of("ok", 10), "ok");
622    }
623}
magi/recovery.rs

magi/
recovery.rs
