magi/
recovery.rs

1//! Carrying an earlier attempt's evidence into the run that takes its branch over.
2//!
3//! A task whose previous run stopped on a failed verification used to reopen
4//! its branch as a context-free review-only run: the new reviewers and fixer
5//! never saw the failing command's output, the review decisions or the
6//! unresolved findings, and round 1 started from scratch.
7//!
8//! [`assess`] is pure. From the earlier run's record and the relation of the
9//! branch tip to the commit that run reviewed it decides between
10//!
11//! * [`Mode::VerifyRepair`]: the earlier panel found nothing blocking and only
12//!   verification failed, on exactly the commit that is about to be reviewed.
13//!   The fixer gets the failure first, then the review is scoped to what
14//!   changed since the reviewed commit.
15//! * [`Mode::FullReview`]: anything doubtful. The reviewers still see the
16//!   earlier context, but nothing is scoped and no approval is carried; each
17//!   [`RecoveryRecord::reasons`] entry says why.
18//!
19//! Inherited verification is evidence for a prompt, never a result: it is not
20//! copied into [`crate::run::ReviewRound`], so verification always runs again
21//! on the commit that is going to land.
22
23use serde::{Deserialize, Serialize};
24
25use crate::run::{CommandOutcome, E2eStatus, ReviewRound, RunState};
26
27/// Restore an interrupted repair and confirm its original commit before any review.
28pub(crate) async fn restore_repair(wt: &std::path::Path, before: &str) -> anyhow::Result<()> {
29    crate::git::restore_to(wt, before).await?;
30    let tip = crate::git::rev_parse(wt, "HEAD").await?;
31    anyhow::ensure!(tip == before, "the winner is at {tip}, not {before}");
32    Ok(())
33}
34
35fn short(sha: &str) -> &str {
36    sha.get(..8).unwrap_or(sha)
37}
38use crate::verdict::{Finding, ReviewVote};
39
40/// How many times evidence may be handed from one run to the next. Past this
41/// the successor starts as a plain full review.
42pub const MAX_GENERATION: u32 = 2;
43
44/// Longest output tail carried into a prompt, in bytes.
45pub const TAIL_MAX: usize = 6_000;
46
47/// Longest scoped diff shown to a reviewer, in bytes.
48pub const SINCE_DIFF_MAX: usize = 60_000;
49
50/// What the review after a takeover does with the earlier run.
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "kebab-case")]
53pub enum Mode {
54    /// One repair fix with the failure in hand, then a review scoped to the
55    /// changes since the reviewed commit.
56    VerifyRepair,
57    /// The whole patch is reviewed again; the earlier run is context only.
58    FullReview,
59}
60
61/// Which check failed.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "kebab-case")]
64pub enum Source {
65    /// The round's `verify.e2e`.
66    E2e,
67    /// The final `verify.gate`.
68    Gate,
69}
70
71/// The failed verification the earlier run stopped on.
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73pub struct Evidence {
74    /// Which check.
75    pub source: Source,
76    /// The command, as configured.
77    pub command: String,
78    /// Exit code, `None` on timeout or signal.
79    pub code: Option<i32>,
80    /// Scrubbed, clipped tail of its output.
81    pub output_tail: String,
82    /// The commit it was run against.
83    pub head: String,
84}
85
86/// One earlier review round, reduced to its decision.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct PriorRound {
89    /// Round number in the earlier run.
90    pub round: usize,
91    /// The commit it reviewed.
92    pub head: String,
93    /// Findings that held the merge.
94    pub blocking: usize,
95    /// Seats that answered, of `expected`.
96    pub answered: usize,
97    /// Seats the round expected.
98    pub expected: usize,
99    /// The round's verdict, when a vote was usable.
100    pub verdict: Option<ReviewVote>,
101}
102
103/// Where the one repair fix stands. Saved before the fixer is called, so a
104/// resume never repairs twice.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107pub enum Repair {
108    /// Not attempted (yet, or not applicable).
109    NotStarted,
110    /// The fixer was called from this commit and has not been settled.
111    Started {
112        /// HEAD before the call.
113        before: String,
114    },
115    /// The seat could not start. Restore under coordination before retrying.
116    WaitingForRestore {
117        /// HEAD before the blocked attempt (or its earlier fallback).
118        before: String,
119    },
120    /// Settled.
121    Done {
122        /// HEAD before the call.
123        before: String,
124        /// HEAD after it.
125        after: String,
126        /// The fixer's own commit or rescue commit moved HEAD.
127        committed: bool,
128        /// Why nothing was kept (the worktree was restored to `before`).
129        failed: Option<String>,
130    },
131}
132
133/// What a run inherited from the attempt it took over, and why.
134#[derive(Debug, Clone, Serialize, Deserialize)]
135pub struct RecoveryRecord {
136    /// The earlier run.
137    pub from_run: String,
138    /// How many hand-overs of evidence lie behind this one.
139    pub generation: u32,
140    /// The decision.
141    pub mode: Mode,
142    /// For [`Mode::FullReview`]: every reason a narrower recovery was not
143    /// justified. Empty for [`Mode::VerifyRepair`].
144    #[serde(default)]
145    pub reasons: Vec<String>,
146    /// The earlier run's last reviewed commit, when it had one.
147    #[serde(default)]
148    pub reviewed_head: Option<String>,
149    /// The earlier rounds' decisions.
150    #[serde(default)]
151    pub rounds: Vec<PriorRound>,
152    /// Findings the last earlier round left open.
153    #[serde(default)]
154    pub open_findings: Vec<Finding>,
155    /// The failed verification, when one was recorded.
156    #[serde(default)]
157    pub evidence: Option<Evidence>,
158    /// The task's own statement replaced the commit-log stand-in.
159    #[serde(default)]
160    pub criteria_inherited: bool,
161    /// The repair fix.
162    #[serde(default = "not_started")]
163    pub repair: Repair,
164}
165
166fn not_started() -> Repair {
167    Repair::NotStarted
168}
169
170/// How the branch tip relates to the commit the earlier run reviewed.
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Relation {
173    /// The same commit.
174    Same,
175    /// The reviewed commit is a proper ancestor: commits were added after it.
176    Descendant,
177    /// Not an ancestor: rebased, rewritten or unknown to this repository.
178    Diverged,
179}
180
181/// Clip `text` to its last `max` bytes on a char boundary.
182pub fn tail_of(text: &str, max: usize) -> String {
183    if text.len() <= max {
184        return text.to_owned();
185    }
186    let mut start = text.len() - max;
187    while !text.is_char_boundary(start) {
188        start += 1;
189    }
190    format!("[...]\n{}", &text[start..])
191}
192
193/// Clip `text` to its first `max` bytes on a char boundary.
194pub fn head_of(text: &str, max: usize) -> String {
195    if text.len() <= max {
196        return text.to_owned();
197    }
198    let mut end = max;
199    while !text.is_char_boundary(end) {
200        end -= 1;
201    }
202    format!("{}\n[... truncated]", &text[..end])
203}
204
205fn evidence_from(round: &ReviewRound, prior: &RunState) -> Option<Evidence> {
206    let scrub =
207        |s: &str| crate::scrub::scrub(&tail_of(s, TAIL_MAX), &crate::scrub::Identity::current());
208    let failed = |o: &&CommandOutcome| !o.ok() && !o.resource_blocked;
209    if round.e2e_status() == E2eStatus::Failed
210        && let (Some(o), Some(head)) = (round.e2e.iter().find(failed), &round.verified_head)
211    {
212        return Some(Evidence {
213            source: Source::E2e,
214            command: o.command.clone(),
215            code: o.code,
216            output_tail: scrub(&o.output_tail),
217            head: head.clone(),
218        });
219    }
220    // The gate runs on the tree the last round left; it is only evidence about
221    // that round's head while no gate fix moved the tree afterwards.
222    if prior.gate_ran
223        && prior.gate_fixes.is_empty()
224        && let Some(o) = prior.gate.iter().find(failed)
225    {
226        return Some(Evidence {
227            source: Source::Gate,
228            command: o.command.clone(),
229            code: o.code,
230            output_tail: scrub(&o.output_tail),
231            head: round.head.clone(),
232        });
233    }
234    None
235}
236
237/// The record for a run that takes `prior` over. Pure.
238///
239/// `prior` is `None` when the earlier attempt's record could not be read.
240/// `base_now` is the base commit the new run reviews against and `relation`
241/// how the branch tip stands to the earlier run's last reviewed commit.
242pub fn assess(
243    from_run: &str,
244    prior: Option<&RunState>,
245    base_now: &str,
246    relation: Relation,
247) -> RecoveryRecord {
248    let mut rec = RecoveryRecord {
249        from_run: from_run.to_owned(),
250        generation: 0,
251        mode: Mode::FullReview,
252        reasons: Vec::new(),
253        reviewed_head: None,
254        rounds: Vec::new(),
255        open_findings: Vec::new(),
256        evidence: None,
257        criteria_inherited: false,
258        repair: Repair::NotStarted,
259    };
260    let Some(prior) = prior else {
261        rec.reasons
262            .push("the earlier run's record could not be read".to_owned());
263        return rec;
264    };
265    rec.generation = prior.recovery.as_ref().map_or(0, |r| r.generation) + 1;
266    rec.rounds = prior
267        .reviews
268        .iter()
269        .map(|r| PriorRound {
270            round: r.round,
271            head: r.head.clone(),
272            blocking: r.blocking,
273            answered: r.answered,
274            expected: r.expected,
275            verdict: r.verdict,
276        })
277        .collect();
278    let Some(last) = prior.reviews.last() else {
279        rec.reasons
280            .push("the earlier run never completed a review round".to_owned());
281        return rec;
282    };
283    rec.reviewed_head = Some(last.head.clone());
284    rec.open_findings = last
285        .reviews
286        .iter()
287        .flat_map(|r| r.findings.iter().cloned())
288        .collect();
289    rec.evidence = evidence_from(last, prior);
290
291    if rec.generation > MAX_GENERATION {
292        rec.reasons.push(format!(
293            "evidence has already been handed over {MAX_GENERATION} times"
294        ));
295    }
296    if last.answered < last.expected || last.reviews.iter().any(|r| r.failed.is_some()) {
297        rec.reasons.push(format!(
298            "only {} of {} reviewer seat(s) answered the last round; a panel that did not \
299             answer is not a clean one",
300            last.answered, last.expected
301        ));
302    }
303    if last.blocking > 0 {
304        rec.reasons.push(format!(
305            "the last round still had {} blocking finding(s)",
306            last.blocking
307        ));
308    }
309    match &rec.evidence {
310        None => rec
311            .reasons
312            .push("no failed verification with a known command was recorded".to_owned()),
313        Some(e) if e.output_tail.trim().is_empty() => rec
314            .reasons
315            .push("the failed verification kept no output to act on".to_owned()),
316        Some(e) if e.head != last.head => rec.reasons.push(
317            "the failed verification was about a different commit than the one reviewed".to_owned(),
318        ),
319        Some(_) => {}
320    }
321    match relation {
322        Relation::Same => {}
323        Relation::Descendant => rec.reasons.push(
324            "commits were added after the reviewed commit; they were never reviewed".to_owned(),
325        ),
326        Relation::Diverged => rec.reasons.push(
327            "the reviewed commit is not an ancestor of the branch: it was rebased or rewritten"
328                .to_owned(),
329        ),
330    }
331    if prior.base_commit != base_now {
332        rec.reasons.push(format!(
333            "the base moved from {} to {}",
334            short(&prior.base_commit),
335            short(base_now)
336        ));
337    }
338    if rec.reasons.is_empty() {
339        rec.mode = Mode::VerifyRepair;
340    }
341    rec
342}
343
344/// Re-check a [`Mode::VerifyRepair`] record against the tree the review is
345/// about to look at (a base sync may have rebased it after the takeover was
346/// assessed). Returns true when it was downgraded. Pure.
347pub fn recheck(rec: &mut RecoveryRecord, head: &str) -> bool {
348    if rec.mode != Mode::VerifyRepair {
349        return false;
350    }
351    let Some(reviewed) = rec.reviewed_head.as_deref() else {
352        return false;
353    };
354    if reviewed == head {
355        return false;
356    }
357    rec.mode = Mode::FullReview;
358    rec.reasons.push(format!(
359        "the branch moved from {} to {} before the review started (rebased onto the base or \
360         changed); the earlier approval does not carry",
361        short(reviewed),
362        short(head)
363    ));
364    true
365}
366
367impl RecoveryRecord {
368    /// One line for the event log, `magi show` and the web.
369    pub fn summary(&self) -> String {
370        let from = crate::run::short_of(&self.from_run);
371        match self.mode {
372            Mode::VerifyRepair => format!(
373                "inherited from run {from}: reviewed commit, {} round(s), failed verification \
374                 `{}`; repair first, then a review scoped to the changes since",
375                self.rounds.len(),
376                self.evidence.as_ref().map_or("?", |e| e.command.as_str())
377            ),
378            Mode::FullReview => format!(
379                "full review after run {from}: {}",
380                if self.reasons.is_empty() {
381                    "no narrower recovery applied".to_owned()
382                } else {
383                    self.reasons.join("; ")
384                }
385            ),
386        }
387    }
388}
389
390#[cfg(test)]
391mod tests {
392    use super::*;
393    use crate::run::ReviewRecord;
394    use crate::verdict::Severity;
395
396    fn outcome(code: i32, tail: &str) -> CommandOutcome {
397        CommandOutcome {
398            command: "cargo test".to_owned(),
399            code: Some(code),
400            output_tail: tail.to_owned(),
401            duration_ms: 1,
402            resource_blocked: false,
403        }
404    }
405
406    fn seat(failed: bool, findings: Vec<Finding>) -> ReviewRecord {
407        ReviewRecord {
408            reviewer: 1,
409            agent: "a".to_owned(),
410            summary: String::new(),
411            findings,
412            vote: Some(ReviewVote::Approve),
413            failed: failed.then(|| "timed out".to_owned()),
414            duration_ms: 0,
415            attempts: 0,
416        }
417    }
418
419    fn round(head: &str, e2e: Vec<CommandOutcome>) -> ReviewRound {
420        ReviewRound {
421            round: 3,
422            head: head.to_owned(),
423            verified_head: Some(head.to_owned()),
424            verified_at: None,
425            reviews: vec![seat(false, Vec::new())],
426            e2e,
427            verify_retried: false,
428            e2e_deferred: false,
429            e2e_defer_reason: None,
430            fix: None,
431            blocking: 0,
432            answered: 1,
433            expected: 1,
434            clean: false,
435            progressed: false,
436            vote_split: false,
437            reconsideration: Vec::new(),
438            verdict: Some(ReviewVote::Approve),
439        }
440    }
441
442    fn prior(round: ReviewRound) -> RunState {
443        let mut s = RunState::new(
444            std::path::PathBuf::from("/tmp/x"),
445            "main".to_owned(),
446            "base1".to_owned(),
447            "task".to_owned(),
448            crate::config::Config::default(),
449        );
450        s.reviews.push(round);
451        s
452    }
453
454    fn failing() -> RunState {
455        prior(round("aaa111", vec![outcome(101, "1 failed: quiet_spawn")]))
456    }
457
458    #[test]
459    fn unchanged_commit_with_a_nonblocking_failed_verification_is_a_verify_repair() {
460        let r = assess("r1", Some(&failing()), "base1", Relation::Same);
461        assert_eq!(r.mode, Mode::VerifyRepair, "{:?}", r.reasons);
462        assert!(r.reasons.is_empty());
463        assert_eq!(r.generation, 1);
464        let e = r.evidence.unwrap();
465        assert_eq!(e.command, "cargo test");
466        assert_eq!(e.head, "aaa111");
467        assert!(e.output_tail.contains("quiet_spawn"));
468    }
469
470    #[test]
471    fn descendant_rebased_and_moved_base_widen_the_review() {
472        let p = failing();
473        let d = assess("r1", Some(&p), "base1", Relation::Descendant);
474        assert_eq!(d.mode, Mode::FullReview);
475        assert!(d.reasons[0].contains("never reviewed"));
476        let x = assess("r1", Some(&p), "base1", Relation::Diverged);
477        assert_eq!(x.mode, Mode::FullReview);
478        assert!(x.reasons[0].contains("rebased"));
479        let b = assess("r1", Some(&p), "base2", Relation::Same);
480        assert_eq!(b.mode, Mode::FullReview);
481        assert!(b.reasons[0].contains("base moved"));
482        // The evidence still travels as context.
483        assert!(b.evidence.is_some());
484    }
485
486    #[test]
487    fn missing_or_empty_evidence_falls_back_to_a_full_review() {
488        let none = prior(round("aaa111", Vec::new()));
489        let r = assess("r1", Some(&none), "base1", Relation::Same);
490        assert_eq!(r.mode, Mode::FullReview);
491        assert!(r.reasons[0].contains("no failed verification"));
492        let empty = prior(round("aaa111", vec![outcome(1, "  \n")]));
493        let r = assess("r1", Some(&empty), "base1", Relation::Same);
494        assert_eq!(r.mode, Mode::FullReview);
495        assert!(r.reasons[0].contains("no output"));
496        let r = assess("r1", None, "base1", Relation::Same);
497        assert_eq!(r.mode, Mode::FullReview);
498        assert!(r.reasons[0].contains("could not be read"));
499    }
500
501    #[test]
502    fn evidence_about_another_commit_is_not_reused() {
503        let mut p = failing();
504        p.reviews[0].verified_head = Some("bbb222".to_owned());
505        let r = assess("r1", Some(&p), "base1", Relation::Same);
506        assert_eq!(r.mode, Mode::FullReview);
507        assert!(r.reasons[0].contains("different commit"));
508    }
509
510    #[test]
511    fn a_timed_out_seat_or_a_blocking_finding_is_never_a_clean_panel() {
512        let mut p = failing();
513        p.reviews[0].answered = 0;
514        p.reviews[0].reviews[0] = seat(true, Vec::new());
515        let r = assess("r1", Some(&p), "base1", Relation::Same);
516        assert_eq!(r.mode, Mode::FullReview);
517        assert!(r.reasons[0].contains("did not answer"));
518
519        let mut p = failing();
520        p.reviews[0].blocking = 1;
521        p.reviews[0].reviews[0].findings = vec![Finding {
522            id: "R3-1-1".to_owned(),
523            severity: Severity::Major,
524            file: None,
525            line: None,
526            title: "t".to_owned(),
527            detail: String::new(),
528        }];
529        let r = assess("r1", Some(&p), "base1", Relation::Same);
530        assert_eq!(r.mode, Mode::FullReview);
531        assert_eq!(r.open_findings.len(), 1);
532    }
533
534    #[test]
535    fn generations_are_bounded() {
536        let mut p = failing();
537        let mut inherited = assess("r0", Some(&failing()), "base1", Relation::Same);
538        inherited.generation = MAX_GENERATION;
539        p.recovery = Some(inherited);
540        let r = assess("r1", Some(&p), "base1", Relation::Same);
541        assert_eq!(r.generation, MAX_GENERATION + 1);
542        assert_eq!(r.mode, Mode::FullReview);
543        assert!(r.reasons[0].contains("handed over"));
544    }
545
546    #[test]
547    fn a_gate_failure_counts_only_while_no_gate_fix_moved_the_tree() {
548        let mut p = prior(round("aaa111", Vec::new()));
549        p.gate_ran = true;
550        p.gate = vec![outcome(2, "gate red")];
551        let r = assess("r1", Some(&p), "base1", Relation::Same);
552        assert_eq!(r.mode, Mode::VerifyRepair);
553        assert_eq!(r.evidence.unwrap().source, Source::Gate);
554        p.gate_fixes.push(crate::run::GateFixRecord {
555            agent: "a".to_owned(),
556            failed: Vec::new(),
557            notes: String::new(),
558            committed: true,
559            error: None,
560        });
561        let r = assess("r1", Some(&p), "base1", Relation::Same);
562        assert_eq!(r.mode, Mode::FullReview);
563    }
564
565    #[test]
566    fn recheck_downgrades_a_moved_branch() {
567        let mut r = assess("r1", Some(&failing()), "base1", Relation::Same);
568        assert!(!recheck(&mut r, "aaa111"));
569        assert_eq!(r.mode, Mode::VerifyRepair);
570        assert!(recheck(&mut r, "ccc333"));
571        assert_eq!(r.mode, Mode::FullReview);
572        assert!(r.reasons[0].contains("moved"));
573    }
574
575    #[test]
576    fn tails_are_clipped_on_char_boundaries() {
577        let s = "あ".repeat(100);
578        assert!(tail_of(&s, 10).len() < 30);
579        assert!(head_of(&s, 10).starts_with('あ'));
580        assert_eq!(tail_of("ok", 10), "ok");
581    }
582}
magi/recovery.rs

magi/
recovery.rs
