magi/
recovery.rs

1//! Carrying an earlier attempt's evidence into the run that takes its branch over.
2//!
3//! A task whose previous run stopped on a failed verification used to reopen
4//! its branch as a context-free review-only run: the new reviewers and fixer
5//! never saw the failing command's output, the review decisions or the
6//! unresolved findings, and round 1 started from scratch.
7//!
8//! [`assess`] is pure. From the earlier run's record and the relation of the
9//! branch tip to the commit that run reviewed it decides between
10//!
11//! * [`Mode::VerifyRepair`]: the earlier panel found nothing blocking and only
12//!   verification failed, on exactly the commit that is about to be reviewed.
13//!   The fixer gets the failure first, then the review is scoped to what
14//!   changed since the reviewed commit.
15//! * [`Mode::FullReview`]: anything doubtful. The reviewers still see the
16//!   earlier context, but nothing is scoped and no approval is carried; each
17//!   [`RecoveryRecord::reasons`] entry says why.
18//!
19//! Inherited verification is evidence for a prompt, never a result: it is not
20//! copied into [`crate::run::ReviewRound`], so verification always runs again
21//! on the commit that is going to land.
22
23use serde::{Deserialize, Serialize};
24
25use crate::run::{CommandOutcome, E2eStatus, ReviewRound, RunState};
26
27/// Restore an interrupted repair and confirm its original commit before any review.
28pub(crate) async fn restore_repair(wt: &std::path::Path, before: &str) -> anyhow::Result<()> {
29    crate::git::restore_to(wt, before).await?;
30    let tip = crate::git::rev_parse(wt, "HEAD").await?;
31    anyhow::ensure!(tip == before, "the winner is at {tip}, not {before}");
32    Ok(())
33}
34
35fn short(sha: &str) -> &str {
36    sha.get(..8).unwrap_or(sha)
37}
38use crate::verdict::{Finding, ReviewVote};
39
40/// How many times evidence may be handed from one run to the next. Past this
41/// the successor starts as a plain full review.
42pub const MAX_GENERATION: u32 = 2;
43
44/// Longest output tail carried into a prompt, in bytes.
45pub const TAIL_MAX: usize = 6_000;
46
47/// Longest scoped diff shown to a reviewer, in bytes.
48pub const SINCE_DIFF_MAX: usize = 60_000;
49
50/// What the review after a takeover does with the earlier run.
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "kebab-case")]
53pub enum Mode {
54    /// One repair fix with the failure in hand, then a review scoped to the
55    /// changes since the reviewed commit.
56    VerifyRepair,
57    /// The whole patch is reviewed again; the earlier run is context only.
58    FullReview,
59}
60
61/// Which check failed.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "kebab-case")]
64pub enum Source {
65    /// The round's `verify.e2e`.
66    E2e,
67    /// The final `verify.gate`.
68    Gate,
69}
70
71/// The failed verification the earlier run stopped on.
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73pub struct Evidence {
74    /// Which check.
75    pub source: Source,
76    /// The command, as configured.
77    pub command: String,
78    /// Exit code, `None` on timeout or signal.
79    pub code: Option<i32>,
80    /// Scrubbed, clipped tail of its output.
81    pub output_tail: String,
82    /// The commit it was run against.
83    pub head: String,
84}
85
86/// One earlier review round, reduced to its decision.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88pub struct PriorRound {
89    /// Round number in the earlier run.
90    pub round: usize,
91    /// The commit it reviewed.
92    pub head: String,
93    /// Findings that held the merge.
94    pub blocking: usize,
95    /// Seats that answered, of `expected`.
96    pub answered: usize,
97    /// Seats the round expected.
98    pub expected: usize,
99    /// The round's verdict, when a vote was usable.
100    pub verdict: Option<ReviewVote>,
101}
102
103/// Where the one repair fix stands. Saved before the fixer is called, so a
104/// resume never repairs twice.
105#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107pub enum Repair {
108    /// Not attempted (yet, or not applicable).
109    NotStarted,
110    /// The fixer was called from this commit and has not been settled.
111    Started {
112        /// HEAD before the call.
113        before: String,
114    },
115    /// The seat could not start. Restore under coordination before retrying.
116    WaitingForRestore {
117        /// HEAD before the blocked attempt (or its earlier fallback).
118        before: String,
119    },
120    /// Settled.
121    Done {
122        /// HEAD before the call.
123        before: String,
124        /// HEAD after it.
125        after: String,
126        /// The fixer's own commit or rescue commit moved HEAD.
127        committed: bool,
128        /// Why nothing was kept (the worktree was restored to `before`).
129        failed: Option<String>,
130    },
131}
132
133/// What a run inherited from the attempt it took over, and why.
134#[derive(Debug, Clone, Serialize, Deserialize)]
135pub struct RecoveryRecord {
136    /// The earlier run.
137    pub from_run: String,
138    /// How many hand-overs of evidence lie behind this one.
139    pub generation: u32,
140    /// The decision.
141    pub mode: Mode,
142    /// For [`Mode::FullReview`]: every reason a narrower recovery was not
143    /// justified. Empty for [`Mode::VerifyRepair`].
144    #[serde(default)]
145    pub reasons: Vec<String>,
146    /// The earlier run's last reviewed commit, when it had one.
147    #[serde(default)]
148    pub reviewed_head: Option<String>,
149    /// The earlier rounds' decisions.
150    #[serde(default)]
151    pub rounds: Vec<PriorRound>,
152    /// Findings the last earlier round left open.
153    #[serde(default)]
154    pub open_findings: Vec<Finding>,
155    /// The failed verification, when one was recorded.
156    #[serde(default)]
157    pub evidence: Option<Evidence>,
158    /// The task's own statement replaced the commit-log stand-in.
159    #[serde(default)]
160    pub criteria_inherited: bool,
161    /// The repair fix.
162    #[serde(default = "not_started")]
163    pub repair: Repair,
164}
165
166fn not_started() -> Repair {
167    Repair::NotStarted
168}
169
170/// How the branch tip relates to the commit the earlier run reviewed.
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Relation {
173    /// The same commit.
174    Same,
175    /// The reviewed commit is a proper ancestor: commits were added after it.
176    Descendant,
177    /// Not an ancestor: rebased, rewritten or unknown to this repository.
178    Diverged,
179}
180
181/// Clip `text` to its last `max` bytes on a char boundary.
182pub fn tail_of(text: &str, max: usize) -> String {
183    if text.len() <= max {
184        return text.to_owned();
185    }
186    let mut start = text.len() - max;
187    while !text.is_char_boundary(start) {
188        start += 1;
189    }
190    format!("[...]\n{}", &text[start..])
191}
192
193/// Clip `text` to its first `max` bytes on a char boundary.
194pub fn head_of(text: &str, max: usize) -> String {
195    if text.len() <= max {
196        return text.to_owned();
197    }
198    let mut end = max;
199    while !text.is_char_boundary(end) {
200        end -= 1;
201    }
202    format!("{}\n[... truncated]", &text[..end])
203}
204
205fn evidence_from(round: &ReviewRound, prior: &RunState) -> Option<Evidence> {
206    let scrub =
207        |s: &str| crate::scrub::scrub(&tail_of(s, TAIL_MAX), &crate::scrub::Identity::current());
208    let failed = |o: &&CommandOutcome| !o.ok() && !o.resource_blocked;
209    if round.e2e_status() == E2eStatus::Failed
210        && let (Some(o), Some(head)) = (round.e2e.iter().find(failed), &round.verified_head)
211    {
212        return Some(Evidence {
213            source: Source::E2e,
214            command: o.command.clone(),
215            code: o.code,
216            output_tail: scrub(&o.output_tail),
217            head: head.clone(),
218        });
219    }
220    // The gate runs on the tree the last round left; it is only evidence about
221    // that round's head while no gate fix moved the tree afterwards.
222    if prior.gate_ran
223        && prior.gate_fixes.is_empty()
224        && let Some(o) = prior.gate.iter().find(failed)
225    {
226        return Some(Evidence {
227            source: Source::Gate,
228            command: o.command.clone(),
229            code: o.code,
230            output_tail: scrub(&o.output_tail),
231            head: round.head.clone(),
232        });
233    }
234    None
235}
236
237/// The record for a run that takes `prior` over. Pure.
238///
239/// `prior` is `None` when the earlier attempt's record could not be read.
240/// `base_now` is the base commit the new run reviews against and `relation`
241/// how the branch tip stands to the earlier run's last reviewed commit.
242pub fn assess(
243    from_run: &str,
244    prior: Option<&RunState>,
245    base_now: &str,
246    relation: Relation,
247) -> RecoveryRecord {
248    let mut rec = RecoveryRecord {
249        from_run: from_run.to_owned(),
250        generation: 0,
251        mode: Mode::FullReview,
252        reasons: Vec::new(),
253        reviewed_head: None,
254        rounds: Vec::new(),
255        open_findings: Vec::new(),
256        evidence: None,
257        criteria_inherited: false,
258        repair: Repair::NotStarted,
259    };
260    let Some(prior) = prior else {
261        rec.reasons
262            .push("the earlier run's record could not be read".to_owned());
263        return rec;
264    };
265    rec.generation = prior.recovery.as_ref().map_or(0, |r| r.generation) + 1;
266    rec.rounds = prior
267        .reviews
268        .iter()
269        .map(|r| PriorRound {
270            round: r.round,
271            head: r.head.clone(),
272            blocking: r.blocking,
273            answered: r.answered,
274            expected: r.expected,
275            verdict: r.verdict,
276        })
277        .collect();
278    let Some(last) = prior.reviews.last() else {
279        rec.reasons
280            .push("the earlier run never completed a review round".to_owned());
281        return rec;
282    };
283    rec.reviewed_head = Some(last.head.clone());
284    rec.open_findings = last
285        .reviews
286        .iter()
287        .flat_map(|r| r.findings.iter().cloned())
288        .collect();
289    rec.evidence = evidence_from(last, prior);
290
291    if rec.generation > MAX_GENERATION {
292        rec.reasons.push(format!(
293            "evidence has already been handed over {MAX_GENERATION} times"
294        ));
295    }
296    if last.answered < last.expected || last.reviews.iter().any(|r| r.failed.is_some()) {
297        rec.reasons.push(format!(
298            "only {} of {} reviewer seat(s) answered the last round; a panel that did not \
299             answer is not a clean one",
300            last.answered, last.expected
301        ));
302    }
303    if last.blocking > 0 {
304        rec.reasons.push(format!(
305            "the last round still had {} blocking finding(s)",
306            last.blocking
307        ));
308    }
309    match &rec.evidence {
310        None => rec
311            .reasons
312            .push("no failed verification with a known command was recorded".to_owned()),
313        Some(e) if e.output_tail.trim().is_empty() => rec
314            .reasons
315            .push("the failed verification kept no output to act on".to_owned()),
316        Some(e) if e.head != last.head => {
317            let sync = prior.base_sync.as_ref().map_or_else(
318                || "no base sync recorded".to_owned(),
319                |sync| {
320                    format!(
321                        "base sync to {}, {} rebase attempt(s), conflict: {}",
322                        sync.tip,
323                        sync.attempts,
324                        sync.conflict.as_deref().unwrap_or("none")
325                    )
326                },
327            );
328            rec.reasons.push(format!(
329                "the failed verification was about a different commit {} than the one reviewed {}; \
330                 {sync}; additional review is required before approval can carry forward",
331                e.head, last.head,
332            ));
333        }
334        Some(_) => {}
335    }
336    match relation {
337        Relation::Same => {}
338        Relation::Descendant => rec.reasons.push(
339            "commits were added after the reviewed commit; they were never reviewed".to_owned(),
340        ),
341        Relation::Diverged => rec.reasons.push(
342            "the reviewed commit is not an ancestor of the branch: it was rebased or rewritten"
343                .to_owned(),
344        ),
345    }
346    if prior.base_commit != base_now {
347        rec.reasons.push(format!(
348            "the base moved from {} to {}",
349            short(&prior.base_commit),
350            short(base_now)
351        ));
352    }
353    if rec.reasons.is_empty() {
354        rec.mode = Mode::VerifyRepair;
355    }
356    rec
357}
358
359/// Re-check a [`Mode::VerifyRepair`] record against the tree the review is
360/// about to look at (a base sync may have rebased it after the takeover was
361/// assessed). Returns true when it was downgraded. Pure.
362pub fn recheck(rec: &mut RecoveryRecord, head: &str) -> bool {
363    if rec.mode != Mode::VerifyRepair {
364        return false;
365    }
366    let Some(reviewed) = rec.reviewed_head.as_deref() else {
367        return false;
368    };
369    if reviewed == head {
370        return false;
371    }
372    rec.mode = Mode::FullReview;
373    rec.reasons.push(format!(
374        "the branch moved from {} to {} before the review started (rebased onto the base or \
375         changed); the earlier approval does not carry",
376        short(reviewed),
377        short(head)
378    ));
379    true
380}
381
382impl RecoveryRecord {
383    /// One line for the event log, `magi show` and the web.
384    pub fn summary(&self) -> String {
385        let from = crate::run::short_of(&self.from_run);
386        match self.mode {
387            Mode::VerifyRepair => format!(
388                "inherited from run {from}: reviewed commit, {} round(s), failed verification \
389                 `{}`; repair first, then a review scoped to the changes since",
390                self.rounds.len(),
391                self.evidence.as_ref().map_or("?", |e| e.command.as_str())
392            ),
393            Mode::FullReview => format!(
394                "full review after run {from}: {}",
395                if self.reasons.is_empty() {
396                    "no narrower recovery applied".to_owned()
397                } else {
398                    self.reasons.join("; ")
399                }
400            ),
401        }
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408    use crate::run::ReviewRecord;
409    use crate::verdict::Severity;
410
411    fn outcome(code: i32, tail: &str) -> CommandOutcome {
412        CommandOutcome {
413            command: "cargo test".to_owned(),
414            code: Some(code),
415            output_tail: tail.to_owned(),
416            duration_ms: 1,
417            resource_blocked: false,
418        }
419    }
420
421    fn seat(failed: bool, findings: Vec<Finding>) -> ReviewRecord {
422        ReviewRecord {
423            reviewer: 1,
424            agent: "a".to_owned(),
425            summary: String::new(),
426            findings,
427            vote: Some(ReviewVote::Approve),
428            failed: failed.then(|| "timed out".to_owned()),
429            duration_ms: 0,
430            attempts: 0,
431        }
432    }
433
434    fn round(head: &str, e2e: Vec<CommandOutcome>) -> ReviewRound {
435        ReviewRound {
436            round: 3,
437            head: head.to_owned(),
438            verified_head: Some(head.to_owned()),
439            verified_at: None,
440            reviews: vec![seat(false, Vec::new())],
441            e2e,
442            verify_retried: false,
443            e2e_deferred: false,
444            e2e_defer_reason: None,
445            fix: None,
446            blocking: 0,
447            answered: 1,
448            expected: 1,
449            clean: false,
450            progressed: false,
451            vote_split: false,
452            reconsideration: Vec::new(),
453            verdict: Some(ReviewVote::Approve),
454        }
455    }
456
457    fn prior(round: ReviewRound) -> RunState {
458        let mut s = RunState::new(
459            std::path::PathBuf::from("/tmp/x"),
460            "main".to_owned(),
461            "base1".to_owned(),
462            "task".to_owned(),
463            crate::config::Config::default(),
464        );
465        s.reviews.push(round);
466        s
467    }
468
469    fn failing() -> RunState {
470        prior(round("aaa111", vec![outcome(101, "1 failed: quiet_spawn")]))
471    }
472
473    #[test]
474    fn unchanged_commit_with_a_nonblocking_failed_verification_is_a_verify_repair() {
475        let r = assess("r1", Some(&failing()), "base1", Relation::Same);
476        assert_eq!(r.mode, Mode::VerifyRepair, "{:?}", r.reasons);
477        assert!(r.reasons.is_empty());
478        assert_eq!(r.generation, 1);
479        let e = r.evidence.unwrap();
480        assert_eq!(e.command, "cargo test");
481        assert_eq!(e.head, "aaa111");
482        assert!(e.output_tail.contains("quiet_spawn"));
483    }
484
485    #[test]
486    fn descendant_rebased_and_moved_base_widen_the_review() {
487        let p = failing();
488        let d = assess("r1", Some(&p), "base1", Relation::Descendant);
489        assert_eq!(d.mode, Mode::FullReview);
490        assert!(d.reasons[0].contains("never reviewed"));
491        let x = assess("r1", Some(&p), "base1", Relation::Diverged);
492        assert_eq!(x.mode, Mode::FullReview);
493        assert!(x.reasons[0].contains("rebased"));
494        let b = assess("r1", Some(&p), "base2", Relation::Same);
495        assert_eq!(b.mode, Mode::FullReview);
496        assert!(b.reasons[0].contains("base moved"));
497        // The evidence still travels as context.
498        assert!(b.evidence.is_some());
499    }
500
501    #[test]
502    fn missing_or_empty_evidence_falls_back_to_a_full_review() {
503        let none = prior(round("aaa111", Vec::new()));
504        let r = assess("r1", Some(&none), "base1", Relation::Same);
505        assert_eq!(r.mode, Mode::FullReview);
506        assert!(r.reasons[0].contains("no failed verification"));
507        let empty = prior(round("aaa111", vec![outcome(1, "  \n")]));
508        let r = assess("r1", Some(&empty), "base1", Relation::Same);
509        assert_eq!(r.mode, Mode::FullReview);
510        assert!(r.reasons[0].contains("no output"));
511        let r = assess("r1", None, "base1", Relation::Same);
512        assert_eq!(r.mode, Mode::FullReview);
513        assert!(r.reasons[0].contains("could not be read"));
514    }
515
516    #[test]
517    fn evidence_about_another_commit_is_not_reused() {
518        let mut p = failing();
519        p.reviews[0].verified_head = Some("bbb222".to_owned());
520        let r = assess("r1", Some(&p), "base1", Relation::Same);
521        assert_eq!(r.mode, Mode::FullReview);
522        assert!(r.reasons[0].contains("different commit"));
523    }
524
525    #[test]
526    fn a_timed_out_seat_or_a_blocking_finding_is_never_a_clean_panel() {
527        let mut p = failing();
528        p.reviews[0].answered = 0;
529        p.reviews[0].reviews[0] = seat(true, Vec::new());
530        let r = assess("r1", Some(&p), "base1", Relation::Same);
531        assert_eq!(r.mode, Mode::FullReview);
532        assert!(r.reasons[0].contains("did not answer"));
533
534        let mut p = failing();
535        p.reviews[0].blocking = 1;
536        p.reviews[0].reviews[0].findings = vec![Finding {
537            id: "R3-1-1".to_owned(),
538            severity: Severity::Major,
539            file: None,
540            line: None,
541            title: "t".to_owned(),
542            detail: String::new(),
543        }];
544        let r = assess("r1", Some(&p), "base1", Relation::Same);
545        assert_eq!(r.mode, Mode::FullReview);
546        assert_eq!(r.open_findings.len(), 1);
547    }
548
549    #[test]
550    fn generations_are_bounded() {
551        let mut p = failing();
552        let mut inherited = assess("r0", Some(&failing()), "base1", Relation::Same);
553        inherited.generation = MAX_GENERATION;
554        p.recovery = Some(inherited);
555        let r = assess("r1", Some(&p), "base1", Relation::Same);
556        assert_eq!(r.generation, MAX_GENERATION + 1);
557        assert_eq!(r.mode, Mode::FullReview);
558        assert!(r.reasons[0].contains("handed over"));
559    }
560
561    #[test]
562    fn a_gate_failure_counts_only_while_no_gate_fix_moved_the_tree() {
563        let mut p = prior(round("aaa111", Vec::new()));
564        p.gate_ran = true;
565        p.gate = vec![outcome(2, "gate red")];
566        let r = assess("r1", Some(&p), "base1", Relation::Same);
567        assert_eq!(r.mode, Mode::VerifyRepair);
568        assert_eq!(r.evidence.unwrap().source, Source::Gate);
569        p.gate_fixes.push(crate::run::GateFixRecord {
570            agent: "a".to_owned(),
571            failed: Vec::new(),
572            notes: String::new(),
573            committed: true,
574            error: None,
575        });
576        let r = assess("r1", Some(&p), "base1", Relation::Same);
577        assert_eq!(r.mode, Mode::FullReview);
578    }
579
580    #[test]
581    fn recheck_downgrades_a_moved_branch() {
582        let mut r = assess("r1", Some(&failing()), "base1", Relation::Same);
583        assert!(!recheck(&mut r, "aaa111"));
584        assert_eq!(r.mode, Mode::VerifyRepair);
585        assert!(recheck(&mut r, "ccc333"));
586        assert_eq!(r.mode, Mode::FullReview);
587        assert!(r.reasons[0].contains("moved"));
588    }
589
590    #[test]
591    fn tails_are_clipped_on_char_boundaries() {
592        let s = "あ".repeat(100);
593        assert!(tail_of(&s, 10).len() < 30);
594        assert!(head_of(&s, 10).starts_with('あ'));
595        assert_eq!(tail_of("ok", 10), "ok");
596    }
597}
magi/recovery.rs

magi/
recovery.rs
