Skip to main content

magi/
base_sync_recovery.rs

1//! Durable bounded synchronization intervals and single-use owner recovery approvals.
2use anyhow::{Result, ensure};
3use jiff::Timestamp;
4use serde::{Deserialize, Serialize};
5
6use crate::run::{CommandOutcome, ReviewRound, RunState, RunStatus};
7
8#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9#[serde(rename_all = "snake_case")]
10/// Why synchronization stopped. Only exhaustion admits a new recovery interval.
11pub enum StopKind {
12    /// The ordinary moving-base budget was spent.
13    Exhausted,
14    /// Replaying the retained changes failed.
15    Conflict,
16    /// A remote ownership or publication operation failed.
17    Publication,
18    /// The remote branch contains foreign work.
19    Divergence,
20    /// The change was integrated, but closing its PR failed and may be retried.
21    ClosingIntegratedPr,
22    /// Legacy or unexpected evidence cannot justify recovery.
23    Unknown,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27/// Stable identity of a stopped synchronization.
28pub struct Stop {
29    /// Typed cause.
30    pub kind: StopKind,
31    /// Retained candidate commit.
32    pub head: String,
33    /// Base tip observed at the stop.
34    pub tip: String,
35    /// Cumulative attempts when stopped.
36    pub attempts: usize,
37    /// Original diagnostic, retained verbatim.
38    pub detail: String,
39    /// When this record was created.
40    pub at: Timestamp,
41    /// A synchronized HEAD had not reached its gate before the base advanced.
42    #[serde(default)]
43    pub unverified_sync: bool,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47/// One explicit owner request, consumed durably before rebasing.
48pub struct Authorization {
49    #[serde(default)]
50    pub scope: Option<Scope>,
51    /// Older approvals authorized one attempt only.
52    #[serde(default)]
53    pub interval: bool,
54    #[serde(default)]
55    pub invalidated: Option<String>,
56    /// Task whose claim admitted the request.
57    pub task: String,
58    /// Exact saved execution authorized to recover.
59    pub run: String,
60    /// Immutable stop authorized by this request.
61    pub stop: Stop,
62    /// When this record was created.
63    pub at: Timestamp,
64    /// Persisted before any Git mutation. An interrupted consumption grants no retry.
65    pub consumed: bool,
66    /// Queue approval generation, absent for an explicit recovery command.
67    #[serde(default)]
68    pub generation: Option<u64>,
69    /// Historical evidence stays attached to its original heads.
70    pub reviews: Vec<ReviewRound>,
71    /// Prior gate outcomes, never evidence for the recovered head.
72    pub gate: Vec<CommandOutcome>,
73    /// Head targeted by the archived gate.
74    pub gate_head: Option<String>,
75    /// Whether the archived gate was attempted.
76    pub gate_ran: bool,
77}
78
79impl Authorization {
80    /// Consume a refused approval without granting recovery, retaining the reason.
81    pub(crate) fn refuse(&mut self, reason: impl Into<String>) {
82        self.invalidated = Some(reason.into());
83        self.consumed = true;
84    }
85}
86
87/// A bounded active interval. Lifetime attempts remain in `BaseSync`.
88#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct Episode {
90    /// Monotonic interval identity.
91    pub id: usize,
92    /// Why this interval was opened.
93    pub reason: String,
94    /// Reservations, including interrupted operations.
95    pub used: usize,
96    /// Last successful synchronization or actually executed verification in this interval.
97    pub synced_head: Option<String>,
98    /// Last successful synchronization, distinct from verification progress.
99    #[serde(default)]
100    pub last_sync_head: Option<String>,
101    /// An interruption after progress; consumed at most once.
102    pub boundary: Option<Boundary>,
103    /// Earlier interruption evidence superseded by actual progress.
104    #[serde(default)]
105    pub prior_boundaries: Vec<Boundary>,
106}
107
108/// Durable interruption evidence, never inferred from elapsed time.
109#[derive(Debug, Clone, Serialize, Deserialize)]
110pub struct Boundary {
111    pub pass: usize,
112    pub at: Timestamp,
113    pub head: String,
114    pub consumed: bool,
115}
116
117/// Write-ahead reservation. A restart continues this exact operation.
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct Operation {
120    pub id: usize,
121    pub episode: usize,
122    pub head: String,
123    pub tip: String,
124    pub finished: bool,
125    /// Exact result accepted by the rebase validator, saved before scratch cleanup.
126    #[serde(default)]
127    pub validated_head: Option<String>,
128}
129
130/// Consumed owner approval for an infrastructure boundary.
131#[derive(Debug, Clone, Serialize, Deserialize)]
132pub struct Renewal {
133    pub task: String,
134    pub run: String,
135    pub generation: u64,
136    pub instruction: String,
137    pub episode: usize,
138    pub boundary_pass: usize,
139    pub head: String,
140    pub tip: String,
141}
142
143/// Approval scope additional to the immutable stop.
144#[derive(Debug, Clone, Serialize, Deserialize)]
145pub struct Scope {
146    pub tip: String,
147    pub instruction: String,
148    pub episode: usize,
149}
150
151#[derive(Debug, Clone, Default, Serialize, Deserialize)]
152/// Persistent stop and authorization history, additive for legacy runs.
153pub struct Recovery {
154    /// Append-only interval and operation evidence, absent in old records.
155    #[serde(default)]
156    pub episodes: Vec<Episode>,
157    #[serde(default)]
158    pub operations: Vec<Operation>,
159    /// Stops remain evidence after successful recovery.
160    #[serde(default)]
161    pub stops: Vec<Stop>,
162    #[serde(default)]
163    pub renewals: Vec<Renewal>,
164    /// Current stop, if any.
165    pub stop: Option<Stop>,
166    /// Requests and archived evidence in authorization order.
167    pub authorizations: Vec<Authorization>,
168}
169
170impl RunState {
171    /// Shared CLI, notice and web instruction, based on the typed checkpoint.
172    pub fn base_sync_next_action(&self) -> String {
173        let episode = self.base_sync_recovery.episodes.last().map_or(0, |e| e.id);
174        let lifetime = self.base_sync.as_ref().map_or(0, |s| s.attempts);
175        format!(
176            "{} ({}/{} in episode {}, {} lifetime synchronizations)",
177            self.base_sync_action(),
178            self.base_episode_used(),
179            crate::graph::BASE_SYNC_ROUNDS,
180            episode,
181            lifetime
182        )
183    }
184
185    fn base_sync_action(&self) -> String {
186        if let Some(reason) = self
187            .base_sync_recovery
188            .authorizations
189            .last()
190            .and_then(|a| a.invalidated.as_ref())
191        {
192            return format!("Approval invalidated: {reason}");
193        }
194        if self.resource_block.is_some() {
195            return "Restore the required resource, then explicitly resume the retained run; automatic retries retain the synchronization budget".into();
196        }
197        if self.base_sync_recovery.stop.is_none()
198            && let Some(op) = self
199                .base_sync_recovery
200                .operations
201                .last()
202                .filter(|o| !o.finished)
203        {
204            return format!(
205                "Resume reserved synchronization {} against base {}; inspect retained Git state if reconciliation is refused",
206                op.id, op.tip
207            );
208        }
209        match self.base_sync_recovery.stop.as_ref().map(|s| &s.kind) {
210            Some(StopKind::Exhausted) => {
211                let why = if self.base_sync_recovery.stop.as_ref().is_some_and(|s| s.unverified_sync) {
212                    "The synchronized HEAD never reached its gate before the base moved again. "
213                } else {
214                    ""
215                };
216                if self.base_recovery_pending() {
217                    format!("{why}Approval recorded; resume the retained run to open the approved bounded synchronization interval")
218                } else {
219                    format!("{why}Authorize a new bounded synchronization interval with `magi run --resume {} --recover-base`, or a new owner resumption approval", self.id)
220                }
221            },
222            Some(StopKind::Conflict) => {
223                let stop = self.base_sync_recovery.stop.as_ref().unwrap();
224                let task = self.origin.as_ref().and_then(|origin| origin.task.as_deref()).unwrap_or("<task>");
225                format!("Authorize a narrow history-preserving repair with `magi task grant-repair {task} --run {} --kind base-conflict --rounds 1 --head {} --tip {} --expect-pass {} --request <unique-id> --reason <reason> --instruction <approved-resolution>`; retain the original review criteria. Base-budget recovery cannot resolve a textual conflict", self.id, stop.head, stop.tip, self.passes.last().map_or(0, |p| p.index))
226            },
227            Some(StopKind::Divergence) => "Reconcile foreign remote branch commits before resuming; do not overwrite them".into(),
228            Some(StopKind::Publication) => "Inspect the retained publication failure and remote lease before resuming".into(),
229            Some(StopKind::ClosingIntegratedPr) => "Resume to retry closing the already integrated pull request".into(),
230            Some(StopKind::Unknown) => "Inspect the retained synchronization state before resuming; unknown evidence grants no new budget".into(),
231            None => "Continue verification on the current synchronized HEAD".into(),
232        }
233    }
234
235    pub(crate) fn ensure_base_episode(&mut self) {
236        if let Some(stop) = self.base_sync_recovery.stop.clone()
237            && !self.base_sync_recovery.stops.contains(&stop)
238        {
239            self.base_sync_recovery.stops.push(stop);
240        }
241        if self.base_sync_recovery.episodes.is_empty() {
242            let used = self.base_sync.as_ref().map_or(0, |s| s.attempts);
243            self.base_sync_recovery.episodes.push(Episode {
244                id: 0,
245                reason: "initial or conservative legacy migration".into(),
246                used,
247                synced_head: None,
248                last_sync_head: None,
249                boundary: None,
250                prior_boundaries: Vec::new(),
251            });
252        }
253    }
254
255    fn open_base_episode(&mut self, reason: &str) {
256        self.ensure_base_episode();
257        let id = self.base_sync_recovery.episodes.len();
258        self.base_sync_recovery.episodes.push(Episode {
259            id,
260            reason: reason.into(),
261            used: 0,
262            synced_head: None,
263            last_sync_head: None,
264            boundary: None,
265            prior_boundaries: Vec::new(),
266        });
267    }
268
269    pub(crate) fn base_episode_used(&self) -> usize {
270        self.base_sync_recovery.episodes.last().map_or_else(
271            || self.base_sync.as_ref().map_or(0, |s| s.attempts),
272            |e| e.used,
273        )
274    }
275
276    pub(crate) fn record_base_progress(&mut self, head: &str) {
277        self.ensure_base_episode();
278        let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
279        if let Some(boundary) = episode.boundary.take() {
280            episode.prior_boundaries.push(boundary);
281        }
282        episode.synced_head = Some(head.into());
283    }
284
285    /// Called by the same atomic state save that retains the resource stop.
286    pub(crate) fn record_base_interruption(&mut self) {
287        let Some(block) = &self.resource_block else {
288            return;
289        };
290        if matches!(block.cause, crate::run::ResourceBlockCause::Unrecorded) {
291            return;
292        }
293        let Some(episode) = self.base_sync_recovery.episodes.last_mut() else {
294            return;
295        };
296        let Some(head) = episode.synced_head.clone() else {
297            return;
298        };
299        if episode.boundary.is_none() {
300            episode.boundary = Some(Boundary {
301                pass: self.passes.last().map_or(0, |p| p.index),
302                at: block.at,
303                head,
304                consumed: false,
305            });
306        }
307    }
308
309    pub(crate) fn renew_base_episode(
310        &mut self,
311        task: &str,
312        approval: &crate::queue::Resumption,
313        head: &str,
314        tip: &str,
315    ) -> bool {
316        if self
317            .base_sync_recovery
318            .renewals
319            .iter()
320            .any(|r| r.task == task && r.generation >= approval.generation)
321            || self
322                .base_sync_recovery
323                .authorizations
324                .iter()
325                .any(|a| a.task == task && a.generation.is_some_and(|g| g >= approval.generation))
326        {
327            return false;
328        }
329        let Some(e) = self.base_sync_recovery.episodes.last_mut() else {
330            return false;
331        };
332        let Some(b) = &mut e.boundary else {
333            return false;
334        };
335        if b.consumed || b.head != head || approval.at <= b.at {
336            return false;
337        }
338        b.consumed = true;
339        self.base_sync_recovery.renewals.push(Renewal {
340            task: task.into(),
341            run: self.id.clone(),
342            generation: approval.generation,
343            instruction: approval.instruction.clone(),
344            episode: e.id,
345            boundary_pass: b.pass,
346            head: head.into(),
347            tip: tip.into(),
348        });
349        self.open_base_episode("owner resumption after recorded infrastructure interruption");
350        true
351    }
352
353    /// Reserve once before Git; retrying an unfinished operation spends nothing more.
354    pub(crate) fn reserve_base_operation(&mut self, head: &str, tip: &str) {
355        self.ensure_base_episode();
356        if self
357            .base_sync_recovery
358            .operations
359            .last()
360            .is_some_and(|o| !o.finished)
361        {
362            return;
363        }
364        let e = self.base_sync_recovery.episodes.last_mut().unwrap();
365        e.used += 1;
366        let id = self.base_sync_recovery.operations.len();
367        self.base_sync_recovery.operations.push(Operation {
368            id,
369            episode: e.id,
370            head: head.into(),
371            tip: tip.into(),
372            finished: false,
373            validated_head: None,
374        });
375        if let Some(sync) = &mut self.base_sync {
376            sync.attempts += 1;
377        }
378    }
379
380    pub(crate) fn finish_base_operation(&mut self, head: Option<&str>) {
381        if let Some(op) = self.base_sync_recovery.operations.last_mut() {
382            op.finished = true;
383        }
384        if let Some(head) = head {
385            let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
386            if let Some(boundary) = episode.boundary.take() {
387                episode.prior_boundaries.push(boundary);
388            }
389            episode.synced_head = Some(head.into());
390            episode.last_sync_head = Some(head.into());
391            self.reconcile_verification_head(head);
392        }
393    }
394
395    pub(crate) fn base_sync_blocked(&self) -> bool {
396        self.base_sync
397            .as_ref()
398            .is_some_and(|s| s.conflict.is_some())
399    }
400
401    /// This checkpoint only rechecks integration and retries PR closure. It
402    /// never clears the conflict guard or grants an additional rebase.
403    pub(crate) fn base_sync_closure_pending(&self) -> bool {
404        let Some(detail) = self
405            .base_sync
406            .as_ref()
407            .and_then(|sync| sync.conflict.as_ref())
408        else {
409            return false;
410        };
411        let Some(winner) = self.winner() else {
412            return false;
413        };
414        // Also recognizes the diagnostic written before typed stops existed.
415        let prefix = format!(
416            "{} is already in {}/{}, but its pull request could not be closed (",
417            winner.branch, self.config.merge.remote, self.base_branch
418        );
419        detail.starts_with(&prefix) && detail.ends_with("); resume to retry")
420    }
421
422    /// Legacy exhaustion is recognized only by the exact message and budget.
423    /// Once recorded, its identity remains stable across no-progress passes.
424    pub(crate) fn record_base_sync_stop(&mut self, head: &str) {
425        let Some(sync) = self.base_sync.as_ref() else {
426            return;
427        };
428        let Some(detail) = sync.conflict.clone() else {
429            return;
430        };
431        if self
432            .base_sync_recovery
433            .stop
434            .as_ref()
435            .is_some_and(|s| s.head == head && s.tip == sync.tip && s.detail == detail)
436        {
437            self.status = RunStatus::Blocked;
438            return;
439        }
440        let exhausted = self.winner().is_some_and(|w| {
441            [
442                ("rebase(s)", "rebasing"),
443                ("synchronization(s)", "synchronizing"),
444            ].iter().any(|(noun, verb)| detail == format!(
445                "{} moved {} commit(s) ahead of {} after {} {}; {} again would only race it",
446                self.base_branch, sync.behind, w.branch, crate::graph::BASE_SYNC_ROUNDS, noun, verb
447            ))
448        });
449        let kind = if self.base_sync_closure_pending() {
450            StopKind::ClosingIntegratedPr
451        } else if sync.attempts >= crate::graph::BASE_SYNC_ROUNDS && exhausted {
452            StopKind::Exhausted
453        } else if detail.contains("not rebasing over") {
454            StopKind::Divergence
455        } else if detail.contains("refused the push")
456            || detail.contains("could not push")
457            || detail.contains("could not be pushed")
458        {
459            StopKind::Publication
460        } else if detail.contains("conflicts with") && detail.contains("did not rebase") {
461            StopKind::Conflict
462        } else {
463            StopKind::Unknown
464        };
465        let at = self
466            .events
467            .iter()
468            .rev()
469            .find(|event| event.node == "land" && event.message == detail)
470            .map_or(self.updated_at, |event| event.at);
471        let unverified_sync = sync.behind > 0
472            && self
473                .base_sync_recovery
474                .episodes
475                .last()
476                .is_some_and(|e| e.last_sync_head.as_deref() == Some(head))
477            && !(self.gate_ran && self.gate_head.as_deref() == Some(head));
478        let stop = Stop {
479            unverified_sync,
480            kind,
481            head: head.to_owned(),
482            tip: sync.tip.clone(),
483            attempts: sync.attempts,
484            detail,
485            at,
486        };
487        self.base_sync_recovery.stops.push(stop.clone());
488        self.base_sync_recovery.stop = Some(stop);
489        self.status = RunStatus::Blocked;
490    }
491
492    pub(crate) fn base_recovery_pending(&self) -> bool {
493        self.base_sync_recovery
494            .authorizations
495            .last()
496            .is_some_and(|a| {
497                !a.consumed
498                    && a.invalidated.is_none()
499                    && a.run == self.id
500                    && self.base_sync_recovery.stop.as_ref() == Some(&a.stop)
501            })
502    }
503
504    pub(crate) fn authorize_base_sync(&mut self, task: &str) -> Result<()> {
505        let stop = self
506            .base_sync_recovery
507            .stop
508            .clone()
509            .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
510        ensure!(
511            stop.kind == StopKind::Exhausted,
512            "only exhausted base movement permits recovery"
513        );
514        ensure!(
515            !self
516                .base_sync_recovery
517                .authorizations
518                .last()
519                .is_some_and(|a| !a.consumed && a.stop == stop),
520            "this stop already has a pending authorization; resume the existing request"
521        );
522        self.base_sync_recovery.authorizations.push(Authorization {
523            scope: None,
524            interval: true,
525            invalidated: None,
526            task: task.to_owned(),
527            run: self.id.clone(),
528            stop,
529            at: Timestamp::now(),
530            consumed: false,
531            generation: None,
532            reviews: Vec::new(),
533            gate: Vec::new(),
534            gate_head: None,
535            gate_ran: false,
536        });
537        Ok(())
538    }
539
540    pub(crate) fn authorize_base_sync_resumption(
541        &mut self,
542        task: &str,
543        approval: &crate::queue::Resumption,
544    ) -> Result<()> {
545        let stop = self
546            .base_sync_recovery
547            .stop
548            .as_ref()
549            .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
550        ensure!(
551            approval.at > stop.at,
552            "approval predates the base-sync stop"
553        );
554        ensure!(
555            !self.base_sync_recovery.authorizations.iter().any(|a| {
556                a.task == task && a.generation.is_some_and(|g| g >= approval.generation)
557            }) && !self
558                .base_sync_recovery
559                .renewals
560                .iter()
561                .any(|r| r.task == task && r.generation >= approval.generation),
562            "operator resumption generation already used"
563        );
564        self.authorize_base_sync(task)?;
565        self.base_sync_recovery
566            .authorizations
567            .last_mut()
568            .unwrap()
569            .generation = Some(approval.generation);
570        Ok(())
571    }
572
573    /// Open the approved bounded interval and detach old gate evidence atomically
574    /// with consumption. The caller must save before reserving Git work.
575    pub(crate) fn consume_base_recovery(&mut self, head: &str) -> bool {
576        if !self.base_recovery_pending() {
577            return false;
578        }
579        let a = self.base_sync_recovery.authorizations.last_mut().unwrap();
580        a.consumed = true;
581        if a.stop.head != head {
582            a.refuse(format!(
583                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
584                a.stop.head
585            ));
586            return false;
587        }
588        a.reviews = self.reviews.clone();
589        a.gate = std::mem::take(&mut self.gate);
590        a.gate_head = self.gate_head.take();
591        a.gate_ran = std::mem::replace(&mut self.gate_ran, false);
592        let interval = a.interval;
593        self.base_sync.as_mut().unwrap().conflict = None;
594        self.open_base_episode("explicit exhausted-base recovery");
595        self.base_sync_recovery.stop = None;
596        if !interval {
597            self.base_sync_recovery.episodes.last_mut().unwrap().used =
598                crate::graph::BASE_SYNC_ROUNDS - 1;
599        }
600        true
601    }
602}
603
604/// A generic task release cannot authorize a stopped base synchronization.
605/// Use the queue's own home, including isolated triage stores.
606pub(crate) fn held_task(queue: &crate::queue::Queue, task: &crate::queue::Task) -> bool {
607    held_run(queue, task).is_some()
608}
609
610pub(crate) fn held_run(queue: &crate::queue::Queue, task: &crate::queue::Task) -> Option<RunState> {
611    let home = queue.root().parent()?;
612    task.runs
613        .last()
614        .and_then(|id| RunState::load_under(id, home).ok())
615        .filter(|state| {
616            state.base_sync_blocked()
617                && !state.base_recovery_pending()
618                && !state.base_sync_closure_pending()
619        })
620}
621
622impl crate::queue::Task {
623    /// Keep a producer stop stable across generic releases and driver passes.
624    pub(crate) fn base_sync_notice(&mut self, state: &RunState) {
625        let Some(sync) = state
626            .base_sync
627            .as_ref()
628            .filter(|_| state.base_sync_blocked())
629        else {
630            return;
631        };
632        let stop = state.base_sync_recovery.stop.as_ref();
633        self.held_at = Some(stop.map_or(state.updated_at, |s| s.at));
634        self.notice_cause("base-sync-stop", Some(&state.id), None);
635        if let Some(cause) = &mut self.hold_notice_cause {
636            cause.target = Some(match stop {
637                Some(s) => format!("{}:{}:{}", s.head, s.tip, s.at),
638                None => format!("legacy:{}:{}", sync.tip, sync.attempts),
639            });
640        }
641    }
642}
643
644/// Refuse a generic release at a textual stop before changing queue state.
645pub fn ensure_generic_release(
646    queue: &crate::queue::Queue,
647    task: &crate::queue::Task,
648) -> Result<()> {
649    if let Some(state) = task
650        .runs
651        .last()
652        .and_then(|id| RunState::load_under(id, queue.root().parent()?).ok())
653        && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
654    {
655        anyhow::bail!(reason);
656    }
657    if let Some(state) = held_run(queue, task)
658        && state
659            .base_sync_recovery
660            .stop
661            .as_ref()
662            .is_some_and(|s| s.kind == StopKind::Conflict)
663        && !crate::repair_budget::conflict_pending(task, &state)
664    {
665        anyhow::bail!(
666            "generic release cannot authorize textual conflict editing. {}",
667            state.base_sync_next_action()
668        );
669    }
670    Ok(())
671}