Skip to main content

magi/
base_sync_recovery.rs

1//! Durable bounded synchronization intervals and single-use owner recovery approvals.
2use anyhow::{Result, ensure};
3use jiff::Timestamp;
4use serde::{Deserialize, Serialize};
5
6use crate::run::{CommandOutcome, ReviewRound, RunState, RunStatus};
7
8#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9#[serde(rename_all = "snake_case")]
10/// Why synchronization stopped. Only exhaustion admits a new recovery interval.
11pub enum StopKind {
12    /// The ordinary moving-base budget was spent.
13    Exhausted,
14    /// Replaying the retained changes failed.
15    Conflict,
16    /// A remote ownership or publication operation failed.
17    Publication,
18    /// The remote branch contains foreign work.
19    Divergence,
20    /// The change was integrated, but closing its PR failed and may be retried.
21    ClosingIntegratedPr,
22    /// Legacy or unexpected evidence cannot justify recovery.
23    Unknown,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27/// Stable identity of a stopped synchronization.
28pub struct Stop {
29    /// Typed cause.
30    pub kind: StopKind,
31    /// Retained candidate commit.
32    pub head: String,
33    /// Base tip observed at the stop.
34    pub tip: String,
35    /// Cumulative attempts when stopped.
36    pub attempts: usize,
37    /// Original diagnostic, retained verbatim.
38    pub detail: String,
39    /// When this record was created.
40    pub at: Timestamp,
41    /// A synchronized HEAD had not reached its gate before the base advanced.
42    #[serde(default)]
43    pub unverified_sync: bool,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47/// One explicit owner request, consumed durably before rebasing.
48pub struct Authorization {
49    #[serde(default)]
50    pub scope: Option<Scope>,
51    /// Standing-control request key, absent for terminal recovery.
52    #[serde(default)]
53    pub request: Option<String>,
54    /// Owner decision retained by the exact queue transition.
55    #[serde(default)]
56    pub decision: Option<Timestamp>,
57    /// Exact routed decision, when recovery came from a typed owner choice.
58    #[serde(default)]
59    pub question: Option<String>,
60    /// Older approvals authorized one attempt only.
61    #[serde(default)]
62    pub interval: bool,
63    #[serde(default)]
64    pub invalidated: Option<String>,
65    /// Task whose claim admitted the request.
66    pub task: String,
67    /// Exact saved execution authorized to recover.
68    pub run: String,
69    /// Immutable stop authorized by this request.
70    pub stop: Stop,
71    /// When this record was created.
72    pub at: Timestamp,
73    /// Persisted before any Git mutation. An interrupted consumption grants no retry.
74    pub consumed: bool,
75    /// Queue approval generation, absent for an explicit recovery command.
76    #[serde(default)]
77    pub generation: Option<u64>,
78    /// Historical evidence stays attached to its original heads.
79    pub reviews: Vec<ReviewRound>,
80    /// Prior gate outcomes, never evidence for the recovered head.
81    pub gate: Vec<CommandOutcome>,
82    /// Head targeted by the archived gate.
83    pub gate_head: Option<String>,
84    /// Whether the archived gate was attempted.
85    pub gate_ran: bool,
86}
87
88impl Authorization {
89    /// Consume a refused approval without granting recovery, retaining the reason.
90    pub(crate) fn refuse(&mut self, reason: impl Into<String>) {
91        self.invalidated = Some(reason.into());
92        self.consumed = true;
93    }
94}
95
96/// A bounded active interval. Lifetime attempts remain in `BaseSync`.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct Episode {
99    /// Monotonic interval identity.
100    pub id: usize,
101    /// Why this interval was opened.
102    pub reason: String,
103    /// Reservations, including interrupted operations.
104    pub used: usize,
105    /// Last successful synchronization or actually executed verification in this interval.
106    pub synced_head: Option<String>,
107    /// Last successful synchronization, distinct from verification progress.
108    #[serde(default)]
109    pub last_sync_head: Option<String>,
110    /// An interruption after progress; consumed at most once.
111    pub boundary: Option<Boundary>,
112    /// Earlier interruption evidence superseded by actual progress.
113    #[serde(default)]
114    pub prior_boundaries: Vec<Boundary>,
115}
116
117/// Durable interruption evidence, never inferred from elapsed time.
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct Boundary {
120    pub pass: usize,
121    pub at: Timestamp,
122    pub head: String,
123    pub consumed: bool,
124}
125
126/// Write-ahead reservation. A restart continues this exact operation.
127#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct Operation {
129    pub id: usize,
130    pub episode: usize,
131    pub head: String,
132    pub tip: String,
133    pub finished: bool,
134    /// Exact result accepted by the rebase validator, saved before scratch cleanup.
135    #[serde(default)]
136    pub validated_head: Option<String>,
137}
138
139/// Consumed owner approval for an infrastructure boundary.
140#[derive(Debug, Clone, Serialize, Deserialize)]
141pub struct Renewal {
142    pub task: String,
143    pub run: String,
144    pub generation: u64,
145    pub instruction: String,
146    pub episode: usize,
147    pub boundary_pass: usize,
148    pub head: String,
149    pub tip: String,
150}
151
152/// Approval scope additional to the immutable stop.
153#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct Scope {
155    pub tip: String,
156    pub instruction: String,
157    pub episode: usize,
158}
159
160#[derive(Debug, Clone, Default, Serialize, Deserialize)]
161/// Persistent stop and authorization history, additive for legacy runs.
162pub struct Recovery {
163    /// Append-only interval and operation evidence, absent in old records.
164    #[serde(default)]
165    pub episodes: Vec<Episode>,
166    #[serde(default)]
167    pub operations: Vec<Operation>,
168    /// Stops remain evidence after successful recovery.
169    #[serde(default)]
170    pub stops: Vec<Stop>,
171    #[serde(default)]
172    pub renewals: Vec<Renewal>,
173    /// Current stop, if any.
174    pub stop: Option<Stop>,
175    /// Requests and archived evidence in authorization order.
176    pub authorizations: Vec<Authorization>,
177}
178
179impl RunState {
180    /// Shared CLI, notice and web instruction, based on the typed checkpoint.
181    pub fn base_sync_next_action(&self) -> String {
182        let episode = self.base_sync_recovery.episodes.last().map_or(0, |e| e.id);
183        let lifetime = self.base_sync.as_ref().map_or(0, |s| s.attempts);
184        format!(
185            "{} ({}/{} in episode {}, {} lifetime synchronizations)",
186            self.base_sync_action(),
187            self.base_episode_used(),
188            crate::graph::BASE_SYNC_ROUNDS,
189            episode,
190            lifetime
191        )
192    }
193
194    fn base_sync_action(&self) -> String {
195        if let Some(reason) = self
196            .base_sync_recovery
197            .authorizations
198            .last()
199            .and_then(|a| a.invalidated.as_ref())
200        {
201            return format!("Approval invalidated: {reason}");
202        }
203        if self.resource_block.is_some() {
204            return "Restore the required resource, then explicitly resume the retained run; automatic retries retain the synchronization budget".into();
205        }
206        if self.base_sync_recovery.stop.is_none()
207            && let Some(op) = self
208                .base_sync_recovery
209                .operations
210                .last()
211                .filter(|o| !o.finished)
212        {
213            return format!(
214                "Resume reserved synchronization {} against base {}; inspect retained Git state if reconciliation is refused",
215                op.id, op.tip
216            );
217        }
218        match self.base_sync_recovery.stop.as_ref().map(|s| &s.kind) {
219            Some(StopKind::Exhausted) => {
220                let why = if self.base_sync_recovery.stop.as_ref().is_some_and(|s| s.unverified_sync) {
221                    "The synchronized HEAD never reached its gate before the base moved again. "
222                } else {
223                    ""
224                };
225                if self.base_recovery_pending() {
226                    format!("{why}Approval recorded; resume the retained run to open the approved bounded synchronization interval")
227                } else {
228                    format!("{why}Authorize a new bounded synchronization interval with `magi run --resume {} --recover-base`, or, from a standing chat, `magi task control recover-base {} --request <unique-id> --head {}`", self.id, self.id, self.base_sync_recovery.stop.as_ref().unwrap().head)
229                }
230            },
231            Some(StopKind::Conflict) => {
232                let stop = self.base_sync_recovery.stop.as_ref().unwrap();
233                let task = self.origin.as_ref().and_then(|origin| origin.task.as_deref()).unwrap_or("<task>");
234                format!("Authorize a narrow history-preserving repair with `magi task grant-repair {task} --run {} --kind base-conflict --rounds 1 --head {} --tip {} --expect-pass {} --request <unique-id> --reason <reason> --instruction <approved-resolution>`; retain the original review criteria. Base-budget recovery cannot resolve a textual conflict", self.id, stop.head, stop.tip, self.passes.last().map_or(0, |p| p.index))
235            },
236            Some(StopKind::Divergence) => "Reconcile foreign remote branch commits before resuming; do not overwrite them".into(),
237            Some(StopKind::Publication) => "Inspect the retained publication failure and remote lease before resuming".into(),
238            Some(StopKind::ClosingIntegratedPr) => "Resume to retry closing the already integrated pull request".into(),
239            Some(StopKind::Unknown) => "Inspect the retained synchronization state before resuming; unknown evidence grants no new budget".into(),
240            None => "Continue verification on the current synchronized HEAD".into(),
241        }
242    }
243
244    pub(crate) fn ensure_base_episode(&mut self) {
245        if let Some(stop) = self.base_sync_recovery.stop.clone()
246            && !self.base_sync_recovery.stops.contains(&stop)
247        {
248            self.base_sync_recovery.stops.push(stop);
249        }
250        if self.base_sync_recovery.episodes.is_empty() {
251            let used = self.base_sync.as_ref().map_or(0, |s| s.attempts);
252            self.base_sync_recovery.episodes.push(Episode {
253                id: 0,
254                reason: "initial or conservative legacy migration".into(),
255                used,
256                synced_head: None,
257                last_sync_head: None,
258                boundary: None,
259                prior_boundaries: Vec::new(),
260            });
261        }
262    }
263
264    fn open_base_episode(&mut self, reason: &str) {
265        self.ensure_base_episode();
266        let id = self.base_sync_recovery.episodes.len();
267        self.base_sync_recovery.episodes.push(Episode {
268            id,
269            reason: reason.into(),
270            used: 0,
271            synced_head: None,
272            last_sync_head: None,
273            boundary: None,
274            prior_boundaries: Vec::new(),
275        });
276    }
277
278    pub(crate) fn base_episode_used(&self) -> usize {
279        self.base_sync_recovery.episodes.last().map_or_else(
280            || self.base_sync.as_ref().map_or(0, |s| s.attempts),
281            |e| e.used,
282        )
283    }
284
285    pub(crate) fn record_base_progress(&mut self, head: &str) {
286        self.ensure_base_episode();
287        let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
288        if let Some(boundary) = episode.boundary.take() {
289            episode.prior_boundaries.push(boundary);
290        }
291        episode.synced_head = Some(head.into());
292    }
293
294    /// Called by the same atomic state save that retains the resource stop.
295    pub(crate) fn record_base_interruption(&mut self) {
296        let Some(block) = &self.resource_block else {
297            return;
298        };
299        if matches!(block.cause, crate::run::ResourceBlockCause::Unrecorded) {
300            return;
301        }
302        let Some(episode) = self.base_sync_recovery.episodes.last_mut() else {
303            return;
304        };
305        let Some(head) = episode.synced_head.clone() else {
306            return;
307        };
308        if episode.boundary.is_none() {
309            episode.boundary = Some(Boundary {
310                pass: self.passes.last().map_or(0, |p| p.index),
311                at: block.at,
312                head,
313                consumed: false,
314            });
315        }
316    }
317
318    pub(crate) fn renew_base_episode(
319        &mut self,
320        task: &str,
321        approval: &crate::queue::Resumption,
322        head: &str,
323        tip: &str,
324    ) -> bool {
325        if self
326            .base_sync_recovery
327            .renewals
328            .iter()
329            .any(|r| r.task == task && r.generation >= approval.generation)
330            || self
331                .base_sync_recovery
332                .authorizations
333                .iter()
334                .any(|a| a.task == task && a.generation.is_some_and(|g| g >= approval.generation))
335        {
336            return false;
337        }
338        let Some(e) = self.base_sync_recovery.episodes.last_mut() else {
339            return false;
340        };
341        let Some(b) = &mut e.boundary else {
342            return false;
343        };
344        if b.consumed || b.head != head || approval.at <= b.at {
345            return false;
346        }
347        b.consumed = true;
348        self.base_sync_recovery.renewals.push(Renewal {
349            task: task.into(),
350            run: self.id.clone(),
351            generation: approval.generation,
352            instruction: approval.instruction.clone(),
353            episode: e.id,
354            boundary_pass: b.pass,
355            head: head.into(),
356            tip: tip.into(),
357        });
358        self.open_base_episode("owner resumption after recorded infrastructure interruption");
359        true
360    }
361
362    /// Reserve once before Git; retrying an unfinished operation spends nothing more.
363    pub(crate) fn reserve_base_operation(&mut self, head: &str, tip: &str) {
364        self.ensure_base_episode();
365        if self
366            .base_sync_recovery
367            .operations
368            .last()
369            .is_some_and(|o| !o.finished)
370        {
371            return;
372        }
373        let e = self.base_sync_recovery.episodes.last_mut().unwrap();
374        e.used += 1;
375        let id = self.base_sync_recovery.operations.len();
376        self.base_sync_recovery.operations.push(Operation {
377            id,
378            episode: e.id,
379            head: head.into(),
380            tip: tip.into(),
381            finished: false,
382            validated_head: None,
383        });
384        if let Some(sync) = &mut self.base_sync {
385            sync.attempts += 1;
386        }
387    }
388
389    pub(crate) fn finish_base_operation(&mut self, head: Option<&str>) {
390        if let Some(op) = self.base_sync_recovery.operations.last_mut() {
391            op.finished = true;
392        }
393        if let Some(head) = head {
394            let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
395            if let Some(boundary) = episode.boundary.take() {
396                episode.prior_boundaries.push(boundary);
397            }
398            episode.synced_head = Some(head.into());
399            episode.last_sync_head = Some(head.into());
400            self.reconcile_verification_head(head);
401        }
402    }
403
404    pub(crate) fn base_sync_blocked(&self) -> bool {
405        self.base_sync
406            .as_ref()
407            .is_some_and(|s| s.conflict.is_some())
408    }
409
410    /// This checkpoint only rechecks integration and retries PR closure. It
411    /// never clears the conflict guard or grants an additional rebase.
412    pub(crate) fn base_sync_closure_pending(&self) -> bool {
413        let Some(detail) = self
414            .base_sync
415            .as_ref()
416            .and_then(|sync| sync.conflict.as_ref())
417        else {
418            return false;
419        };
420        let Some(winner) = self.winner() else {
421            return false;
422        };
423        // Also recognizes the diagnostic written before typed stops existed.
424        let prefix = format!(
425            "{} is already in {}/{}, but its pull request could not be closed (",
426            winner.branch, self.config.merge.remote, self.base_branch
427        );
428        detail.starts_with(&prefix) && detail.ends_with("); resume to retry")
429    }
430
431    /// Legacy exhaustion is recognized only by the exact message and budget.
432    /// Once recorded, its identity remains stable across no-progress passes.
433    pub(crate) fn record_base_sync_stop(&mut self, head: &str) {
434        let Some(sync) = self.base_sync.as_ref() else {
435            return;
436        };
437        let Some(detail) = sync.conflict.clone() else {
438            return;
439        };
440        if self
441            .base_sync_recovery
442            .stop
443            .as_ref()
444            .is_some_and(|s| s.head == head && s.tip == sync.tip && s.detail == detail)
445        {
446            self.status = RunStatus::Blocked;
447            return;
448        }
449        let exhausted = self.winner().is_some_and(|w| {
450            [
451                ("rebase(s)", "rebasing"),
452                ("synchronization(s)", "synchronizing"),
453            ].iter().any(|(noun, verb)| detail == format!(
454                "{} moved {} commit(s) ahead of {} after {} {}; {} again would only race it",
455                self.base_branch, sync.behind, w.branch, crate::graph::BASE_SYNC_ROUNDS, noun, verb
456            ))
457        });
458        let kind = if self.base_sync_closure_pending() {
459            StopKind::ClosingIntegratedPr
460        } else if sync.attempts >= crate::graph::BASE_SYNC_ROUNDS && exhausted {
461            StopKind::Exhausted
462        } else if detail.contains("not rebasing over") {
463            StopKind::Divergence
464        } else if detail.contains("refused the push")
465            || detail.contains("could not push")
466            || detail.contains("could not be pushed")
467        {
468            StopKind::Publication
469        } else if detail.contains("conflicts with") && detail.contains("did not rebase") {
470            StopKind::Conflict
471        } else {
472            StopKind::Unknown
473        };
474        let at = self
475            .events
476            .iter()
477            .rev()
478            .find(|event| event.node == "land" && event.message == detail)
479            .map_or(self.updated_at, |event| event.at);
480        let unverified_sync = sync.behind > 0
481            && self
482                .base_sync_recovery
483                .episodes
484                .last()
485                .is_some_and(|e| e.last_sync_head.as_deref() == Some(head))
486            && !(self.gate_ran && self.gate_head.as_deref() == Some(head));
487        let stop = Stop {
488            unverified_sync,
489            kind,
490            head: head.to_owned(),
491            tip: sync.tip.clone(),
492            attempts: sync.attempts,
493            detail,
494            at,
495        };
496        self.base_sync_recovery.stops.push(stop.clone());
497        self.base_sync_recovery.stop = Some(stop);
498        self.status = RunStatus::Blocked;
499    }
500
501    pub(crate) fn base_recovery_pending(&self) -> bool {
502        self.base_sync_recovery
503            .authorizations
504            .last()
505            .is_some_and(|a| {
506                !a.consumed
507                    && a.invalidated.is_none()
508                    && a.run == self.id
509                    && self.base_sync_recovery.stop.as_ref() == Some(&a.stop)
510            })
511    }
512
513    pub(crate) fn authorize_base_sync(&mut self, task: &str) -> Result<()> {
514        let stop = self
515            .base_sync_recovery
516            .stop
517            .clone()
518            .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
519        ensure!(
520            stop.kind == StopKind::Exhausted,
521            "only exhausted base movement permits recovery"
522        );
523        ensure!(
524            !self
525                .base_sync_recovery
526                .authorizations
527                .last()
528                .is_some_and(|a| !a.consumed && a.stop == stop),
529            "this stop already has a pending authorization; resume the existing request"
530        );
531        self.base_sync_recovery.authorizations.push(Authorization {
532            scope: None,
533            request: None,
534            decision: None,
535            question: None,
536            interval: true,
537            invalidated: None,
538            task: task.to_owned(),
539            run: self.id.clone(),
540            stop,
541            at: Timestamp::now(),
542            consumed: false,
543            generation: None,
544            reviews: Vec::new(),
545            gate: Vec::new(),
546            gate_head: None,
547            gate_ran: false,
548        });
549        Ok(())
550    }
551
552    pub(crate) fn authorize_base_sync_resumption(
553        &mut self,
554        task: &str,
555        approval: &crate::queue::Resumption,
556    ) -> Result<()> {
557        let stop = self
558            .base_sync_recovery
559            .stop
560            .as_ref()
561            .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
562        ensure!(
563            approval.at > stop.at,
564            "approval predates the base-sync stop"
565        );
566        ensure!(
567            !self.base_sync_recovery.authorizations.iter().any(|a| {
568                a.task == task && a.generation.is_some_and(|g| g >= approval.generation)
569            }) && !self
570                .base_sync_recovery
571                .renewals
572                .iter()
573                .any(|r| r.task == task && r.generation >= approval.generation),
574            "operator resumption generation already used"
575        );
576        self.authorize_base_sync(task)?;
577        self.base_sync_recovery
578            .authorizations
579            .last_mut()
580            .unwrap()
581            .generation = Some(approval.generation);
582        Ok(())
583    }
584
585    /// Open the approved bounded interval and detach old gate evidence atomically
586    /// with consumption. The caller must save before reserving Git work.
587    pub(crate) fn consume_base_recovery(&mut self, head: &str) -> bool {
588        if !self.base_recovery_pending() {
589            return false;
590        }
591        let a = self.base_sync_recovery.authorizations.last_mut().unwrap();
592        a.consumed = true;
593        if a.stop.head != head {
594            a.refuse(format!(
595                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
596                a.stop.head
597            ));
598            return false;
599        }
600        a.reviews = self.reviews.clone();
601        a.gate = std::mem::take(&mut self.gate);
602        a.gate_head = self.gate_head.take();
603        a.gate_ran = std::mem::replace(&mut self.gate_ran, false);
604        let interval = a.interval;
605        self.base_sync.as_mut().unwrap().conflict = None;
606        self.open_base_episode("explicit exhausted-base recovery");
607        self.base_sync_recovery.stop = None;
608        if !interval {
609            self.base_sync_recovery.episodes.last_mut().unwrap().used =
610                crate::graph::BASE_SYNC_ROUNDS - 1;
611        }
612        true
613    }
614}
615
616/// A generic task release cannot authorize a stopped base synchronization.
617/// Use the queue's own home, including isolated triage stores.
618pub(crate) fn held_task(queue: &crate::queue::Queue, task: &crate::queue::Task) -> bool {
619    held_run(queue, task).is_some()
620}
621
622pub(crate) fn held_run(queue: &crate::queue::Queue, task: &crate::queue::Task) -> Option<RunState> {
623    let home = queue.root().parent()?;
624    task.execution_runs()
625        .last()
626        .and_then(|id| RunState::load_under(id, home).ok())
627        .filter(|state| {
628            state.base_sync_blocked()
629                && !state.base_recovery_pending()
630                && !state.base_sync_closure_pending()
631        })
632}
633
634impl crate::queue::Task {
635    /// Keep a producer stop stable across generic releases and driver passes.
636    pub(crate) fn base_sync_notice(&mut self, state: &RunState) {
637        let Some(sync) = state
638            .base_sync
639            .as_ref()
640            .filter(|_| state.base_sync_blocked())
641        else {
642            return;
643        };
644        let stop = state.base_sync_recovery.stop.as_ref();
645        self.held_at = Some(stop.map_or(state.updated_at, |s| s.at));
646        self.notice_cause("base-sync-stop", Some(&state.id), None);
647        if let Some(cause) = &mut self.hold_notice_cause {
648            cause.target = Some(match stop {
649                Some(s) => format!("{}:{}:{}", s.head, s.tip, s.at),
650                None => format!("legacy:{}:{}", sync.tip, sync.attempts),
651            });
652        }
653    }
654}
655
656/// Refuse a generic release at a textual stop before changing queue state.
657pub fn ensure_generic_release(
658    queue: &crate::queue::Queue,
659    task: &crate::queue::Task,
660) -> Result<()> {
661    if let Some(state) = task
662        .execution_runs()
663        .last()
664        .and_then(|id| RunState::load_under(id, queue.root().parent()?).ok())
665        && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
666    {
667        anyhow::bail!(reason);
668    }
669    if let Some(state) = held_run(queue, task)
670        && state
671            .base_sync_recovery
672            .stop
673            .as_ref()
674            .is_some_and(|s| s.kind == StopKind::Conflict)
675        && !crate::repair_budget::conflict_pending(task, &state)
676    {
677        anyhow::bail!(
678            "generic release cannot authorize textual conflict editing. {}",
679            state.base_sync_next_action()
680        );
681    }
682    Ok(())
683}