1use anyhow::{Result, ensure};
3use jiff::Timestamp;
4use serde::{Deserialize, Serialize};
5
6use crate::run::{CommandOutcome, ReviewRound, RunState, RunStatus};
7
8#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9#[serde(rename_all = "snake_case")]
10pub enum StopKind {
12 Exhausted,
14 Conflict,
16 Publication,
18 Divergence,
20 ClosingIntegratedPr,
22 Unknown,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Stop {
29 pub kind: StopKind,
31 pub head: String,
33 pub tip: String,
35 pub attempts: usize,
37 pub detail: String,
39 pub at: Timestamp,
41 #[serde(default)]
43 pub unverified_sync: bool,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct Authorization {
49 #[serde(default)]
50 pub scope: Option<Scope>,
51 #[serde(default)]
53 pub interval: bool,
54 #[serde(default)]
55 pub invalidated: Option<String>,
56 pub task: String,
58 pub run: String,
60 pub stop: Stop,
62 pub at: Timestamp,
64 pub consumed: bool,
66 #[serde(default)]
68 pub generation: Option<u64>,
69 pub reviews: Vec<ReviewRound>,
71 pub gate: Vec<CommandOutcome>,
73 pub gate_head: Option<String>,
75 pub gate_ran: bool,
77}
78
79impl Authorization {
80 pub(crate) fn refuse(&mut self, reason: impl Into<String>) {
82 self.invalidated = Some(reason.into());
83 self.consumed = true;
84 }
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct Episode {
90 pub id: usize,
92 pub reason: String,
94 pub used: usize,
96 pub synced_head: Option<String>,
98 #[serde(default)]
100 pub last_sync_head: Option<String>,
101 pub boundary: Option<Boundary>,
103 #[serde(default)]
105 pub prior_boundaries: Vec<Boundary>,
106}
107
108#[derive(Debug, Clone, Serialize, Deserialize)]
110pub struct Boundary {
111 pub pass: usize,
112 pub at: Timestamp,
113 pub head: String,
114 pub consumed: bool,
115}
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct Operation {
120 pub id: usize,
121 pub episode: usize,
122 pub head: String,
123 pub tip: String,
124 pub finished: bool,
125 #[serde(default)]
127 pub validated_head: Option<String>,
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize)]
132pub struct Renewal {
133 pub task: String,
134 pub run: String,
135 pub generation: u64,
136 pub instruction: String,
137 pub episode: usize,
138 pub boundary_pass: usize,
139 pub head: String,
140 pub tip: String,
141}
142
143#[derive(Debug, Clone, Serialize, Deserialize)]
145pub struct Scope {
146 pub tip: String,
147 pub instruction: String,
148 pub episode: usize,
149}
150
151#[derive(Debug, Clone, Default, Serialize, Deserialize)]
152pub struct Recovery {
154 #[serde(default)]
156 pub episodes: Vec<Episode>,
157 #[serde(default)]
158 pub operations: Vec<Operation>,
159 #[serde(default)]
161 pub stops: Vec<Stop>,
162 #[serde(default)]
163 pub renewals: Vec<Renewal>,
164 pub stop: Option<Stop>,
166 pub authorizations: Vec<Authorization>,
168}
169
170impl RunState {
171 pub fn base_sync_next_action(&self) -> String {
173 let episode = self.base_sync_recovery.episodes.last().map_or(0, |e| e.id);
174 let lifetime = self.base_sync.as_ref().map_or(0, |s| s.attempts);
175 format!(
176 "{} ({}/{} in episode {}, {} lifetime synchronizations)",
177 self.base_sync_action(),
178 self.base_episode_used(),
179 crate::graph::BASE_SYNC_ROUNDS,
180 episode,
181 lifetime
182 )
183 }
184
185 fn base_sync_action(&self) -> String {
186 if let Some(reason) = self
187 .base_sync_recovery
188 .authorizations
189 .last()
190 .and_then(|a| a.invalidated.as_ref())
191 {
192 return format!("Approval invalidated: {reason}");
193 }
194 if self.resource_block.is_some() {
195 return "Restore the required resource, then explicitly resume the retained run; automatic retries retain the synchronization budget".into();
196 }
197 if self.base_sync_recovery.stop.is_none()
198 && let Some(op) = self
199 .base_sync_recovery
200 .operations
201 .last()
202 .filter(|o| !o.finished)
203 {
204 return format!(
205 "Resume reserved synchronization {} against base {}; inspect retained Git state if reconciliation is refused",
206 op.id, op.tip
207 );
208 }
209 match self.base_sync_recovery.stop.as_ref().map(|s| &s.kind) {
210 Some(StopKind::Exhausted) => {
211 let why = if self.base_sync_recovery.stop.as_ref().is_some_and(|s| s.unverified_sync) {
212 "The synchronized HEAD never reached its gate before the base moved again. "
213 } else {
214 ""
215 };
216 if self.base_recovery_pending() {
217 format!("{why}Approval recorded; resume the retained run to open the approved bounded synchronization interval")
218 } else {
219 format!("{why}Authorize a new bounded synchronization interval with `magi run --resume {} --recover-base`, or a new owner resumption approval", self.id)
220 }
221 },
222 Some(StopKind::Conflict) => {
223 let stop = self.base_sync_recovery.stop.as_ref().unwrap();
224 let task = self.origin.as_ref().and_then(|origin| origin.task.as_deref()).unwrap_or("<task>");
225 format!("Authorize a narrow history-preserving repair with `magi task grant-repair {task} --run {} --kind base-conflict --rounds 1 --head {} --tip {} --expect-pass {} --request <unique-id> --reason <reason> --instruction <approved-resolution>`; retain the original review criteria. Base-budget recovery cannot resolve a textual conflict", self.id, stop.head, stop.tip, self.passes.last().map_or(0, |p| p.index))
226 },
227 Some(StopKind::Divergence) => "Reconcile foreign remote branch commits before resuming; do not overwrite them".into(),
228 Some(StopKind::Publication) => "Inspect the retained publication failure and remote lease before resuming".into(),
229 Some(StopKind::ClosingIntegratedPr) => "Resume to retry closing the already integrated pull request".into(),
230 Some(StopKind::Unknown) => "Inspect the retained synchronization state before resuming; unknown evidence grants no new budget".into(),
231 None => "Continue verification on the current synchronized HEAD".into(),
232 }
233 }
234
235 pub(crate) fn ensure_base_episode(&mut self) {
236 if let Some(stop) = self.base_sync_recovery.stop.clone()
237 && !self.base_sync_recovery.stops.contains(&stop)
238 {
239 self.base_sync_recovery.stops.push(stop);
240 }
241 if self.base_sync_recovery.episodes.is_empty() {
242 let used = self.base_sync.as_ref().map_or(0, |s| s.attempts);
243 self.base_sync_recovery.episodes.push(Episode {
244 id: 0,
245 reason: "initial or conservative legacy migration".into(),
246 used,
247 synced_head: None,
248 last_sync_head: None,
249 boundary: None,
250 prior_boundaries: Vec::new(),
251 });
252 }
253 }
254
255 fn open_base_episode(&mut self, reason: &str) {
256 self.ensure_base_episode();
257 let id = self.base_sync_recovery.episodes.len();
258 self.base_sync_recovery.episodes.push(Episode {
259 id,
260 reason: reason.into(),
261 used: 0,
262 synced_head: None,
263 last_sync_head: None,
264 boundary: None,
265 prior_boundaries: Vec::new(),
266 });
267 }
268
269 pub(crate) fn base_episode_used(&self) -> usize {
270 self.base_sync_recovery.episodes.last().map_or_else(
271 || self.base_sync.as_ref().map_or(0, |s| s.attempts),
272 |e| e.used,
273 )
274 }
275
276 pub(crate) fn record_base_progress(&mut self, head: &str) {
277 self.ensure_base_episode();
278 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
279 if let Some(boundary) = episode.boundary.take() {
280 episode.prior_boundaries.push(boundary);
281 }
282 episode.synced_head = Some(head.into());
283 }
284
285 pub(crate) fn record_base_interruption(&mut self) {
287 let Some(block) = &self.resource_block else {
288 return;
289 };
290 if matches!(block.cause, crate::run::ResourceBlockCause::Unrecorded) {
291 return;
292 }
293 let Some(episode) = self.base_sync_recovery.episodes.last_mut() else {
294 return;
295 };
296 let Some(head) = episode.synced_head.clone() else {
297 return;
298 };
299 if episode.boundary.is_none() {
300 episode.boundary = Some(Boundary {
301 pass: self.passes.last().map_or(0, |p| p.index),
302 at: block.at,
303 head,
304 consumed: false,
305 });
306 }
307 }
308
309 pub(crate) fn renew_base_episode(
310 &mut self,
311 task: &str,
312 approval: &crate::queue::Resumption,
313 head: &str,
314 tip: &str,
315 ) -> bool {
316 if self
317 .base_sync_recovery
318 .renewals
319 .iter()
320 .any(|r| r.task == task && r.generation >= approval.generation)
321 || self
322 .base_sync_recovery
323 .authorizations
324 .iter()
325 .any(|a| a.task == task && a.generation.is_some_and(|g| g >= approval.generation))
326 {
327 return false;
328 }
329 let Some(e) = self.base_sync_recovery.episodes.last_mut() else {
330 return false;
331 };
332 let Some(b) = &mut e.boundary else {
333 return false;
334 };
335 if b.consumed || b.head != head || approval.at <= b.at {
336 return false;
337 }
338 b.consumed = true;
339 self.base_sync_recovery.renewals.push(Renewal {
340 task: task.into(),
341 run: self.id.clone(),
342 generation: approval.generation,
343 instruction: approval.instruction.clone(),
344 episode: e.id,
345 boundary_pass: b.pass,
346 head: head.into(),
347 tip: tip.into(),
348 });
349 self.open_base_episode("owner resumption after recorded infrastructure interruption");
350 true
351 }
352
353 pub(crate) fn reserve_base_operation(&mut self, head: &str, tip: &str) {
355 self.ensure_base_episode();
356 if self
357 .base_sync_recovery
358 .operations
359 .last()
360 .is_some_and(|o| !o.finished)
361 {
362 return;
363 }
364 let e = self.base_sync_recovery.episodes.last_mut().unwrap();
365 e.used += 1;
366 let id = self.base_sync_recovery.operations.len();
367 self.base_sync_recovery.operations.push(Operation {
368 id,
369 episode: e.id,
370 head: head.into(),
371 tip: tip.into(),
372 finished: false,
373 validated_head: None,
374 });
375 if let Some(sync) = &mut self.base_sync {
376 sync.attempts += 1;
377 }
378 }
379
380 pub(crate) fn finish_base_operation(&mut self, head: Option<&str>) {
381 if let Some(op) = self.base_sync_recovery.operations.last_mut() {
382 op.finished = true;
383 }
384 if let Some(head) = head {
385 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
386 if let Some(boundary) = episode.boundary.take() {
387 episode.prior_boundaries.push(boundary);
388 }
389 episode.synced_head = Some(head.into());
390 episode.last_sync_head = Some(head.into());
391 self.reconcile_verification_head(head);
392 }
393 }
394
395 pub(crate) fn base_sync_blocked(&self) -> bool {
396 self.base_sync
397 .as_ref()
398 .is_some_and(|s| s.conflict.is_some())
399 }
400
401 pub(crate) fn base_sync_closure_pending(&self) -> bool {
404 let Some(detail) = self
405 .base_sync
406 .as_ref()
407 .and_then(|sync| sync.conflict.as_ref())
408 else {
409 return false;
410 };
411 let Some(winner) = self.winner() else {
412 return false;
413 };
414 let prefix = format!(
416 "{} is already in {}/{}, but its pull request could not be closed (",
417 winner.branch, self.config.merge.remote, self.base_branch
418 );
419 detail.starts_with(&prefix) && detail.ends_with("); resume to retry")
420 }
421
422 pub(crate) fn record_base_sync_stop(&mut self, head: &str) {
425 let Some(sync) = self.base_sync.as_ref() else {
426 return;
427 };
428 let Some(detail) = sync.conflict.clone() else {
429 return;
430 };
431 if self
432 .base_sync_recovery
433 .stop
434 .as_ref()
435 .is_some_and(|s| s.head == head && s.tip == sync.tip && s.detail == detail)
436 {
437 self.status = RunStatus::Blocked;
438 return;
439 }
440 let exhausted = self.winner().is_some_and(|w| {
441 [
442 ("rebase(s)", "rebasing"),
443 ("synchronization(s)", "synchronizing"),
444 ].iter().any(|(noun, verb)| detail == format!(
445 "{} moved {} commit(s) ahead of {} after {} {}; {} again would only race it",
446 self.base_branch, sync.behind, w.branch, crate::graph::BASE_SYNC_ROUNDS, noun, verb
447 ))
448 });
449 let kind = if self.base_sync_closure_pending() {
450 StopKind::ClosingIntegratedPr
451 } else if sync.attempts >= crate::graph::BASE_SYNC_ROUNDS && exhausted {
452 StopKind::Exhausted
453 } else if detail.contains("not rebasing over") {
454 StopKind::Divergence
455 } else if detail.contains("refused the push")
456 || detail.contains("could not push")
457 || detail.contains("could not be pushed")
458 {
459 StopKind::Publication
460 } else if detail.contains("conflicts with") && detail.contains("did not rebase") {
461 StopKind::Conflict
462 } else {
463 StopKind::Unknown
464 };
465 let at = self
466 .events
467 .iter()
468 .rev()
469 .find(|event| event.node == "land" && event.message == detail)
470 .map_or(self.updated_at, |event| event.at);
471 let unverified_sync = sync.behind > 0
472 && self
473 .base_sync_recovery
474 .episodes
475 .last()
476 .is_some_and(|e| e.last_sync_head.as_deref() == Some(head))
477 && !(self.gate_ran && self.gate_head.as_deref() == Some(head));
478 let stop = Stop {
479 unverified_sync,
480 kind,
481 head: head.to_owned(),
482 tip: sync.tip.clone(),
483 attempts: sync.attempts,
484 detail,
485 at,
486 };
487 self.base_sync_recovery.stops.push(stop.clone());
488 self.base_sync_recovery.stop = Some(stop);
489 self.status = RunStatus::Blocked;
490 }
491
492 pub(crate) fn base_recovery_pending(&self) -> bool {
493 self.base_sync_recovery
494 .authorizations
495 .last()
496 .is_some_and(|a| {
497 !a.consumed
498 && a.invalidated.is_none()
499 && a.run == self.id
500 && self.base_sync_recovery.stop.as_ref() == Some(&a.stop)
501 })
502 }
503
504 pub(crate) fn authorize_base_sync(&mut self, task: &str) -> Result<()> {
505 let stop = self
506 .base_sync_recovery
507 .stop
508 .clone()
509 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
510 ensure!(
511 stop.kind == StopKind::Exhausted,
512 "only exhausted base movement permits recovery"
513 );
514 ensure!(
515 !self
516 .base_sync_recovery
517 .authorizations
518 .last()
519 .is_some_and(|a| !a.consumed && a.stop == stop),
520 "this stop already has a pending authorization; resume the existing request"
521 );
522 self.base_sync_recovery.authorizations.push(Authorization {
523 scope: None,
524 interval: true,
525 invalidated: None,
526 task: task.to_owned(),
527 run: self.id.clone(),
528 stop,
529 at: Timestamp::now(),
530 consumed: false,
531 generation: None,
532 reviews: Vec::new(),
533 gate: Vec::new(),
534 gate_head: None,
535 gate_ran: false,
536 });
537 Ok(())
538 }
539
540 pub(crate) fn authorize_base_sync_resumption(
541 &mut self,
542 task: &str,
543 approval: &crate::queue::Resumption,
544 ) -> Result<()> {
545 let stop = self
546 .base_sync_recovery
547 .stop
548 .as_ref()
549 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
550 ensure!(
551 approval.at > stop.at,
552 "approval predates the base-sync stop"
553 );
554 ensure!(
555 !self.base_sync_recovery.authorizations.iter().any(|a| {
556 a.task == task && a.generation.is_some_and(|g| g >= approval.generation)
557 }) && !self
558 .base_sync_recovery
559 .renewals
560 .iter()
561 .any(|r| r.task == task && r.generation >= approval.generation),
562 "operator resumption generation already used"
563 );
564 self.authorize_base_sync(task)?;
565 self.base_sync_recovery
566 .authorizations
567 .last_mut()
568 .unwrap()
569 .generation = Some(approval.generation);
570 Ok(())
571 }
572
573 pub(crate) fn consume_base_recovery(&mut self, head: &str) -> bool {
576 if !self.base_recovery_pending() {
577 return false;
578 }
579 let a = self.base_sync_recovery.authorizations.last_mut().unwrap();
580 a.consumed = true;
581 if a.stop.head != head {
582 a.refuse(format!(
583 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
584 a.stop.head
585 ));
586 return false;
587 }
588 a.reviews = self.reviews.clone();
589 a.gate = std::mem::take(&mut self.gate);
590 a.gate_head = self.gate_head.take();
591 a.gate_ran = std::mem::replace(&mut self.gate_ran, false);
592 let interval = a.interval;
593 self.base_sync.as_mut().unwrap().conflict = None;
594 self.open_base_episode("explicit exhausted-base recovery");
595 self.base_sync_recovery.stop = None;
596 if !interval {
597 self.base_sync_recovery.episodes.last_mut().unwrap().used =
598 crate::graph::BASE_SYNC_ROUNDS - 1;
599 }
600 true
601 }
602}
603
604pub(crate) fn held_task(queue: &crate::queue::Queue, task: &crate::queue::Task) -> bool {
607 held_run(queue, task).is_some()
608}
609
610pub(crate) fn held_run(queue: &crate::queue::Queue, task: &crate::queue::Task) -> Option<RunState> {
611 let home = queue.root().parent()?;
612 task.runs
613 .last()
614 .and_then(|id| RunState::load_under(id, home).ok())
615 .filter(|state| {
616 state.base_sync_blocked()
617 && !state.base_recovery_pending()
618 && !state.base_sync_closure_pending()
619 })
620}
621
622impl crate::queue::Task {
623 pub(crate) fn base_sync_notice(&mut self, state: &RunState) {
625 let Some(sync) = state
626 .base_sync
627 .as_ref()
628 .filter(|_| state.base_sync_blocked())
629 else {
630 return;
631 };
632 let stop = state.base_sync_recovery.stop.as_ref();
633 self.held_at = Some(stop.map_or(state.updated_at, |s| s.at));
634 self.notice_cause("base-sync-stop", Some(&state.id), None);
635 if let Some(cause) = &mut self.hold_notice_cause {
636 cause.target = Some(match stop {
637 Some(s) => format!("{}:{}:{}", s.head, s.tip, s.at),
638 None => format!("legacy:{}:{}", sync.tip, sync.attempts),
639 });
640 }
641 }
642}
643
644pub fn ensure_generic_release(
646 queue: &crate::queue::Queue,
647 task: &crate::queue::Task,
648) -> Result<()> {
649 if let Some(state) = task
650 .runs
651 .last()
652 .and_then(|id| RunState::load_under(id, queue.root().parent()?).ok())
653 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
654 {
655 anyhow::bail!(reason);
656 }
657 if let Some(state) = held_run(queue, task)
658 && state
659 .base_sync_recovery
660 .stop
661 .as_ref()
662 .is_some_and(|s| s.kind == StopKind::Conflict)
663 && !crate::repair_budget::conflict_pending(task, &state)
664 {
665 anyhow::bail!(
666 "generic release cannot authorize textual conflict editing. {}",
667 state.base_sync_next_action()
668 );
669 }
670 Ok(())
671}