1use anyhow::{Result, ensure};
3use jiff::Timestamp;
4use serde::{Deserialize, Serialize};
5
6use crate::run::{CommandOutcome, ReviewRound, RunState, RunStatus};
7
8#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9#[serde(rename_all = "snake_case")]
10pub enum StopKind {
12 Exhausted,
14 Conflict,
16 Publication,
18 Divergence,
20 ClosingIntegratedPr,
22 Unknown,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Stop {
29 pub kind: StopKind,
31 pub head: String,
33 pub tip: String,
35 pub attempts: usize,
37 pub detail: String,
39 pub at: Timestamp,
41 #[serde(default)]
43 pub unverified_sync: bool,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct Authorization {
49 #[serde(default)]
50 pub scope: Option<Scope>,
51 #[serde(default)]
53 pub request: Option<String>,
54 #[serde(default)]
56 pub decision: Option<Timestamp>,
57 #[serde(default)]
59 pub question: Option<String>,
60 #[serde(default)]
62 pub interval: bool,
63 #[serde(default)]
64 pub invalidated: Option<String>,
65 pub task: String,
67 pub run: String,
69 pub stop: Stop,
71 pub at: Timestamp,
73 pub consumed: bool,
75 #[serde(default)]
77 pub generation: Option<u64>,
78 pub reviews: Vec<ReviewRound>,
80 pub gate: Vec<CommandOutcome>,
82 pub gate_head: Option<String>,
84 pub gate_ran: bool,
86}
87
88impl Authorization {
89 pub(crate) fn refuse(&mut self, reason: impl Into<String>) {
91 self.invalidated = Some(reason.into());
92 self.consumed = true;
93 }
94}
95
96#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct Episode {
99 pub id: usize,
101 pub reason: String,
103 pub used: usize,
105 pub synced_head: Option<String>,
107 #[serde(default)]
109 pub last_sync_head: Option<String>,
110 pub boundary: Option<Boundary>,
112 #[serde(default)]
114 pub prior_boundaries: Vec<Boundary>,
115}
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct Boundary {
120 pub pass: usize,
121 pub at: Timestamp,
122 pub head: String,
123 pub consumed: bool,
124}
125
126#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct Operation {
129 pub id: usize,
130 pub episode: usize,
131 pub head: String,
132 pub tip: String,
133 pub finished: bool,
134 #[serde(default)]
136 pub validated_head: Option<String>,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize)]
141pub struct Renewal {
142 pub task: String,
143 pub run: String,
144 pub generation: u64,
145 pub instruction: String,
146 pub episode: usize,
147 pub boundary_pass: usize,
148 pub head: String,
149 pub tip: String,
150}
151
152#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct Scope {
155 pub tip: String,
156 pub instruction: String,
157 pub episode: usize,
158}
159
160#[derive(Debug, Clone, Default, Serialize, Deserialize)]
161pub struct Recovery {
163 #[serde(default)]
165 pub episodes: Vec<Episode>,
166 #[serde(default)]
167 pub operations: Vec<Operation>,
168 #[serde(default)]
170 pub stops: Vec<Stop>,
171 #[serde(default)]
172 pub renewals: Vec<Renewal>,
173 pub stop: Option<Stop>,
175 pub authorizations: Vec<Authorization>,
177}
178
179impl RunState {
180 pub fn base_sync_next_action(&self) -> String {
182 let episode = self.base_sync_recovery.episodes.last().map_or(0, |e| e.id);
183 let lifetime = self.base_sync.as_ref().map_or(0, |s| s.attempts);
184 format!(
185 "{} ({}/{} in episode {}, {} lifetime synchronizations)",
186 self.base_sync_action(),
187 self.base_episode_used(),
188 crate::graph::BASE_SYNC_ROUNDS,
189 episode,
190 lifetime
191 )
192 }
193
194 fn base_sync_action(&self) -> String {
195 if let Some(reason) = self
196 .base_sync_recovery
197 .authorizations
198 .last()
199 .and_then(|a| a.invalidated.as_ref())
200 {
201 return format!("Approval invalidated: {reason}");
202 }
203 if self.resource_block.is_some() {
204 return "Restore the required resource, then explicitly resume the retained run; automatic retries retain the synchronization budget".into();
205 }
206 if self.base_sync_recovery.stop.is_none()
207 && let Some(op) = self
208 .base_sync_recovery
209 .operations
210 .last()
211 .filter(|o| !o.finished)
212 {
213 return format!(
214 "Resume reserved synchronization {} against base {}; inspect retained Git state if reconciliation is refused",
215 op.id, op.tip
216 );
217 }
218 match self.base_sync_recovery.stop.as_ref().map(|s| &s.kind) {
219 Some(StopKind::Exhausted) => {
220 let why = if self.base_sync_recovery.stop.as_ref().is_some_and(|s| s.unverified_sync) {
221 "The synchronized HEAD never reached its gate before the base moved again. "
222 } else {
223 ""
224 };
225 if self.base_recovery_pending() {
226 format!("{why}Approval recorded; resume the retained run to open the approved bounded synchronization interval")
227 } else {
228 format!("{why}Authorize a new bounded synchronization interval with `magi run --resume {} --recover-base`, or, from a standing chat, `magi task control recover-base {} --request <unique-id> --head {}`", self.id, self.id, self.base_sync_recovery.stop.as_ref().unwrap().head)
229 }
230 },
231 Some(StopKind::Conflict) => {
232 let stop = self.base_sync_recovery.stop.as_ref().unwrap();
233 let task = self.origin.as_ref().and_then(|origin| origin.task.as_deref()).unwrap_or("<task>");
234 format!("Authorize a narrow history-preserving repair with `magi task grant-repair {task} --run {} --kind base-conflict --rounds 1 --head {} --tip {} --expect-pass {} --request <unique-id> --reason <reason> --instruction <approved-resolution>`; retain the original review criteria. Base-budget recovery cannot resolve a textual conflict", self.id, stop.head, stop.tip, self.passes.last().map_or(0, |p| p.index))
235 },
236 Some(StopKind::Divergence) => "Reconcile foreign remote branch commits before resuming; do not overwrite them".into(),
237 Some(StopKind::Publication) => "Inspect the retained publication failure and remote lease before resuming".into(),
238 Some(StopKind::ClosingIntegratedPr) => "Resume to retry closing the already integrated pull request".into(),
239 Some(StopKind::Unknown) => "Inspect the retained synchronization state before resuming; unknown evidence grants no new budget".into(),
240 None => "Continue verification on the current synchronized HEAD".into(),
241 }
242 }
243
244 pub(crate) fn ensure_base_episode(&mut self) {
245 if let Some(stop) = self.base_sync_recovery.stop.clone()
246 && !self.base_sync_recovery.stops.contains(&stop)
247 {
248 self.base_sync_recovery.stops.push(stop);
249 }
250 if self.base_sync_recovery.episodes.is_empty() {
251 let used = self.base_sync.as_ref().map_or(0, |s| s.attempts);
252 self.base_sync_recovery.episodes.push(Episode {
253 id: 0,
254 reason: "initial or conservative legacy migration".into(),
255 used,
256 synced_head: None,
257 last_sync_head: None,
258 boundary: None,
259 prior_boundaries: Vec::new(),
260 });
261 }
262 }
263
264 fn open_base_episode(&mut self, reason: &str) {
265 self.ensure_base_episode();
266 let id = self.base_sync_recovery.episodes.len();
267 self.base_sync_recovery.episodes.push(Episode {
268 id,
269 reason: reason.into(),
270 used: 0,
271 synced_head: None,
272 last_sync_head: None,
273 boundary: None,
274 prior_boundaries: Vec::new(),
275 });
276 }
277
278 pub(crate) fn base_episode_used(&self) -> usize {
279 self.base_sync_recovery.episodes.last().map_or_else(
280 || self.base_sync.as_ref().map_or(0, |s| s.attempts),
281 |e| e.used,
282 )
283 }
284
285 pub(crate) fn record_base_progress(&mut self, head: &str) {
286 self.ensure_base_episode();
287 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
288 if let Some(boundary) = episode.boundary.take() {
289 episode.prior_boundaries.push(boundary);
290 }
291 episode.synced_head = Some(head.into());
292 }
293
294 pub(crate) fn record_base_interruption(&mut self) {
296 let Some(block) = &self.resource_block else {
297 return;
298 };
299 if matches!(block.cause, crate::run::ResourceBlockCause::Unrecorded) {
300 return;
301 }
302 let Some(episode) = self.base_sync_recovery.episodes.last_mut() else {
303 return;
304 };
305 let Some(head) = episode.synced_head.clone() else {
306 return;
307 };
308 if episode.boundary.is_none() {
309 episode.boundary = Some(Boundary {
310 pass: self.passes.last().map_or(0, |p| p.index),
311 at: block.at,
312 head,
313 consumed: false,
314 });
315 }
316 }
317
318 pub(crate) fn renew_base_episode(
319 &mut self,
320 task: &str,
321 approval: &crate::queue::Resumption,
322 head: &str,
323 tip: &str,
324 ) -> bool {
325 if self
326 .base_sync_recovery
327 .renewals
328 .iter()
329 .any(|r| r.task == task && r.generation >= approval.generation)
330 || self
331 .base_sync_recovery
332 .authorizations
333 .iter()
334 .any(|a| a.task == task && a.generation.is_some_and(|g| g >= approval.generation))
335 {
336 return false;
337 }
338 let Some(e) = self.base_sync_recovery.episodes.last_mut() else {
339 return false;
340 };
341 let Some(b) = &mut e.boundary else {
342 return false;
343 };
344 if b.consumed || b.head != head || approval.at <= b.at {
345 return false;
346 }
347 b.consumed = true;
348 self.base_sync_recovery.renewals.push(Renewal {
349 task: task.into(),
350 run: self.id.clone(),
351 generation: approval.generation,
352 instruction: approval.instruction.clone(),
353 episode: e.id,
354 boundary_pass: b.pass,
355 head: head.into(),
356 tip: tip.into(),
357 });
358 self.open_base_episode("owner resumption after recorded infrastructure interruption");
359 true
360 }
361
362 pub(crate) fn reserve_base_operation(&mut self, head: &str, tip: &str) {
364 self.ensure_base_episode();
365 if self
366 .base_sync_recovery
367 .operations
368 .last()
369 .is_some_and(|o| !o.finished)
370 {
371 return;
372 }
373 let e = self.base_sync_recovery.episodes.last_mut().unwrap();
374 e.used += 1;
375 let id = self.base_sync_recovery.operations.len();
376 self.base_sync_recovery.operations.push(Operation {
377 id,
378 episode: e.id,
379 head: head.into(),
380 tip: tip.into(),
381 finished: false,
382 validated_head: None,
383 });
384 if let Some(sync) = &mut self.base_sync {
385 sync.attempts += 1;
386 }
387 }
388
389 pub(crate) fn finish_base_operation(&mut self, head: Option<&str>) {
390 if let Some(op) = self.base_sync_recovery.operations.last_mut() {
391 op.finished = true;
392 }
393 if let Some(head) = head {
394 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
395 if let Some(boundary) = episode.boundary.take() {
396 episode.prior_boundaries.push(boundary);
397 }
398 episode.synced_head = Some(head.into());
399 episode.last_sync_head = Some(head.into());
400 self.reconcile_verification_head(head);
401 }
402 }
403
404 pub(crate) fn base_sync_blocked(&self) -> bool {
405 self.base_sync
406 .as_ref()
407 .is_some_and(|s| s.conflict.is_some())
408 }
409
410 pub(crate) fn base_sync_closure_pending(&self) -> bool {
413 let Some(detail) = self
414 .base_sync
415 .as_ref()
416 .and_then(|sync| sync.conflict.as_ref())
417 else {
418 return false;
419 };
420 let Some(winner) = self.winner() else {
421 return false;
422 };
423 let prefix = format!(
425 "{} is already in {}/{}, but its pull request could not be closed (",
426 winner.branch, self.config.merge.remote, self.base_branch
427 );
428 detail.starts_with(&prefix) && detail.ends_with("); resume to retry")
429 }
430
431 pub(crate) fn record_base_sync_stop(&mut self, head: &str) {
434 let Some(sync) = self.base_sync.as_ref() else {
435 return;
436 };
437 let Some(detail) = sync.conflict.clone() else {
438 return;
439 };
440 if self
441 .base_sync_recovery
442 .stop
443 .as_ref()
444 .is_some_and(|s| s.head == head && s.tip == sync.tip && s.detail == detail)
445 {
446 self.status = RunStatus::Blocked;
447 return;
448 }
449 let exhausted = self.winner().is_some_and(|w| {
450 [
451 ("rebase(s)", "rebasing"),
452 ("synchronization(s)", "synchronizing"),
453 ].iter().any(|(noun, verb)| detail == format!(
454 "{} moved {} commit(s) ahead of {} after {} {}; {} again would only race it",
455 self.base_branch, sync.behind, w.branch, crate::graph::BASE_SYNC_ROUNDS, noun, verb
456 ))
457 });
458 let kind = if self.base_sync_closure_pending() {
459 StopKind::ClosingIntegratedPr
460 } else if sync.attempts >= crate::graph::BASE_SYNC_ROUNDS && exhausted {
461 StopKind::Exhausted
462 } else if detail.contains("not rebasing over") {
463 StopKind::Divergence
464 } else if detail.contains("refused the push")
465 || detail.contains("could not push")
466 || detail.contains("could not be pushed")
467 {
468 StopKind::Publication
469 } else if detail.contains("conflicts with") && detail.contains("did not rebase") {
470 StopKind::Conflict
471 } else {
472 StopKind::Unknown
473 };
474 let at = self
475 .events
476 .iter()
477 .rev()
478 .find(|event| event.node == "land" && event.message == detail)
479 .map_or(self.updated_at, |event| event.at);
480 let unverified_sync = sync.behind > 0
481 && self
482 .base_sync_recovery
483 .episodes
484 .last()
485 .is_some_and(|e| e.last_sync_head.as_deref() == Some(head))
486 && !(self.gate_ran && self.gate_head.as_deref() == Some(head));
487 let stop = Stop {
488 unverified_sync,
489 kind,
490 head: head.to_owned(),
491 tip: sync.tip.clone(),
492 attempts: sync.attempts,
493 detail,
494 at,
495 };
496 self.base_sync_recovery.stops.push(stop.clone());
497 self.base_sync_recovery.stop = Some(stop);
498 self.status = RunStatus::Blocked;
499 }
500
501 pub(crate) fn base_recovery_pending(&self) -> bool {
502 self.base_sync_recovery
503 .authorizations
504 .last()
505 .is_some_and(|a| {
506 !a.consumed
507 && a.invalidated.is_none()
508 && a.run == self.id
509 && self.base_sync_recovery.stop.as_ref() == Some(&a.stop)
510 })
511 }
512
513 pub(crate) fn authorize_base_sync(&mut self, task: &str) -> Result<()> {
514 let stop = self
515 .base_sync_recovery
516 .stop
517 .clone()
518 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
519 ensure!(
520 stop.kind == StopKind::Exhausted,
521 "only exhausted base movement permits recovery"
522 );
523 ensure!(
524 !self
525 .base_sync_recovery
526 .authorizations
527 .last()
528 .is_some_and(|a| !a.consumed && a.stop == stop),
529 "this stop already has a pending authorization; resume the existing request"
530 );
531 self.base_sync_recovery.authorizations.push(Authorization {
532 scope: None,
533 request: None,
534 decision: None,
535 question: None,
536 interval: true,
537 invalidated: None,
538 task: task.to_owned(),
539 run: self.id.clone(),
540 stop,
541 at: Timestamp::now(),
542 consumed: false,
543 generation: None,
544 reviews: Vec::new(),
545 gate: Vec::new(),
546 gate_head: None,
547 gate_ran: false,
548 });
549 Ok(())
550 }
551
552 pub(crate) fn authorize_base_sync_resumption(
553 &mut self,
554 task: &str,
555 approval: &crate::queue::Resumption,
556 ) -> Result<()> {
557 let stop = self
558 .base_sync_recovery
559 .stop
560 .as_ref()
561 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
562 ensure!(
563 approval.at > stop.at,
564 "approval predates the base-sync stop"
565 );
566 ensure!(
567 !self.base_sync_recovery.authorizations.iter().any(|a| {
568 a.task == task && a.generation.is_some_and(|g| g >= approval.generation)
569 }) && !self
570 .base_sync_recovery
571 .renewals
572 .iter()
573 .any(|r| r.task == task && r.generation >= approval.generation),
574 "operator resumption generation already used"
575 );
576 self.authorize_base_sync(task)?;
577 self.base_sync_recovery
578 .authorizations
579 .last_mut()
580 .unwrap()
581 .generation = Some(approval.generation);
582 Ok(())
583 }
584
585 pub(crate) fn consume_base_recovery(&mut self, head: &str) -> bool {
588 if !self.base_recovery_pending() {
589 return false;
590 }
591 let a = self.base_sync_recovery.authorizations.last_mut().unwrap();
592 a.consumed = true;
593 if a.stop.head != head {
594 a.refuse(format!(
595 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
596 a.stop.head
597 ));
598 return false;
599 }
600 a.reviews = self.reviews.clone();
601 a.gate = std::mem::take(&mut self.gate);
602 a.gate_head = self.gate_head.take();
603 a.gate_ran = std::mem::replace(&mut self.gate_ran, false);
604 let interval = a.interval;
605 self.base_sync.as_mut().unwrap().conflict = None;
606 self.open_base_episode("explicit exhausted-base recovery");
607 self.base_sync_recovery.stop = None;
608 if !interval {
609 self.base_sync_recovery.episodes.last_mut().unwrap().used =
610 crate::graph::BASE_SYNC_ROUNDS - 1;
611 }
612 true
613 }
614}
615
616pub(crate) fn held_task(queue: &crate::queue::Queue, task: &crate::queue::Task) -> bool {
619 held_run(queue, task).is_some()
620}
621
622pub(crate) fn held_run(queue: &crate::queue::Queue, task: &crate::queue::Task) -> Option<RunState> {
623 let home = queue.root().parent()?;
624 task.execution_runs()
625 .last()
626 .and_then(|id| RunState::load_under(id, home).ok())
627 .filter(|state| {
628 state.base_sync_blocked()
629 && !state.base_recovery_pending()
630 && !state.base_sync_closure_pending()
631 })
632}
633
634impl crate::queue::Task {
635 pub(crate) fn base_sync_notice(&mut self, state: &RunState) {
637 let Some(sync) = state
638 .base_sync
639 .as_ref()
640 .filter(|_| state.base_sync_blocked())
641 else {
642 return;
643 };
644 let stop = state.base_sync_recovery.stop.as_ref();
645 self.held_at = Some(stop.map_or(state.updated_at, |s| s.at));
646 self.notice_cause("base-sync-stop", Some(&state.id), None);
647 if let Some(cause) = &mut self.hold_notice_cause {
648 cause.target = Some(match stop {
649 Some(s) => format!("{}:{}:{}", s.head, s.tip, s.at),
650 None => format!("legacy:{}:{}", sync.tip, sync.attempts),
651 });
652 }
653 }
654}
655
656pub fn ensure_generic_release(
658 queue: &crate::queue::Queue,
659 task: &crate::queue::Task,
660) -> Result<()> {
661 if let Some(state) = task
662 .execution_runs()
663 .last()
664 .and_then(|id| RunState::load_under(id, queue.root().parent()?).ok())
665 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
666 {
667 anyhow::bail!(reason);
668 }
669 if let Some(state) = held_run(queue, task)
670 && state
671 .base_sync_recovery
672 .stop
673 .as_ref()
674 .is_some_and(|s| s.kind == StopKind::Conflict)
675 && !crate::repair_budget::conflict_pending(task, &state)
676 {
677 anyhow::bail!(
678 "generic release cannot authorize textual conflict editing. {}",
679 state.base_sync_next_action()
680 );
681 }
682 Ok(())
683}