1use anyhow::{Result, ensure};
3use jiff::Timestamp;
4use serde::{Deserialize, Serialize};
5
6use crate::run::{CommandOutcome, ReviewRound, RunState, RunStatus};
7
8#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9#[serde(rename_all = "snake_case")]
10pub enum StopKind {
12 Exhausted,
14 Conflict,
16 Publication,
18 Divergence,
20 ClosingIntegratedPr,
22 Unknown,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Stop {
29 pub kind: StopKind,
31 pub head: String,
33 pub tip: String,
35 pub attempts: usize,
37 pub detail: String,
39 pub at: Timestamp,
41 #[serde(default)]
43 pub unverified_sync: bool,
44}
45
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct Authorization {
49 #[serde(default)]
50 pub scope: Option<Scope>,
51 #[serde(default)]
53 pub request: Option<String>,
54 #[serde(default)]
56 pub decision: Option<Timestamp>,
57 #[serde(default)]
59 pub question: Option<String>,
60 #[serde(default)]
62 pub interval: bool,
63 #[serde(default)]
64 pub invalidated: Option<String>,
65 pub task: String,
67 pub run: String,
69 pub stop: Stop,
71 pub at: Timestamp,
73 pub consumed: bool,
75 #[serde(default)]
77 pub generation: Option<u64>,
78 pub reviews: Vec<ReviewRound>,
80 pub gate: Vec<CommandOutcome>,
82 pub gate_head: Option<String>,
84 pub gate_ran: bool,
86}
87
88impl Authorization {
89 pub(crate) fn refuse(&mut self, reason: impl Into<String>) {
91 self.invalidated = Some(reason.into());
92 self.consumed = true;
93 }
94}
95
96#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct Episode {
99 pub id: usize,
101 pub reason: String,
103 pub used: usize,
105 pub synced_head: Option<String>,
107 #[serde(default)]
109 pub last_sync_head: Option<String>,
110 pub boundary: Option<Boundary>,
112 #[serde(default)]
114 pub prior_boundaries: Vec<Boundary>,
115}
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct Boundary {
120 pub pass: usize,
121 pub at: Timestamp,
122 pub head: String,
123 pub consumed: bool,
124}
125
126#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct Operation {
129 pub id: usize,
130 pub episode: usize,
131 pub head: String,
132 pub tip: String,
133 pub finished: bool,
134 #[serde(default)]
136 pub validated_head: Option<String>,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize)]
141pub struct Renewal {
142 pub task: String,
143 pub run: String,
144 pub generation: u64,
145 pub instruction: String,
146 pub episode: usize,
147 pub boundary_pass: usize,
148 pub head: String,
149 pub tip: String,
150}
151
152#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct Scope {
155 pub tip: String,
156 pub instruction: String,
157 pub episode: usize,
158}
159
160#[derive(Debug, Clone, Default, Serialize, Deserialize)]
161pub struct Recovery {
163 #[serde(default)]
165 pub episodes: Vec<Episode>,
166 #[serde(default)]
167 pub operations: Vec<Operation>,
168 #[serde(default)]
170 pub stops: Vec<Stop>,
171 #[serde(default)]
172 pub renewals: Vec<Renewal>,
173 pub stop: Option<Stop>,
175 pub authorizations: Vec<Authorization>,
177}
178
179impl RunState {
180 pub fn base_sync_next_action(&self) -> String {
182 let episode = self.base_sync_recovery.episodes.last().map_or(0, |e| e.id);
183 let lifetime = self.base_sync.as_ref().map_or(0, |s| s.attempts);
184 format!(
185 "{} ({}/{} in episode {}, {} lifetime synchronizations)",
186 self.base_sync_action(),
187 self.base_episode_used(),
188 crate::graph::BASE_SYNC_ROUNDS,
189 episode,
190 lifetime
191 )
192 }
193
194 fn base_sync_action(&self) -> String {
195 if let Some(reason) = self
196 .base_sync_recovery
197 .authorizations
198 .last()
199 .and_then(|a| a.invalidated.as_ref())
200 {
201 return format!("Approval invalidated: {reason}");
202 }
203 if self.resource_block.is_some() {
204 return "Restore the required resource, then explicitly resume the retained run; automatic retries retain the synchronization budget".into();
205 }
206 if self.base_sync_recovery.stop.is_none()
207 && let Some(op) = self
208 .base_sync_recovery
209 .operations
210 .last()
211 .filter(|o| !o.finished)
212 {
213 return format!(
214 "Resume reserved synchronization {} against base {}; inspect retained Git state if reconciliation is refused",
215 op.id, op.tip
216 );
217 }
218 if self.base_sync_blocked()
219 && (self.base_sync_recovery.stop.is_none()
220 || self
221 .rebase_fixes
222 .iter()
223 .any(|r| r.stop.is_none() && !r.finished))
224 {
225 let task = self
226 .origin
227 .as_ref()
228 .and_then(|o| o.task.as_deref())
229 .unwrap_or("<task>");
230 return format!(
231 "Inspect legacy or missing structured stop evidence with `magi task inspect-base-stop {task} --run {}`; diagnostic-only textual stops require explicit evidence-bound base-conflict authorization, not movement budget recovery",
232 self.id
233 );
234 }
235 match self.base_sync_recovery.stop.as_ref().map(|s| &s.kind) {
236 Some(StopKind::Exhausted) => {
237 let why = if self.base_sync_recovery.stop.as_ref().is_some_and(|s| s.unverified_sync) {
238 "The synchronized HEAD never reached its gate before the base moved again. "
239 } else {
240 ""
241 };
242 if self.base_recovery_pending() {
243 format!("{why}Approval recorded; resume the retained run to open the approved bounded synchronization interval")
244 } else {
245 format!("{why}Authorize a new bounded synchronization interval with `magi run --resume {} --recover-base`, or, from a standing chat, `magi task control recover-base {} --request <unique-id> --head {}`", self.id, self.id, self.base_sync_recovery.stop.as_ref().unwrap().head)
246 }
247 },
248 Some(StopKind::Conflict) => {
249 let stop = self.base_sync_recovery.stop.as_ref().unwrap();
250 let task = self.origin.as_ref().and_then(|origin| origin.task.as_deref()).unwrap_or("<task>");
251 format!("Authorize a narrow history-preserving repair with `magi task grant-repair {task} --run {} --kind base-conflict --rounds 1 --head {} --tip {} --expect-pass {} --request <unique-id> --reason <reason> --instruction <approved-resolution>`; retain the original review criteria. Base-budget recovery cannot resolve a textual conflict", self.id, stop.head, stop.tip, self.passes.last().map_or(0, |p| p.index))
252 },
253 Some(StopKind::Divergence) => "Reconcile foreign remote branch commits before resuming; do not overwrite them".into(),
254 Some(StopKind::Publication) => "Inspect the retained publication failure and remote lease before resuming".into(),
255 Some(StopKind::ClosingIntegratedPr) => "Resume to retry closing the already integrated pull request".into(),
256 Some(StopKind::Unknown) => "Inspect the retained synchronization state before resuming; unknown evidence grants no new budget".into(),
257 None => "Continue verification on the current synchronized HEAD".into(),
258 }
259 }
260
261 pub(crate) fn ensure_base_episode(&mut self) {
262 if let Some(stop) = self.base_sync_recovery.stop.clone()
263 && !self.base_sync_recovery.stops.contains(&stop)
264 {
265 self.base_sync_recovery.stops.push(stop);
266 }
267 if self.base_sync_recovery.episodes.is_empty() {
268 let used = self.base_sync.as_ref().map_or(0, |s| s.attempts);
269 self.base_sync_recovery.episodes.push(Episode {
270 id: 0,
271 reason: "initial or conservative legacy migration".into(),
272 used,
273 synced_head: None,
274 last_sync_head: None,
275 boundary: None,
276 prior_boundaries: Vec::new(),
277 });
278 }
279 }
280
281 fn open_base_episode(&mut self, reason: &str) {
282 self.ensure_base_episode();
283 let id = self.base_sync_recovery.episodes.len();
284 self.base_sync_recovery.episodes.push(Episode {
285 id,
286 reason: reason.into(),
287 used: 0,
288 synced_head: None,
289 last_sync_head: None,
290 boundary: None,
291 prior_boundaries: Vec::new(),
292 });
293 }
294
295 pub(crate) fn base_episode_used(&self) -> usize {
296 self.base_sync_recovery.episodes.last().map_or_else(
297 || self.base_sync.as_ref().map_or(0, |s| s.attempts),
298 |e| e.used,
299 )
300 }
301
302 pub(crate) fn record_base_progress(&mut self, head: &str) {
303 self.ensure_base_episode();
304 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
305 if let Some(boundary) = episode.boundary.take() {
306 episode.prior_boundaries.push(boundary);
307 }
308 episode.synced_head = Some(head.into());
309 }
310
311 pub(crate) fn record_base_interruption(&mut self) {
313 let Some(block) = &self.resource_block else {
314 return;
315 };
316 if matches!(block.cause, crate::run::ResourceBlockCause::Unrecorded) {
317 return;
318 }
319 let Some(episode) = self.base_sync_recovery.episodes.last_mut() else {
320 return;
321 };
322 let Some(head) = episode.synced_head.clone() else {
323 return;
324 };
325 if episode.boundary.is_none() {
326 episode.boundary = Some(Boundary {
327 pass: self.passes.last().map_or(0, |p| p.index),
328 at: block.at,
329 head,
330 consumed: false,
331 });
332 }
333 }
334
335 pub(crate) fn renew_base_episode(
336 &mut self,
337 task: &str,
338 approval: &crate::queue::Resumption,
339 head: &str,
340 tip: &str,
341 ) -> bool {
342 if self
343 .base_sync_recovery
344 .renewals
345 .iter()
346 .any(|r| r.task == task && r.generation >= approval.generation)
347 || self
348 .base_sync_recovery
349 .authorizations
350 .iter()
351 .any(|a| a.task == task && a.generation.is_some_and(|g| g >= approval.generation))
352 {
353 return false;
354 }
355 let Some(e) = self.base_sync_recovery.episodes.last_mut() else {
356 return false;
357 };
358 let Some(b) = &mut e.boundary else {
359 return false;
360 };
361 if b.consumed || b.head != head || approval.at <= b.at {
362 return false;
363 }
364 b.consumed = true;
365 self.base_sync_recovery.renewals.push(Renewal {
366 task: task.into(),
367 run: self.id.clone(),
368 generation: approval.generation,
369 instruction: approval.instruction.clone(),
370 episode: e.id,
371 boundary_pass: b.pass,
372 head: head.into(),
373 tip: tip.into(),
374 });
375 self.open_base_episode("owner resumption after recorded infrastructure interruption");
376 true
377 }
378
379 pub(crate) fn reserve_base_operation(&mut self, head: &str, tip: &str) {
381 self.ensure_base_episode();
382 if self
383 .base_sync_recovery
384 .operations
385 .last()
386 .is_some_and(|o| !o.finished)
387 {
388 return;
389 }
390 let e = self.base_sync_recovery.episodes.last_mut().unwrap();
391 e.used += 1;
392 let id = self.base_sync_recovery.operations.len();
393 self.base_sync_recovery.operations.push(Operation {
394 id,
395 episode: e.id,
396 head: head.into(),
397 tip: tip.into(),
398 finished: false,
399 validated_head: None,
400 });
401 if let Some(sync) = &mut self.base_sync {
402 sync.attempts += 1;
403 }
404 }
405
406 pub(crate) fn finish_base_operation(&mut self, head: Option<&str>) {
407 if let Some(op) = self.base_sync_recovery.operations.last_mut() {
408 op.finished = true;
409 }
410 if let Some(head) = head {
411 let episode = self.base_sync_recovery.episodes.last_mut().unwrap();
412 if let Some(boundary) = episode.boundary.take() {
413 episode.prior_boundaries.push(boundary);
414 }
415 episode.synced_head = Some(head.into());
416 episode.last_sync_head = Some(head.into());
417 self.reconcile_verification_head(head);
418 }
419 }
420
421 pub(crate) fn base_sync_blocked(&self) -> bool {
422 self.base_sync
423 .as_ref()
424 .is_some_and(|s| s.conflict.is_some())
425 }
426
427 pub(crate) fn base_sync_closure_pending(&self) -> bool {
430 let Some(detail) = self
431 .base_sync
432 .as_ref()
433 .and_then(|sync| sync.conflict.as_ref())
434 else {
435 return false;
436 };
437 let Some(winner) = self.winner() else {
438 return false;
439 };
440 let prefix = format!(
442 "{} is already in {}/{}, but its pull request could not be closed (",
443 winner.branch, self.config.merge.remote, self.base_branch
444 );
445 detail.starts_with(&prefix) && detail.ends_with("); resume to retry")
446 }
447
448 pub(crate) fn record_base_sync_stop(&mut self, head: &str) {
451 let Some(sync) = self.base_sync.as_ref() else {
452 return;
453 };
454 let Some(detail) = sync.conflict.clone() else {
455 return;
456 };
457 if self
458 .base_sync_recovery
459 .stop
460 .as_ref()
461 .is_some_and(|s| s.head == head && s.tip == sync.tip && s.detail == detail)
462 {
463 self.status = RunStatus::Blocked;
464 return;
465 }
466 let exhausted = self.winner().is_some_and(|w| {
467 [
468 ("rebase(s)", "rebasing"),
469 ("synchronization(s)", "synchronizing"),
470 ].iter().any(|(noun, verb)| detail == format!(
471 "{} moved {} commit(s) ahead of {} after {} {}; {} again would only race it",
472 self.base_branch, sync.behind, w.branch, crate::graph::BASE_SYNC_ROUNDS, noun, verb
473 ))
474 });
475 let kind = if self.base_sync_closure_pending() {
476 StopKind::ClosingIntegratedPr
477 } else if sync.attempts >= crate::graph::BASE_SYNC_ROUNDS && exhausted {
478 StopKind::Exhausted
479 } else if detail.contains("not rebasing over") {
480 StopKind::Divergence
481 } else if detail.contains("refused the push")
482 || detail.contains("could not push")
483 || detail.contains("could not be pushed")
484 {
485 StopKind::Publication
486 } else if detail.contains("conflicts with") && detail.contains("did not rebase") {
487 StopKind::Conflict
488 } else {
489 StopKind::Unknown
490 };
491 let at = self
492 .events
493 .iter()
494 .rev()
495 .find(|event| event.node == "land" && event.message == detail)
496 .map_or(self.updated_at, |event| event.at);
497 let unverified_sync = sync.behind > 0
498 && self
499 .base_sync_recovery
500 .episodes
501 .last()
502 .is_some_and(|e| e.last_sync_head.as_deref() == Some(head))
503 && !(self.gate_ran && self.gate_head.as_deref() == Some(head));
504 let stop = Stop {
505 unverified_sync,
506 kind,
507 head: head.to_owned(),
508 tip: sync.tip.clone(),
509 attempts: sync.attempts,
510 detail,
511 at,
512 };
513 self.base_sync_recovery.stops.push(stop.clone());
514 self.base_sync_recovery.stop = Some(stop);
515 self.status = RunStatus::Blocked;
516 }
517
518 pub(crate) fn base_recovery_pending(&self) -> bool {
519 self.base_sync_recovery
520 .authorizations
521 .last()
522 .is_some_and(|a| {
523 !a.consumed
524 && a.invalidated.is_none()
525 && a.run == self.id
526 && self.base_sync_recovery.stop.as_ref() == Some(&a.stop)
527 })
528 }
529
530 pub(crate) fn authorize_base_sync(&mut self, task: &str) -> Result<()> {
531 let stop = self
532 .base_sync_recovery
533 .stop
534 .clone()
535 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
536 ensure!(
537 stop.kind == StopKind::Exhausted,
538 "only exhausted base movement permits recovery"
539 );
540 ensure!(
541 !self
542 .base_sync_recovery
543 .authorizations
544 .last()
545 .is_some_and(|a| !a.consumed && a.stop == stop),
546 "this stop already has a pending authorization; resume the existing request"
547 );
548 self.base_sync_recovery.authorizations.push(Authorization {
549 scope: None,
550 request: None,
551 decision: None,
552 question: None,
553 interval: true,
554 invalidated: None,
555 task: task.to_owned(),
556 run: self.id.clone(),
557 stop,
558 at: Timestamp::now(),
559 consumed: false,
560 generation: None,
561 reviews: Vec::new(),
562 gate: Vec::new(),
563 gate_head: None,
564 gate_ran: false,
565 });
566 Ok(())
567 }
568
569 pub(crate) fn authorize_base_sync_resumption(
570 &mut self,
571 task: &str,
572 approval: &crate::queue::Resumption,
573 ) -> Result<()> {
574 let stop = self
575 .base_sync_recovery
576 .stop
577 .as_ref()
578 .ok_or_else(|| anyhow::anyhow!("no recorded base-sync stop"))?;
579 ensure!(
580 approval.at > stop.at,
581 "approval predates the base-sync stop"
582 );
583 ensure!(
584 !self.base_sync_recovery.authorizations.iter().any(|a| {
585 a.task == task && a.generation.is_some_and(|g| g >= approval.generation)
586 }) && !self
587 .base_sync_recovery
588 .renewals
589 .iter()
590 .any(|r| r.task == task && r.generation >= approval.generation),
591 "operator resumption generation already used"
592 );
593 self.authorize_base_sync(task)?;
594 self.base_sync_recovery
595 .authorizations
596 .last_mut()
597 .unwrap()
598 .generation = Some(approval.generation);
599 Ok(())
600 }
601
602 pub(crate) fn consume_base_recovery(&mut self, head: &str) -> bool {
605 if !self.base_recovery_pending() {
606 return false;
607 }
608 let a = self.base_sync_recovery.authorizations.last_mut().unwrap();
609 a.consumed = true;
610 if a.stop.head != head {
611 a.refuse(format!(
612 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
613 a.stop.head
614 ));
615 return false;
616 }
617 a.reviews = self.reviews.clone();
618 a.gate = std::mem::take(&mut self.gate);
619 a.gate_head = self.gate_head.take();
620 a.gate_ran = std::mem::replace(&mut self.gate_ran, false);
621 let interval = a.interval;
622 self.base_sync.as_mut().unwrap().conflict = None;
623 self.open_base_episode("explicit exhausted-base recovery");
624 self.base_sync_recovery.stop = None;
625 if !interval {
626 self.base_sync_recovery.episodes.last_mut().unwrap().used =
627 crate::graph::BASE_SYNC_ROUNDS - 1;
628 }
629 true
630 }
631}
632
633pub(crate) fn held_task(queue: &crate::queue::Queue, task: &crate::queue::Task) -> bool {
636 held_run(queue, task).is_some()
637}
638
639pub(crate) fn held_run(queue: &crate::queue::Queue, task: &crate::queue::Task) -> Option<RunState> {
640 let home = queue.root().parent()?;
641 task.execution_runs()
642 .last()
643 .and_then(|id| RunState::load_under(id, home).ok())
644 .filter(|state| {
645 state.base_sync_blocked()
646 && !state.base_recovery_pending()
647 && !state.base_sync_closure_pending()
648 })
649}
650
651impl crate::queue::Task {
652 pub(crate) fn base_sync_notice(&mut self, state: &RunState) {
654 let Some(sync) = state
655 .base_sync
656 .as_ref()
657 .filter(|_| state.base_sync_blocked())
658 else {
659 return;
660 };
661 let stop = state.base_sync_recovery.stop.as_ref();
662 self.held_at = Some(stop.map_or(state.updated_at, |s| s.at));
663 self.notice_cause("base-sync-stop", Some(&state.id), None);
664 if let Some(cause) = &mut self.hold_notice_cause {
665 cause.target = Some(match stop {
666 Some(s) => format!("{}:{}:{}", s.head, s.tip, s.at),
667 None => format!("legacy:{}:{}", sync.tip, sync.attempts),
668 });
669 }
670 }
671}
672
673pub fn ensure_generic_release(
675 queue: &crate::queue::Queue,
676 task: &crate::queue::Task,
677) -> Result<()> {
678 if let Some(state) = task
679 .execution_runs()
680 .last()
681 .and_then(|id| RunState::load_under(id, queue.root().parent()?).ok())
682 && let Some(reason) = crate::repair_budget::conflict_failure(task, &state)
683 {
684 anyhow::bail!(reason);
685 }
686 if let Some(state) = held_run(queue, task)
687 && state
688 .base_sync_recovery
689 .stop
690 .as_ref()
691 .is_some_and(|s| s.kind == StopKind::Conflict)
692 && !crate::repair_budget::conflict_pending(task, &state)
693 {
694 anyhow::bail!(
695 "generic release cannot authorize textual conflict editing. {}",
696 state.base_sync_next_action()
697 );
698 }
699 Ok(())
700}