1pub mod reconcile;
4
5use crate::{
6 git,
7 proc::Quiet as _,
8 run::{Liveness, RunState},
9};
10use anyhow::{Context, Result, ensure};
11use jiff::Timestamp;
12use serde::{Deserialize, Serialize};
13use std::{
14 fs::{File, OpenOptions},
15 path::{Path, PathBuf},
16};
17
18#[derive(Debug)]
22pub(crate) struct OutputLease {
23 _file: File,
24}
25
26impl OutputLease {
27 #[cfg(unix)]
28 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
29 use std::os::fd::AsRawFd;
30 self._file.as_raw_fd()
31 }
32
33 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
34 if !worktree.join(".git").is_file() {
35 return Ok(None);
36 }
37 Self::take(worktree, false).map(Some)
38 }
39 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
40 use std::hash::{Hash, Hasher};
41 let mut hash = std::collections::hash_map::DefaultHasher::new();
42 let mut normalized = cache.to_owned();
43 let mut missing = Vec::new();
44 while !normalized.exists() {
45 let Some(name) = normalized.file_name() else {
46 break;
47 };
48 missing.push(name.to_owned());
49 if !normalized.pop() {
50 break;
51 }
52 }
53 normalized = normalized.canonicalize().unwrap_or(normalized);
54 for part in missing.into_iter().rev() {
55 normalized.push(part);
56 }
57 normalized.hash(&mut hash);
58 let dir = home.join("cache-leases");
59 std::fs::create_dir_all(&dir)?;
60 Self::lock(
61 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
62 exclusive,
63 )
64 }
65 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
66 let out = std::process::Command::new("git")
67 .args(["rev-parse", "--absolute-git-dir"])
68 .current_dir(worktree)
69 .quiet()
70 .output()?;
71 ensure!(
72 out.status.success(),
73 "cannot identify output lease directory"
74 );
75 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
76 Self::lock(&dir.join("magi-output.lock"), exclusive)
77 }
78 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
79 let file = OpenOptions::new()
80 .create(true)
81 .truncate(false)
82 .read(true)
83 .write(true)
84 .open(path)?;
85 #[cfg(unix)]
86 {
87 use std::os::fd::AsRawFd;
88 let mode = if exclusive {
89 libc::LOCK_EX
90 } else {
91 libc::LOCK_SH
92 };
93 ensure!(
95 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
96 "worktree outputs are in use or being reclaimed"
97 );
98 }
99 #[cfg(not(unix))]
100 let _ = exclusive;
101 Ok(Self { _file: file })
102 }
103}
104#[cfg(unix)]
105impl Drop for OutputLease {
106 fn drop(&mut self) {
107 use std::os::fd::AsRawFd;
108 unsafe {
110 libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
111 }
112 }
113}
114
115#[derive(Default, Debug, Serialize, Deserialize)]
116#[serde(default)]
117pub(crate) struct Report {
118 pub checked_at: Option<Timestamp>,
119 pub limit_bytes: u64,
120 pub reclaim_budget_bytes: u64,
121 pub measured_bytes: u64,
122 pub freed_bytes: u64,
123 pub retired: usize,
124 pub retained: Vec<String>,
125 pub failures: Vec<String>,
126}
127
128fn plain_dir(path: &Path) -> bool {
129 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
130}
131
132fn run_paths_missing(state: &RunState) -> Result<bool> {
135 for path in [state.worktree_root(), state.repo.clone()] {
136 match std::fs::metadata(&path) {
137 Ok(_) => {}
138 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
139 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
140 }
141 }
142 Ok(false)
143}
144
145async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
147 let root = state.worktree_root();
148 if !plain_dir(&root) {
149 return Ok(Vec::new());
150 }
151 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
152 registered_worktrees(state, &list)
153}
154
155fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
156 let root = state.worktree_root();
157 if !plain_dir(&root) {
158 return Ok(Vec::new());
159 }
160 let root = root.canonicalize()?;
161 let mut paths = Vec::new();
162 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
163 let path = PathBuf::from(path);
164 if plain_dir(&path)
165 && path.parent() == Some(root.as_path())
166 && path.canonicalize()? == path
167 && path.join(".git").is_file()
168 {
169 paths.push(path);
170 }
171 }
172 Ok(paths)
173}
174
175async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
178 let target = worktree.join("target");
179 ensure!(plain_dir(&target), "target absent or not a plain directory");
180 ensure!(target.canonicalize()? == target, "target path changed");
181 ensure!(
182 git::git(worktree, &["ls-files", "--", "target"])
183 .await?
184 .is_empty(),
185 "tracked target contents"
186 );
187 ensure!(
188 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
189 .await?
190 .ok(),
191 "target is not ignored"
192 );
193 for name in ["CACHEDIR.TAG", ".rustc_info.json"] {
194 let path = target.join(name);
195 ensure!(
196 std::fs::symlink_metadata(&path)?.is_file() && path.canonicalize()? == path,
197 "cache identity is not a plain file"
198 );
199 }
200 let tag = std::fs::read_to_string(target.join("CACHEDIR.TAG"))?;
201 ensure!(
202 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55d"),
203 "unknown Cargo cache tag"
204 );
205 let info: serde_json::Value =
206 serde_json::from_slice(&std::fs::read(target.join(".rustc_info.json"))?)?;
207 ensure!(
208 info.get("rustc_fingerprint").is_some(),
209 "unknown Cargo identity"
210 );
211 for entry in std::fs::read_dir(&target)? {
212 let entry = entry?;
213 let name = entry.file_name();
214 let name = name.to_str().context("non-UTF8 output")?;
215 ensure!(
216 matches!(
217 name,
218 "debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json"
219 ),
220 "unrecognized target entry {name}"
221 );
222 if matches!(name, "debug" | "release") {
223 ensure!(
224 plain_dir(&entry.path()) && entry.path().canonicalize()? == entry.path(),
225 "profile is not a plain directory"
226 );
227 }
228 }
229 Ok(target)
230}
231
232fn artifact_signature(path: &Path) -> Result<bool> {
233 use std::io::Read as _;
234 let mut header = [0; 8];
235 let n = File::open(path)?.read(&mut header)?;
236 let header = &header[..n];
237 Ok(header.starts_with(b"\x7fELF")
238 || header.starts_with(b"!<arch>\n")
239 || header.starts_with(b"rust\0\0\0")
240 || header.starts_with(b"MZ\x90\0")
241 || [
242 [0xfe, 0xed, 0xfa, 0xce],
243 [0xce, 0xfa, 0xed, 0xfe],
244 [0xfe, 0xed, 0xfa, 0xcf],
245 [0xcf, 0xfa, 0xed, 0xfe],
246 [0xca, 0xfe, 0xba, 0xbe],
247 [0xca, 0xfe, 0xba, 0xbf],
248 ]
249 .iter()
250 .any(|magic| header.starts_with(magic)))
251}
252
253fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
257 let mut outputs = std::collections::BTreeSet::new();
258 for profile in ["debug", "release"] {
259 for dir in [target.join(profile), target.join(profile).join("deps")] {
260 if !plain_dir(&dir) || dir.canonicalize()? != dir {
261 continue;
262 }
263 for entry in std::fs::read_dir(&dir)? {
264 let path = entry?.path();
265 if path.extension().is_none_or(|ext| ext != "d")
266 || !std::fs::symlink_metadata(&path)?.is_file()
267 || std::fs::metadata(&path)?.len() > 1024 * 1024
268 || path.canonicalize()? != path
269 {
270 continue;
271 }
272 let body = std::fs::read_to_string(&path)?;
273 for line in body.lines() {
274 let Some((names, dependencies)) = line.split_once(": ") else {
275 continue;
276 };
277 if dependencies.trim().is_empty() {
278 continue;
279 }
280 let mut words = Vec::new();
283 let mut word = String::new();
284 let mut escaped = false;
285 for c in names.chars() {
286 if escaped {
287 word.push(c);
288 escaped = false;
289 } else if c == '\\' {
290 escaped = true;
291 } else if c.is_whitespace() {
292 if !word.is_empty() {
293 words.push(std::mem::take(&mut word));
294 }
295 } else {
296 word.push(c);
297 }
298 }
299 if escaped || word.contains(['$', '%']) {
300 continue;
301 }
302 if !word.is_empty() {
303 words.push(word);
304 }
305 for name in words {
306 let output = PathBuf::from(name);
307 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
308 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
309 let artifact_name =
310 output_stem == stem || output_stem == format!("lib{stem}");
311 let artifact_kind = output.extension().is_none_or(|ext| {
312 matches!(
313 ext.to_str(),
314 Some(
315 "rlib"
316 | "rmeta"
317 | "so"
318 | "dylib"
319 | "a"
320 | "lib"
321 | "dll"
322 | "exe"
323 | "pdb"
324 )
325 )
326 });
327 if artifact_name
328 && artifact_kind
329 && output.parent() == Some(dir.as_path())
330 && output.extension().is_none_or(|ext| ext != "d")
331 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
332 && output.canonicalize().is_ok_and(|p| p == output)
333 && artifact_signature(&output)?
334 {
335 outputs.insert(output);
336 }
337 }
338 }
339 }
340 }
341 }
342 Ok(outputs.into_iter().collect())
343}
344
345#[derive(Serialize, Deserialize)]
346struct CargoResidue {
347 target: PathBuf,
348 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
349}
350
351fn residue_files(
354 target: &Path,
355 outputs: &[PathBuf],
356) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
357 let mut files = std::collections::BTreeMap::new();
358 let mut dirs = vec![target.to_owned()];
359 let mut bytes = 0usize;
360 let mut entries = 0usize;
361 while let Some(dir) = dirs.pop() {
362 for entry in std::fs::read_dir(dir)? {
363 let path = entry?.path();
364 entries += 1;
365 ensure!(entries <= 4096, "residue inventory too large");
366 let metadata = std::fs::symlink_metadata(&path)?;
367 ensure!(
368 !metadata.file_type().is_symlink(),
369 "symlink in Cargo residue"
370 );
371 if metadata.is_dir() {
372 dirs.push(path);
373 } else {
374 ensure!(metadata.is_file(), "non-file in Cargo residue");
375 if outputs.contains(&path) {
376 continue;
377 }
378 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
379 let body = std::fs::read(&path)?;
380 bytes = bytes.saturating_add(body.len());
381 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
382 files.insert(path.strip_prefix(target)?.to_owned(), body);
383 }
384 }
385 }
386 Ok(files)
387}
388
389fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
390 Ok(home
391 .join("runs")
392 .join(id)
393 .join("build-residue")
394 .join(worktree.file_name().context("missing worktree name")?))
395}
396
397fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
398 let files = residue_files(target, outputs)?;
399 for (relative, body) in &files {
400 let path = target.join(relative);
401 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
402 let lock = matches!(
403 relative.to_str(),
404 Some("debug/.cargo-lock" | "release/.cargo-lock")
405 ) && body.is_empty();
406 let dep_info = path.extension().is_some_and(|ext| ext == "d")
407 && outputs.iter().any(|output| {
408 output.parent() == path.parent()
409 && (output.file_stem() == path.file_stem()
410 || output.file_stem().is_some_and(|stem| {
411 stem.to_string_lossy()
412 == format!(
413 "lib{}",
414 path.file_stem().unwrap_or_default().to_string_lossy()
415 )
416 }))
417 });
418 ensure!(identity || lock || dep_info, "unproven Cargo residue");
419 }
420 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
421 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
422 let tmp = proof.with_extension("tmp");
423 std::fs::write(
424 &tmp,
425 serde_json::to_vec(&CargoResidue {
426 target: target.to_owned(),
427 files,
428 })?,
429 )?;
430 std::fs::rename(tmp, proof)?;
431 Ok(())
432}
433
434pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
435 let target = cargo_target(worktree).await?;
436 let locks = cargo_locks(&target)?;
437 let proof: CargoResidue =
438 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
439 ensure!(
440 proof.target == target && proof.files == residue_files(&target, &[])?,
441 "Cargo residue changed or contains unproven files"
442 );
443 Ok(locks)
444}
445
446pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
448 let mut held = Vec::new();
449 for profile in ["debug", "release"] {
450 let lock = target.join(profile).join(".cargo-lock");
451 if lock.exists() {
452 ensure!(
453 std::fs::symlink_metadata(&lock)?.is_file() && lock.canonicalize()? == lock,
454 "Cargo lock is not a plain file"
455 );
456 let file = OpenOptions::new().read(true).write(true).open(lock)?;
457 crate::pr_head::try_lock_file(&file).context("Cargo build is active")?;
458 held.push(file);
459 }
460 }
461 Ok(held)
462}
463
464pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
467 let home = home.to_owned();
468 let disk = disk.clone();
469 let runtime = tokio::runtime::Handle::current();
470 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
471 .await
472 .unwrap_or_else(|e| Report {
473 failures: vec![format!("retention worker failed: {e}")],
474 ..Report::default()
475 })
476}
477
478#[derive(Default, Serialize, Deserialize)]
479struct Measurement {
480 at: Option<Timestamp>,
481 bytes: u64,
482}
483
484async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
485 let mut report = Report {
486 checked_at: Some(now),
487 limit_bytes: disk.worktree_limit_bytes,
488 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
489 ..Report::default()
490 };
491 if !cfg!(unix) {
492 report
493 .failures
494 .push("Automatic output reclamation requires Unix process containment".into());
495 return report;
496 }
497 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
498 return report;
499 };
500 let mut targets = Vec::new();
501 let mut registrations = std::collections::HashMap::new();
502 let measurements_path = home.join("output-measurements.json");
503 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
504 std::fs::read(&measurements_path)
505 .ok()
506 .and_then(|b| serde_json::from_slice(&b).ok())
507 .unwrap_or_default();
508 let mut observed = std::collections::BTreeSet::new();
509 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
510 .into_iter()
511 .flatten()
512 .flatten()
513 .map(|e| e.file_name().to_string_lossy().into_owned())
514 .collect();
515 ids.sort();
516 for id in ids {
517 if !crate::run::is_run_id(&id) {
518 continue;
519 }
520 let state = match RunState::load_under(&id, home) {
521 Ok(state) => state,
522 Err(e) => {
523 report
524 .retained
525 .push(format!("{id}: unreadable run ownership: {e:#}"));
526 continue;
527 }
528 };
529 if disk.worktree_limit_bytes == 0 {
530 continue;
531 }
532 match run_paths_missing(&state) {
533 Ok(true) => continue,
534 Ok(false) => {}
535 Err(e) => {
536 report.failures.push(format!("{id}: {e:#}"));
537 continue;
538 }
539 }
540 let list = match registrations.entry(state.repo.clone()) {
541 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
542 std::collections::hash_map::Entry::Vacant(entry) => {
543 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
544 Ok(list) => entry.insert(list),
545 Err(e) => {
546 if matches!(run_paths_missing(&state), Ok(true)) {
549 continue;
550 }
551 report.failures.push(format!("{id}: {e:#}"));
552 continue;
553 }
554 }
555 }
556 };
557 let paths = match registered_worktrees(&state, list) {
558 Ok(paths) => paths,
559 Err(e) => {
560 report.failures.push(format!("{id}: {e:#}"));
561 continue;
562 }
563 };
564 for path in paths {
565 if !path.join("target").exists() {
566 continue;
567 }
568 let target = path.join("target");
569 observed.insert(target.clone());
570 let pressure =
571 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
572 let measurement = measurements.entry(target).or_default();
573 if pressure
574 || measurement
575 .at
576 .is_none_or(|at| crate::clean::due(now, at, 300))
577 {
578 measurement.bytes = crate::disk::dir_size(&path.join("target"));
579 measurement.at = Some(now);
580 }
581 let size = measurement.bytes;
582 report.measured_bytes = report.measured_bytes.saturating_add(size);
583 targets.push((state.updated_at, id.clone(), path, size));
584 }
585 }
586 targets.sort_by_key(|t| t.0);
587 let mut remaining = report.measured_bytes;
588 for (updated, id, path, size) in targets {
589 if disk.worktree_limit_bytes == 0
590 || remaining <= disk.worktree_limit_bytes
591 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
592 {
593 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
594 continue;
595 }
596 if report.freed_bytes.saturating_add(size) > disk.worktree_reclaim_bytes {
597 report.retained.push(format!(
598 "{id}: target exceeds remaining per-pass byte budget"
599 ));
600 continue;
601 }
602 let result = async {
603 ensure!(
604 crate::clean::due(now, updated, disk.fold_grace_secs),
605 "within grace period"
606 );
607 let _owner = crate::pr_head::owner_lock(home, &id)?;
608 let state = RunState::load_under(&id, home)?;
609 ensure!(
610 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
611 == Liveness::Dead,
612 "run live or ownership unknown"
613 );
614 ensure!(crate::clean::due(now, state.updated_at, disk.fold_grace_secs), "recently used after inventory");
615 let _driver = crate::driver::maintenance(home, &id)?;
616 let _outputs = OutputLease::take(&path, true)?;
617 ensure!(
618 worktrees(&state).await?.contains(&path),
619 "worktree registration changed"
620 );
621 let target = cargo_target(&path).await?;
622 let _cargo = cargo_locks(&target)?;
623 let outputs = declared_outputs(&target)?;
624 ensure!(!outputs.is_empty(), "no dep-info-proven outputs; unknown files retained");
625 let current_size = outputs.iter().try_fold(0u64, |bytes, p| {
626 Ok::<_, std::io::Error>(bytes.saturating_add(std::fs::metadata(p)?.len()))
627 })?;
628 ensure!(report.freed_bytes.saturating_add(current_size) <= disk.worktree_reclaim_bytes,
629 "target grew beyond remaining per-pass budget");
630 let ledger = home.join("runs").join(&id).join("build-outputs.json");
633 std::fs::write(
634 ledger,
635 serde_json::to_vec(
636 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","files":outputs}),
637 )?,
638 )?;
639 if let Err(error) = record_residue(home, &id, &target, &outputs) {
642 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
643 }
644 for output in &outputs {
645 std::fs::remove_file(output)?;
646 }
647 measurements.remove(&target);
648 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
649 Ok::<_, anyhow::Error>(current_size)
650 }
651 .await;
652 match result {
653 Ok(freed) => {
654 report.freed_bytes = report.freed_bytes.saturating_add(freed);
655 remaining = remaining.saturating_sub(freed);
656 }
657 Err(e) => report.retained.push(format!(
658 "{id}/{}: {e:#}",
659 path.file_name().unwrap_or_default().to_string_lossy()
660 )),
661 }
662 }
663 measurements.retain(|path, _| observed.contains(path));
664 if let Ok(body) = serde_json::to_vec(&measurements)
665 && let Err(e) = std::fs::write(&measurements_path, body)
666 {
667 report.failures.push(format!("output measurements: {e}"));
668 }
669 retire_empty(home, disk, now, &mut report).await;
670 if !report.failures.is_empty()
671 || disk.worktree_limit_bytes > 0
672 && remaining > disk.worktree_limit_bytes
673 && report.freed_bytes == 0
674 {
675 crate::notices::raise_in(
676 home,
677 crate::notices::Notice::warn(
678 "disk:retention",
679 "Retained build outputs could not be brought under the disk cap. Inspect the cleanup report for active owners, retained recovery sources and failures.",
680 ),
681 );
682 }
683 let store = home.join("retention.json");
684 if let Ok(body) = serde_json::to_vec_pretty(&report) {
685 let tmp = home.join("retention.json.tmp");
686 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
687 tracing::warn!("retention report could not be saved: {e}");
688 }
689 }
690 tracing::info!(
691 measured_bytes = report.measured_bytes,
692 freed_bytes = report.freed_bytes,
693 retired = report.retired,
694 retained = report.retained.len(),
695 failures = report.failures.len(),
696 "retained worktree cleanup"
697 );
698 for reason in report.retained.iter().chain(&report.failures) {
699 tracing::debug!(%reason, "worktree retained");
700 }
701 report
702}
703
704async fn retire_empty(
705 home: &Path,
706 disk: &crate::config::Disk,
707 now: Timestamp,
708 report: &mut Report,
709) {
710 if !disk.auto_fold {
711 return;
712 }
713 let queue = crate::queue::Queue::at(home.join("queue"));
714 let questions = crate::ask::Questions::at(home.join("questions"));
715 let open = questions.list();
716 for listed in queue.list() {
717 let Ok(_claim) = queue.claim(&listed.id) else {
718 continue;
719 };
720 let Ok(task) = queue.get(&listed.id) else {
721 continue;
722 };
723 if open
724 .iter()
725 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
726 {
727 report.retained.push(format!(
728 "{}: task has an unanswered owner decision",
729 task.id
730 ));
731 continue;
732 }
733 let sources: Vec<_> = task
734 .runs
735 .iter()
736 .collect::<std::collections::BTreeSet<_>>()
737 .into_iter()
738 .cloned()
739 .collect();
740 let mut useful = Vec::new();
741 let mut selected = Vec::new();
742 let mut public = Vec::new();
743 let mut unreadable = false;
744 for id in &sources {
745 let Ok(state) = RunState::load_under(id, home) else {
746 unreadable = true;
747 continue;
748 };
749 if state.released() || state.retired_to.is_some() {
750 continue;
751 }
752 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
753 selected.push(state.clone());
754 }
755 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
756 public.push(state.clone());
757 }
758 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
759 useful.push(state);
760 }
761 }
762 let pool = if !selected.is_empty() {
766 selected
767 } else if !public.is_empty() {
768 public
769 } else {
770 useful
771 };
772 if unreadable || pool.len() != 1 {
773 report.retained.push(format!(
774 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
775 task.id,
776 sources.len()
777 ));
778 continue;
779 }
780 let authority = &pool[0];
781 if authority.candidates.is_empty() {
782 continue;
783 }
784 let latest = &authority.id;
785 for id in &sources {
786 if id == latest || report.retired >= 5 {
787 continue;
788 }
789 if let Ok(state) = RunState::load_under(id, home)
790 && state.retired_to.as_ref() == Some(latest)
791 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
792 {
793 continue;
794 }
795 let result = async {
796 ensure!(
797 !task
798 .resume_override
799 .as_ref()
800 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
801 "operator-pinned recovery run"
802 );
803 ensure!(
804 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
805 "selected replacement recovery source"
806 );
807 let _owner = crate::pr_head::owner_lock(home, id)?;
808 let mut state = RunState::load_under(id, home)?;
809 ensure!(
810 state.repo == authority.repo,
811 "repository differs from recovery authority"
812 );
813 ensure!(
814 !crate::pr_recovery::explicit_sources_retained(home, id),
815 "explicitly retained recovery source"
816 );
817 ensure!(
818 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
819 == Liveness::Dead,
820 "run live or ownership unknown"
821 );
822 let _driver = crate::driver::maintenance(home, id)?;
823 ensure!(
824 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
825 "within grace period"
826 );
827 ensure!(
828 !open.iter().any(|q| q.run == *id && q.status.open()),
829 "unanswered owner decision"
830 );
831 ensure!(
832 state.pr.is_none()
833 && state.pr_head.is_none()
834 && state.pr_recovery.is_none()
835 && state.land_approval.is_none()
836 && state.merge_decision.is_none()
837 && state.reviews.is_empty()
838 && state.recovery.is_none()
839 && state.candidate_preservation.is_empty()
840 && state.interrupted_implementations.is_empty()
841 && !state.released(),
842 "PR ownership or recovery checkpoint"
843 );
844 let proof = prove_empty(home, &state, latest).await?;
845 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
846 for path in &proof.paths {
849 ensure!(
850 git::worktree_remove_clean(&state.repo, path).await?,
851 "Git refused clean worktree removal"
852 );
853 }
854 Ok::<_, anyhow::Error>(())
855 }
856 .await;
857 match result {
858 Ok(()) => report.retired += 1,
859 Err(e) => report.retained.push(format!("{id}: {e:#}")),
860 }
861 }
862 }
863}
864
865struct EmptyProof {
866 paths: Vec<PathBuf>,
867 _outputs: Vec<OutputLease>,
868 _cargo: Vec<File>,
869}
870
871async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
874 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
875 let mut files = Vec::new();
876 let mut blobs = Vec::new();
877 for record in tree.split('\0').filter(|s| !s.is_empty()) {
878 let (header, name) = record.split_once('\t').context("unknown tree record")?;
879 let parts: Vec<_> = header.split_whitespace().collect();
880 ensure!(
881 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
882 "unsupported tracked source kind"
883 );
884 let path = worktree.join(name);
885 let meta = std::fs::symlink_metadata(&path)?;
886 ensure!(
887 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
888 "noncanonical tracked source"
889 );
890 #[cfg(unix)]
891 {
892 use std::os::unix::fs::PermissionsExt as _;
893 ensure!(
894 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
895 "tracked executable mode differs from original base"
896 );
897 }
898 files.push(path.to_str().context("non UTF-8 source")?.to_owned());
899 blobs.push(parts[2].to_owned());
900 }
901 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
902 let mut args = vec!["hash-object", "--no-filters", "--"];
903 args.extend(paths.iter().map(String::as_str));
904 let actual = git::git(worktree, &args).await?;
905 ensure!(
906 actual.lines().eq(expected.iter().map(String::as_str)),
907 "tracked bytes differ from original base"
908 );
909 }
910 Ok(())
911}
912
913async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
914 let id = &state.id;
915 let paths = worktrees(state).await?;
916 let mut guards = Vec::new();
917 let mut cargo_guards = Vec::new();
918 for path in &paths {
919 guards.push(OutputLease::take(path, true)?);
920 ensure!(
921 git::rev_parse(path, "HEAD").await? == state.base_commit,
922 "unique or unrecorded commits"
923 );
924 ensure!(
925 git::git(
926 path,
927 &[
928 "--no-optional-locks",
929 "status",
930 "--porcelain",
931 "--untracked-files=all"
932 ]
933 )
934 .await?
935 .is_empty(),
936 "tracked or untracked source changes"
937 );
938 ensure!(
939 git::git(
940 path,
941 &[
942 "--no-optional-locks",
943 "status",
944 "--porcelain",
945 "--untracked-files=all",
946 "--ignored",
947 "--",
948 ".",
949 ":(exclude)target"
950 ]
951 )
952 .await?
953 .is_empty(),
954 "source changes, ignored files or build outputs"
955 );
956 tracked_base(path, &state.base_commit).await?;
957 match std::fs::symlink_metadata(path.join("target")) {
958 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
959 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
960 Err(error) => return Err(error.into()),
961 }
962 for checkpoint in [
963 "rebase-merge",
964 "rebase-apply",
965 "sequencer",
966 "BISECT_LOG",
967 "REVERT_HEAD",
968 "AUTO_MERGE",
969 "MERGE_MSG",
970 "CHERRY_PICK_HEAD",
971 "MERGE_HEAD",
972 "REBASE_HEAD",
973 ] {
974 let location = git::git(
975 path,
976 &[
977 "rev-parse",
978 "--path-format=absolute",
979 "--git-path",
980 checkpoint,
981 ],
982 )
983 .await?;
984 ensure!(
985 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
986 "retained Git recovery metadata: {checkpoint}"
987 );
988 }
989 let origin_path = git::git(
992 path,
993 &[
994 "rev-parse",
995 "--path-format=absolute",
996 "--git-path",
997 "ORIG_HEAD",
998 ],
999 )
1000 .await?;
1001 match std::fs::symlink_metadata(origin_path.trim()) {
1002 Ok(meta) => {
1003 ensure!(
1004 meta.is_file() && !meta.file_type().is_symlink(),
1005 "unknown ORIG_HEAD artifact"
1006 );
1007 git::rev_parse(path, "ORIG_HEAD")
1008 .await
1009 .context("unreadable ORIG_HEAD")?;
1010 }
1011 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1012 Err(e) => return Err(e.into()),
1013 }
1014 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1015 ensure!(
1016 git::git_raw(
1017 path,
1018 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1019 )
1020 .await?
1021 .ok(),
1022 "ORIG_HEAD retains commits outside the original base"
1023 );
1024 }
1025 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1026 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1027 ensure!(
1028 git::git_raw(
1029 path,
1030 &["merge-base", "--is-ancestor", head, &state.base_commit]
1031 )
1032 .await?
1033 .ok(),
1034 "worktree reflog retains commits outside the original base"
1035 );
1036 }
1037 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1038 ensure!(
1039 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1040 .await?
1041 .ok(),
1042 "unfinished Git operation"
1043 );
1044 }
1045 }
1046 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1049 return Ok(EmptyProof {
1050 paths,
1051 _outputs: guards,
1052 _cargo: cargo_guards,
1053 });
1054 }
1055 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1056 for candidate in &state.candidates {
1057 ensure!(
1058 (paths.contains(&candidate.worktree)
1059 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1060 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1061 "candidate recovery source"
1062 );
1063 }
1064 let pattern = format!("refs/heads/magi/{}/", state.short());
1065 let branches = git::git(
1066 &state.repo,
1067 &["for-each-ref", "--format=%(objectname)", &pattern],
1068 )
1069 .await?;
1070 ensure!(
1071 branches.lines().all(|head| head == state.base_commit),
1072 "unrecorded candidate branch changes"
1073 );
1074 Ok(EmptyProof {
1075 paths,
1076 _outputs: guards,
1077 _cargo: cargo_guards,
1078 })
1079}
1080
1081fn commit_retirement(
1086 home: &Path,
1087 queue: &crate::queue::Queue,
1088 expected: &crate::queue::Task,
1089 state: &mut RunState,
1090 authority: &str,
1091 sources: &[String],
1092 now: Timestamp,
1093) -> Result<()> {
1094 let _write = queue.lock_task(&expected.id)?;
1095 let current = queue.get(&expected.id)?;
1096 ensure!(
1097 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1098 "task changed during retirement proof; retaining worktrees for the current owner decision"
1099 );
1100 ensure!(
1101 !current
1102 .resume_override
1103 .as_ref()
1104 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1105 && !current
1106 .replacement
1107 .as_ref()
1108 .is_some_and(|pin| pin.run == state.id),
1109 "current owner selected this recovery source"
1110 );
1111 ensure!(
1112 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1113 "explicitly retained recovery source"
1114 );
1115 ensure!(
1116 !crate::ask::Questions::at(home.join("questions"))
1117 .list()
1118 .iter()
1119 .any(|q| q.status.open()
1120 && (q.run == state.id
1121 || q.run == current.id
1122 || current.blocked_by.contains(&q.id))),
1123 "owner decision arrived during retirement proof"
1124 );
1125 let dir = home.join("recovery-lineages");
1126 std::fs::create_dir_all(&dir)?;
1127 let tmp = dir.join(format!("{}.tmp", current.id));
1128 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1129 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1130 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1131 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1132 state.retired_to = Some(authority.to_owned());
1133 state.event(
1134 "retention",
1135 format!(
1136 "empty attempt retired to {authority}; recovery sources: {}",
1137 sources.join(", ")
1138 ),
1139 );
1140 state.save_under(home)?;
1141 Ok(())
1143}
1144
1145pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1149 if min_free == 0 {
1150 return std::future::pending().await;
1151 }
1152 loop {
1153 for path in &paths {
1154 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1155 match crate::disk::free_bytes(existing) {
1156 Ok(free) if free >= min_free / 2 => {}
1157 Ok(free) => {
1158 return format!(
1159 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1160 );
1161 }
1162 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1163 }
1164 }
1165 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1166 }
1167}
1168
1169#[cfg(all(test, unix))]
1170mod tests {
1171 use super::*;
1172 use crate::{
1173 config::{Config, Disk},
1174 queue::{Source, Task},
1175 run::{Candidate, RunStatus},
1176 };
1177
1178 struct Fx {
1179 _temp: tempfile::TempDir,
1180 home: PathBuf,
1181 repo: PathBuf,
1182 bay: PathBuf,
1183 base: String,
1184 }
1185 impl Fx {
1186 async fn new() -> Self {
1187 let temp = tempfile::tempdir().unwrap();
1188 let root = temp.path().canonicalize().unwrap();
1189 let home = root.join("home");
1190 let repo = root.join("repo");
1191 let bay = root.join("worktrees");
1192 std::fs::create_dir_all(&repo).unwrap();
1193 std::fs::create_dir_all(home.join("runs")).unwrap();
1194 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1195 git::git(&repo, &["config", "user.name", "Fixture"])
1196 .await
1197 .unwrap();
1198 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1199 .await
1200 .unwrap();
1201 git::git(&repo, &["config", "commit.gpgsign", "false"])
1202 .await
1203 .unwrap();
1204 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1205 std::fs::write(repo.join("source"), "base\n").unwrap();
1206 git::git(&repo, &["add", "."]).await.unwrap();
1207 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1208 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1209 Self {
1210 _temp: temp,
1211 home,
1212 repo,
1213 bay,
1214 base,
1215 }
1216 }
1217 async fn run(&self, tag: &str, useful: bool) -> RunState {
1218 let mut cfg = Config::default();
1219 cfg.graph.worktree_root = Some(self.bay.clone());
1220 let mut state = RunState::new(
1221 self.repo.clone(),
1222 "main".into(),
1223 self.base.clone(),
1224 "task".into(),
1225 cfg,
1226 );
1227 state.id = format!("20260101-000000-{tag}");
1228 state.status = RunStatus::Blocked;
1229 state.driver_exited = true;
1230 let wt = state.worktree_root().join("cand-A");
1231 let branch = format!("magi/{tag}/A");
1232 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1233 .await
1234 .unwrap();
1235 if useful {
1236 std::fs::write(wt.join("source"), tag).unwrap();
1237 git::git(&wt, &["add", "."]).await.unwrap();
1238 git::git(&wt, &["commit", "-m", "candidate change"])
1239 .await
1240 .unwrap();
1241 }
1242 state.candidates.push(Candidate {
1243 index: 0,
1244 label: 'A',
1245 agent: "fixture".into(),
1246 branch,
1247 worktree: wt,
1248 summary: String::new(),
1249 stat: String::new(),
1250 files: usize::from(useful),
1251 commits: usize::from(useful),
1252 empty: !useful,
1253 failed: None,
1254 verified_noop: None,
1255 duration_ms: 0,
1256 folded: false,
1257 });
1258 state.save_under(&self.home).unwrap();
1259 state
1260 }
1261 fn disk() -> Disk {
1262 Disk {
1263 worktree_limit_bytes: 1,
1264 worktree_reclaim_bytes: 1024 * 1024,
1265 fold_grace_secs: 0,
1266 min_free_bytes: 0,
1267 ..Disk::default()
1268 }
1269 }
1270 fn target(wt: &Path) {
1271 let target = wt.join("target");
1272 std::fs::create_dir_all(target.join("debug")).unwrap();
1273 let mut artifact = [1; 200];
1274 artifact[..4].copy_from_slice(b"\x7fELF");
1275 std::fs::write(target.join("debug/output"), artifact).unwrap();
1276 let output = target
1277 .join("debug/output")
1278 .display()
1279 .to_string()
1280 .replace('\\', "\\\\")
1281 .replace(' ', "\\ ");
1282 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1283 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1284 std::fs::write(
1285 target.join("CACHEDIR.TAG"),
1286 "Signature: 8a477f597d28d172789f06886806bc55d\n",
1287 )
1288 .unwrap();
1289 std::fs::write(
1290 target.join(".rustc_info.json"),
1291 "{\"rustc_fingerprint\":123}",
1292 )
1293 .unwrap();
1294 }
1295 fn now() -> Timestamp {
1296 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1297 }
1298 fn task(&self, runs: &[RunState]) -> Task {
1299 let mut task = Task::new(
1300 "fixture".into(),
1301 "task".into(),
1302 self.repo.clone(),
1303 Source::Human,
1304 );
1305 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1306 crate::queue::Queue::at(self.home.join("queue"))
1307 .put(&mut task)
1308 .unwrap();
1309 task
1310 }
1311 }
1312
1313 #[tokio::test]
1314 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
1315 for remove_root in [true, false] {
1316 let f = Fx::new().await;
1317 let stale = Fx::new().await;
1318 let mut missing = stale.run("aa00", false).await;
1319 Fx::target(&missing.candidates[0].worktree);
1320 missing.save_under(&f.home).unwrap();
1321 if remove_root {
1322 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
1323 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
1326 } else {
1327 std::fs::remove_dir_all(&stale.repo).unwrap();
1328 }
1329 let healthy = f.run("aa01", false).await;
1330 Fx::target(&healthy.candidates[0].worktree);
1331
1332 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1333 assert!(report.failures.is_empty(), "{report:?}");
1334 assert_eq!(report.freed_bytes, 200, "{report:?}");
1335 assert!(
1336 !healthy.candidates[0]
1337 .worktree
1338 .join("target/debug/output")
1339 .exists()
1340 );
1341 if !remove_root {
1342 assert!(
1344 missing.candidates[0]
1345 .worktree
1346 .join("target/debug/output")
1347 .exists()
1348 );
1349 }
1350 }
1351 }
1352
1353 #[tokio::test]
1354 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
1355 let f = Fx::new().await;
1356 let mut missing = f.run("aa00", false).await;
1357 Fx::target(&missing.candidates[0].worktree);
1358 missing.repo = missing.candidates[0].worktree.clone();
1359 missing.save_under(&f.home).unwrap();
1360 std::fs::remove_dir_all(&missing.repo).unwrap();
1361 assert!(missing.worktree_root().is_dir());
1362 assert!(f.repo.is_dir());
1363
1364 let healthy = f.run("aa01", false).await;
1365 Fx::target(&healthy.candidates[0].worktree);
1366
1367 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1368 assert!(report.failures.is_empty(), "{report:?}");
1369 assert_eq!(report.freed_bytes, 200, "{report:?}");
1370 assert!(
1371 !healthy.candidates[0]
1372 .worktree
1373 .join("target/debug/output")
1374 .exists()
1375 );
1376 }
1377
1378 #[tokio::test]
1379 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
1380 let f = Fx::new().await;
1381 let broken = Fx::new().await;
1382 let mut invalid = broken.run("aa00", false).await;
1383 Fx::target(&invalid.candidates[0].worktree);
1384 invalid.save_under(&f.home).unwrap();
1385 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
1386 let healthy = f.run("aa01", false).await;
1387 Fx::target(&healthy.candidates[0].worktree);
1388
1389 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1390 assert_eq!(report.failures.len(), 1, "{report:?}");
1391 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
1392 assert_eq!(report.freed_bytes, 200, "{report:?}");
1393 assert!(
1394 !healthy.candidates[0]
1395 .worktree
1396 .join("target/debug/output")
1397 .exists()
1398 );
1399 assert!(
1400 invalid.candidates[0]
1401 .worktree
1402 .join("target/debug/output")
1403 .exists()
1404 );
1405 }
1406
1407 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
1408 let path = reconcile::run(&f.home, &task.id, &source.id, None)
1409 .await
1410 .unwrap();
1411 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
1412 }
1413
1414 #[tokio::test]
1415 async fn reconciliation_standing_chat_requires_exact_current_owner_authority() {
1416 use crate::{
1417 chat_control::{Operation, dispatch},
1418 talk::{self, Talks},
1419 };
1420 let f = Fx::new().await;
1421 let source = f.run("ae00", true).await;
1422 let clean = f.run("ae01", false).await;
1423 let queue = crate::queue::Queue::at(f.home.join("queue"));
1424 let talks = Talks::at(f.home.join("talks"));
1425 let mut conversation: talk::Talk = serde_json::from_value(serde_json::json!({
1426 "schema":1,"id":"talk-owner","repo":f.repo,"agent":"worker","status":"open","turns":[],
1427 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
1428 "seat":crate::agent::SeatState::new("talk","worker",1)
1429 }))
1430 .unwrap();
1431 talks.put(&mut conversation).unwrap();
1432 let mut task = f.task(&[source.clone(), clean.clone()]);
1433 let operation = |apply: Option<&str>, quote: Option<&str>| Operation::ReconcileRetention {
1434 task: task.id.clone(),
1435 authority: source.id.clone(),
1436 apply: apply.map(str::to_owned),
1437 quote: quote.map(str::to_owned),
1438 };
1439 assert!(
1440 dispatch(
1441 &queue,
1442 &f.home,
1443 &talks,
1444 &conversation.id,
1445 operation(None, None)
1446 )
1447 .await
1448 .is_err()
1449 );
1450 task.chat = Some(conversation.id.clone());
1451 queue.put(&mut task).unwrap();
1452 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1453 let operation = |apply: Option<&str>, quote: Option<&str>| Operation::ReconcileRetention {
1454 task: task.id.clone(),
1455 authority: source.id.clone(),
1456 apply: apply.map(str::to_owned),
1457 quote: quote.map(str::to_owned),
1458 };
1459 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
1460 .await
1461 .unwrap();
1462 let legacy: serde_json::Value =
1463 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
1464 let legacy_id = legacy["id"].as_str().unwrap();
1465 let legacy_words = reconcile::apply_instruction(legacy_id, &task.id, &source.id);
1466 talk::record(&mut conversation, &talks, &legacy_words, vec![]).unwrap();
1467 assert!(
1468 dispatch(
1469 &queue,
1470 &f.home,
1471 &talks,
1472 &conversation.id,
1473 operation(Some(legacy_id), Some(&legacy_words))
1474 )
1475 .await
1476 .is_err()
1477 );
1478 let summary = dispatch(
1479 &queue,
1480 &f.home,
1481 &talks,
1482 &conversation.id,
1483 operation(None, None),
1484 )
1485 .await
1486 .unwrap();
1487 assert!(summary.contains("Complete review summary"));
1488 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
1489 let root = f.home.join("retention-reconciliations");
1490 let displayed: serde_json::Value =
1491 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
1492 let id = displayed["plan"].as_str().unwrap();
1493 let dir = root.join(id);
1494 let plan: serde_json::Value =
1495 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
1496 assert_eq!(plan["version"], 2);
1497 assert!(clean.candidates[0].worktree.exists());
1498 assert!(!dir.join("approved.json").exists());
1499 let expected = reconcile::apply_instruction(id, &task.id, &source.id);
1500 for words in [
1501 "continue".to_owned(),
1502 format!("{expected}\nand another plan"),
1503 reconcile::apply_instruction("wrong-plan", &task.id, &source.id),
1504 reconcile::apply_instruction(id, "wrong-task", &source.id),
1505 reconcile::apply_instruction(id, &task.id, "wrong-authority"),
1506 ] {
1507 talk::record(&mut conversation, &talks, &words, vec![]).unwrap();
1508 assert!(
1509 dispatch(
1510 &queue,
1511 &f.home,
1512 &talks,
1513 &conversation.id,
1514 operation(Some(id), Some(&words))
1515 )
1516 .await
1517 .is_err()
1518 );
1519 assert!(!dir.join("approved.json").exists());
1520 }
1521 talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1522 assert!(
1523 dispatch(
1524 &queue,
1525 &f.home,
1526 &talks,
1527 &conversation.id,
1528 operation(Some(id), None)
1529 )
1530 .await
1531 .is_err()
1532 );
1533 conversation.pending = "wait".into();
1534 talks.put(&mut conversation).unwrap();
1535 assert!(
1536 dispatch(
1537 &queue,
1538 &f.home,
1539 &talks,
1540 &conversation.id,
1541 operation(Some(id), Some(&expected))
1542 )
1543 .await
1544 .is_err()
1545 );
1546 conversation.pending.clear();
1547 talks.put(&mut conversation).unwrap();
1548 talk::record(&mut conversation, &talks, "wait", vec![]).unwrap();
1549 let error = reconcile::interrupt_chat_after_fence(
1550 &f.home,
1551 &task,
1552 id,
1553 reconcile::ChatRequest {
1554 talks: &talks,
1555 conversation: &conversation.id,
1556 quote: Some(&expected),
1557 },
1558 )
1559 .await
1560 .unwrap_err();
1561 assert!(
1562 !error.to_string().contains("simulated interruption"),
1563 "{error:#}"
1564 );
1565 assert!(!dir.join("approved.json").exists());
1566 assert!(
1567 RunState::load_under(&clean.id, &f.home)
1568 .unwrap()
1569 .retired_to
1570 .is_none()
1571 );
1572 talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1573 let error = reconcile::interrupt_chat_after_fence(
1574 &f.home,
1575 &task,
1576 id,
1577 reconcile::ChatRequest {
1578 talks: &talks,
1579 conversation: &conversation.id,
1580 quote: Some(&expected),
1581 },
1582 )
1583 .await
1584 .unwrap_err();
1585 assert!(
1586 error.to_string().contains("simulated interruption"),
1587 "{error:#}"
1588 );
1589 assert!(clean.candidates[0].worktree.exists());
1590 assert!(dir.join("approved.json").exists());
1591 talk::record(&mut conversation, &talks, "wait", vec![]).unwrap();
1592 assert!(
1593 dispatch(
1594 &queue,
1595 &f.home,
1596 &talks,
1597 &conversation.id,
1598 operation(Some(id), Some(&expected))
1599 )
1600 .await
1601 .is_err()
1602 );
1603 talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1604 dispatch(
1605 &queue,
1606 &f.home,
1607 &talks,
1608 &conversation.id,
1609 operation(Some(id), Some(&expected)),
1610 )
1611 .await
1612 .unwrap();
1613 assert!(!clean.candidates[0].worktree.exists());
1614 assert!(source.candidates[0].worktree.exists());
1615 let audit: serde_json::Value =
1616 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
1617 assert_eq!(audit["version"], 2);
1618 assert_eq!(audit["owner"]["conversation"], conversation.id);
1619 assert_eq!(audit["owner"]["evidence"]["instruction"], expected);
1620 talk::record(&mut conversation, &talks, "continue", vec![]).unwrap();
1621 assert!(
1622 dispatch(
1623 &queue,
1624 &f.home,
1625 &talks,
1626 &conversation.id,
1627 operation(Some(id), Some(&expected))
1628 )
1629 .await
1630 .is_err()
1631 );
1632 talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1633 dispatch(
1634 &queue,
1635 &f.home,
1636 &talks,
1637 &conversation.id,
1638 operation(Some(id), Some(&expected)),
1639 )
1640 .await
1641 .unwrap();
1642 }
1643
1644 #[tokio::test]
1645 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
1646 let f = Fx::new().await;
1647 let source = f.run("af00", true).await;
1648 let mut clean = f.run("af01", false).await;
1649 let mut dirty = f.run("af02", false).await;
1650 for state in [&mut clean, &mut dirty] {
1651 state.status = RunStatus::Prep;
1652 state.prep_complete = Some(false);
1653 state.candidates[0].empty = false;
1654 state.save_under(&f.home).unwrap();
1655 }
1656 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
1657 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
1658 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1659 let plan = reconciliation_plan(&f, &task, &source).await;
1660 let selected: Vec<_> = plan["candidates"]
1661 .as_array()
1662 .unwrap()
1663 .iter()
1664 .filter(|e| e["selected"] == true)
1665 .collect();
1666 assert_eq!(selected.len(), 1, "{plan:#}");
1667 assert_eq!(selected[0]["run"], clean.id);
1668 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1669 .await
1670 .unwrap();
1671 assert!(!clean.candidates[0].worktree.exists());
1672 assert!(dirty.candidates[0].worktree.exists());
1673 assert!(
1674 RunState::load_under(&dirty.id, &f.home)
1675 .unwrap()
1676 .retired_to
1677 .is_none()
1678 );
1679 }
1680
1681 #[tokio::test]
1682 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
1683 let f = Fx::new().await;
1684 let mut source = f.run("aa00", true).await;
1685 let alternative = f.run("aa01", true).await;
1686 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
1687 .await
1688 .unwrap();
1689 source.reviewed_commits = Some(vec![head.clone()]);
1690 source.save_under(&f.home).unwrap();
1691 let mut runs = vec![source.clone(), alternative.clone()];
1692 for i in 2..14 {
1693 runs.push(f.run(&format!("aa{i:02}"), false).await);
1694 }
1695 let mut task = f.task(&runs);
1696 task.pr_recovery = Some(crate::pr_recovery::Selection {
1697 url: "https://example.invalid/pull/1".into(),
1698 branch: "selected".into(),
1699 head: head.clone(),
1700 source_head: Some(head.clone()),
1701 source_run: Some(source.id.clone()),
1702 source_proof: None,
1703 disposition: Some(crate::pr_recovery::Disposition::Land),
1704 instruction: task.instruction.clone(),
1705 approved_at: Timestamp::now(),
1706 });
1707 source.pr_recovery = task.pr_recovery.clone();
1708 source.save_under(&f.home).unwrap();
1709 runs[0] = source.clone();
1710 crate::queue::Queue::at(f.home.join("queue"))
1711 .put(&mut task)
1712 .unwrap();
1713 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1714 let task_before = serde_json::to_value(&task).unwrap();
1715 let mut report = Report::default();
1716 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
1717 assert_eq!(
1718 report.retired, 0,
1719 "automatic cleanup must still refuse blanket markers"
1720 );
1721 assert!(
1722 report
1723 .retained
1724 .iter()
1725 .any(|reason| reason.contains("explicitly retained"))
1726 );
1727 let mut total = 0;
1728 for expected in [5, 5, 2, 0] {
1729 let plan = reconciliation_plan(&f, &task, &source).await;
1730 let selected: Vec<_> = plan["candidates"]
1731 .as_array()
1732 .unwrap()
1733 .iter()
1734 .filter(|e| e["selected"] == true)
1735 .collect();
1736 assert_eq!(selected.len(), expected, "{plan:#}");
1737 for e in &selected {
1738 assert!(
1739 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
1740 .unwrap()
1741 .retired_to
1742 .is_none()
1743 );
1744 }
1745 let operation = plan["id"].as_str().unwrap();
1746 for _ in 0..2 {
1747 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
1748 .await
1749 .unwrap();
1750 }
1751 total += expected;
1752 }
1753 assert_eq!(total, 12);
1754 for run in &runs {
1755 let dir = f.home.join("runs").join(&run.id);
1756 assert!(dir.join("run.json").exists());
1757 assert!(dir.join("retain-recovery-sources").exists());
1758 assert!(
1759 git::branch_exists(&f.repo, &run.candidates[0].branch)
1760 .await
1761 .unwrap()
1762 );
1763 }
1764 assert!(source.candidates[0].worktree.exists());
1765 assert!(alternative.candidates[0].worktree.exists());
1766 assert_eq!(
1767 git::rev_parse(&f.repo, &source.candidates[0].branch)
1768 .await
1769 .unwrap(),
1770 head
1771 );
1772 assert_eq!(
1773 serde_json::to_value(
1774 crate::queue::Queue::at(f.home.join("queue"))
1775 .get(&task.id)
1776 .unwrap()
1777 )
1778 .unwrap(),
1779 task_before
1780 );
1781 assert_eq!(
1782 runs.iter()
1783 .filter(|r| RunState::load_under(&r.id, &f.home)
1784 .unwrap()
1785 .retired_to
1786 .is_none())
1787 .count(),
1788 2
1789 );
1790 }
1791
1792 #[tokio::test]
1793 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
1794 let f = Fx::new().await;
1795 let source = f.run("ab00", true).await;
1796 let mut runs = vec![source.clone()];
1797 for i in 1..9 {
1798 runs.push(f.run(&format!("ab{i:02}"), false).await);
1799 }
1800 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
1801 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
1802 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
1803 std::fs::write(
1804 runs[3].candidates[0].worktree.join("scratch/source"),
1805 "ignored source",
1806 )
1807 .unwrap();
1808 std::fs::write(
1809 runs[4].worktree_root().join("unknown"),
1810 "unregistered source",
1811 )
1812 .unwrap();
1813 let checkpoint = git::git(
1814 &runs[5].candidates[0].worktree,
1815 &[
1816 "rev-parse",
1817 "--path-format=absolute",
1818 "--git-path",
1819 "sequencer",
1820 ],
1821 )
1822 .await
1823 .unwrap();
1824 std::fs::create_dir(checkpoint.trim()).unwrap();
1825 runs[6].driver_exited = false;
1826 runs[6].driver_pid = Some(std::process::id());
1827 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
1828 runs[6].save_under(&f.home).unwrap();
1829 runs[7].driver_exited = false;
1830 runs[7].driver_pid = None;
1831 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
1833 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
1834
1835 let hidden = &runs[8].candidates[0].worktree;
1836 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
1837 .await
1838 .unwrap();
1839 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
1840 assert!(
1841 git::git(hidden, &["status", "--porcelain"])
1842 .await
1843 .unwrap()
1844 .is_empty()
1845 );
1846 let task = f.task(&runs);
1847 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1848 let plan = reconciliation_plan(&f, &task, &source).await;
1849 assert!(
1850 plan["candidates"]
1851 .as_array()
1852 .unwrap()
1853 .iter()
1854 .all(|e| e["selected"] == false),
1855 "{plan:#}"
1856 );
1857 for run in &runs {
1858 assert!(run.candidates[0].worktree.exists());
1859 }
1860 }
1861
1862 #[tokio::test]
1863 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
1864 let f = Fx::new().await;
1865 let source = f.run("ac00", true).await;
1866 let empty = f.run("ac01", false).await;
1867 let task = f.task(&[source.clone(), empty.clone()]);
1868 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1869 let plan = reconciliation_plan(&f, &task, &source).await;
1870 std::fs::write(
1871 f.home.join("runs").join(&source.id).join("new-report"),
1872 "new review checkpoint",
1873 )
1874 .unwrap();
1875 assert!(
1876 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1877 .await
1878 .is_err()
1879 );
1880 assert!(empty.candidates[0].worktree.exists());
1881 assert!(
1882 RunState::load_under(&empty.id, &f.home)
1883 .unwrap()
1884 .retired_to
1885 .is_none()
1886 );
1887 let marker = f
1888 .home
1889 .join("runs")
1890 .join(&empty.id)
1891 .join("retain-recovery-sources");
1892 std::fs::write(&marker, "different owner").unwrap();
1893 let plan = reconciliation_plan(&f, &task, &source).await;
1894 assert!(
1895 plan["candidates"]
1896 .as_array()
1897 .unwrap()
1898 .iter()
1899 .all(|e| e["selected"] == false)
1900 );
1901 }
1902
1903 #[tokio::test]
1904 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
1905 for partial in [false, true] {
1906 let f = Fx::new().await;
1907 let source = f.run("ab10", true).await;
1908 let first = f.run("ab11", false).await;
1909 let second = f.run("ab12", false).await;
1910 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
1911 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1912 let plan = reconciliation_plan(&f, &task, &source).await;
1913 let id = plan["id"].as_str().unwrap();
1914 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
1915 std::fs::write(&staging, b"").unwrap();
1917 assert!(
1918 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1919 .await
1920 .is_err()
1921 );
1922 assert!(staging.exists());
1923 std::fs::remove_file(&staging).unwrap();
1924 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
1925 .await
1926 .unwrap_err();
1927 assert!(
1928 error.to_string().contains("before run state rename"),
1929 "{error:#}"
1930 );
1931 assert!(
1932 RunState::load_under(&first.id, &f.home)
1933 .unwrap()
1934 .retired_to
1935 .is_some()
1936 );
1937 assert!(
1938 RunState::load_under(&second.id, &f.home)
1939 .unwrap()
1940 .retired_to
1941 .is_none()
1942 );
1943 assert!(first.candidates[0].worktree.exists());
1944 assert!(second.candidates[0].worktree.exists());
1945 let interrupted = std::fs::read(&staging).unwrap();
1946 std::fs::write(&staging, b"unrelated recovery source").unwrap();
1948 assert!(
1949 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1950 .await
1951 .is_err()
1952 );
1953 assert_eq!(
1954 std::fs::read(&staging).unwrap(),
1955 b"unrelated recovery source"
1956 );
1957 std::fs::write(&staging, interrupted).unwrap();
1958 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1959 .await
1960 .unwrap();
1961 assert!(!staging.exists());
1962 assert!(!first.candidates[0].worktree.exists());
1963 assert!(!second.candidates[0].worktree.exists());
1964 assert_eq!(
1965 RunState::load_under(&second.id, &f.home)
1966 .unwrap()
1967 .retired_to,
1968 Some(source.id)
1969 );
1970 }
1971 }
1972
1973 #[tokio::test]
1974 async fn reconciliation_resumes_only_its_exact_durable_fence() {
1975 let f = Fx::new().await;
1976 let source = f.run("ad00", true).await;
1977 let empty = f.run("ad01", false).await;
1978 let task = f.task(&[source.clone(), empty.clone()]);
1979 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1980 let plan = reconciliation_plan(&f, &task, &source).await;
1981 let id = plan["id"].as_str().unwrap();
1982 assert!(
1983 reconcile::interrupt_after_fence(&f.home, &task, id)
1984 .await
1985 .is_err()
1986 );
1987 assert!(empty.candidates[0].worktree.exists());
1988 assert_eq!(
1989 RunState::load_under(&empty.id, &f.home)
1990 .unwrap()
1991 .retired_to
1992 .as_ref(),
1993 Some(&source.id)
1994 );
1995 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1996 .await
1997 .unwrap();
1998 assert!(!empty.candidates[0].worktree.exists());
1999 git::worktree_add_branch(
2001 &f.repo,
2002 &empty.candidates[0].worktree,
2003 "restore-empty",
2004 &f.base,
2005 )
2006 .await
2007 .unwrap();
2008 assert!(
2010 reconcile::run(&f.home, &task.id, &source.id, Some(id))
2011 .await
2012 .is_err()
2013 );
2014 assert!(empty.candidates[0].worktree.exists());
2015 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
2016 changed.prep_failure = Some("new owner edit".into());
2017 changed.save_under(&f.home).unwrap();
2018 assert!(
2019 reconcile::run(&f.home, &task.id, &source.id, Some(id))
2020 .await
2021 .is_err()
2022 );
2023 }
2024
2025 #[tokio::test]
2026 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
2027 let f = Fx::new().await;
2028 let source = f.run("ae00", true).await;
2029 let mut prep = f.run("ae01", false).await;
2030 let candidate = prep.candidates.pop().unwrap();
2031 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
2032 prep.save_under(&f.home).unwrap();
2033 let reset = f.run("ae02", false).await;
2034 let wt = &reset.candidates[0].worktree;
2035 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
2036 git::git(wt, &["add", "."]).await.unwrap();
2037 git::git(wt, &["commit", "-m", "unique implementation"])
2038 .await
2039 .unwrap();
2040 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
2041 let unknown = f.run("ae03", false).await;
2042 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
2043 let mut raw: serde_json::Value =
2044 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
2045 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
2046 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
2047 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
2048 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2049 let plan = reconciliation_plan(&f, &task, &source).await;
2050 let selected: Vec<_> = plan["candidates"]
2051 .as_array()
2052 .unwrap()
2053 .iter()
2054 .filter(|e| e["selected"] == true)
2055 .collect();
2056 assert_eq!(selected.len(), 1, "{plan:#}");
2057 assert_eq!(selected[0]["run"], prep.id);
2058 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
2059 .await
2060 .unwrap();
2061 assert!(!candidate.worktree.exists());
2062 assert!(reset.candidates[0].worktree.exists());
2063 assert!(unknown.candidates[0].worktree.exists());
2064 }
2065
2066 #[tokio::test]
2067 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
2068 let f = Fx::new().await;
2069 let mut state = f.run("aaaa", true).await;
2070 let candidate = state.candidates[0].worktree.clone();
2071 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
2072 Fx::target(&candidate);
2073 for seat in ["review-1", "review-2"] {
2074 let wt = state.worktree_root().join(seat);
2075 git::worktree_add_detached(&f.repo, &wt, &f.base)
2076 .await
2077 .unwrap();
2078 Fx::target(&wt);
2079 }
2080 state.status = RunStatus::Landing;
2081 state.save_under(&f.home).unwrap();
2082 let second = Fx::new().await;
2083 let other = second.run("bbbb", true).await;
2084 let mut imported = other.clone();
2086 imported.save_under(&f.home).unwrap();
2087 Fx::target(&other.candidates[0].worktree);
2088 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
2089 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2090 assert!(
2091 candidate.join("target/debug/output").exists(),
2092 "active writer protected"
2093 );
2094 assert!(
2095 !state
2096 .worktree_root()
2097 .join("review-1/target/debug/output")
2098 .exists()
2099 );
2100 assert!(
2101 !state
2102 .worktree_root()
2103 .join("review-2/target/debug/output")
2104 .exists()
2105 );
2106 assert!(
2107 !other.candidates[0]
2108 .worktree
2109 .join("target/debug/output")
2110 .exists()
2111 );
2112 assert!(out.freed_bytes >= 600);
2113 drop(guard);
2114 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2115 assert!(out.freed_bytes >= 200);
2116 assert_eq!(
2117 std::fs::read_to_string(candidate.join("source")).unwrap(),
2118 "uncommitted work"
2119 );
2120 assert!(
2121 git::branch_exists(&f.repo, &state.candidates[0].branch)
2122 .await
2123 .unwrap()
2124 );
2125 }
2126
2127 #[tokio::test]
2128 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
2129 let f = Fx::new().await;
2130 let state = f.run("cc01", false).await;
2131 let wt = &state.candidates[0].worktree;
2132 Fx::target(wt);
2133 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2134 let dir = wt.join("target").join(dir);
2135 std::fs::create_dir_all(&dir).unwrap();
2136 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
2137 }
2138 let manifest = wt.join("target/debug/output.d");
2140 let output = wt.join("target/debug/output");
2141 std::fs::write(
2142 &manifest,
2143 format!(
2144 "{}: {}\n",
2145 output.display(),
2146 wt.join("target/debug/notes.txt").display()
2147 ),
2148 )
2149 .unwrap();
2150 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2151 assert_eq!(report.freed_bytes, 200);
2152 assert!(!output.exists());
2153 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2154 assert_eq!(
2155 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
2156 "user evidence"
2157 );
2158 }
2159 assert!(manifest.exists());
2161 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
2162 std::fs::write(&output, "user note replacing an old binary").unwrap();
2163 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2164 assert_eq!(report.freed_bytes, 0);
2165 assert_eq!(
2166 std::fs::read_to_string(&output).unwrap(),
2167 "user note replacing an old binary"
2168 );
2169 }
2170
2171 #[tokio::test]
2172 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
2173 let f = Fx::new().await;
2174 let state = f.run("cc02", false).await;
2175 let wt = &state.candidates[0].worktree;
2176 Fx::target(wt);
2177 let mut disk = Fx::disk();
2178 disk.worktree_limit_bytes = u64::MAX;
2179 let now = Fx::now();
2180 let first = sweep(&f.home, &disk, now).await;
2181 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
2182 assert_eq!(
2183 sweep(&f.home, &disk, now).await.measured_bytes,
2184 first.measured_bytes
2185 );
2186 assert!(
2187 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
2188 .await
2189 .measured_bytes
2190 > first.measured_bytes
2191 );
2192 disk.worktree_limit_bytes = 0;
2193 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
2194 }
2195
2196 #[tokio::test]
2197 async fn unfinished_source_fold_reports_its_retention_reason() {
2198 let f = Fx::new().await;
2199 let mut source = f.run("cc03", true).await;
2200 let task = f.task(&[source.clone()]);
2201 let error = crate::graph::fold_run(&mut source, true, &f.home)
2202 .await
2203 .unwrap_err();
2204 assert!(error.to_string().contains(&task.id));
2205 assert!(source.candidates[0].worktree.exists());
2206 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2207 let error = crate::graph::fold_run(&mut source, true, &f.home)
2208 .await
2209 .unwrap_err();
2210 assert!(error.to_string().contains("explicitly retained"));
2211 }
2212
2213 #[tokio::test]
2214 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
2215 let f = Fx::new().await;
2216 let mut source = f.run("cc04", true).await;
2217 let mut task = f.task(&[source.clone()]);
2218 task.hold_manual(Some("queue settlement pending".into()));
2219 let queue = crate::queue::Queue::at(f.home.join("queue"));
2220 queue.put(&mut task).unwrap();
2221 source.merge = Some(crate::run::MergeOutcome {
2222 mode: crate::config::MergeMode::Pr,
2223 ok: true,
2224 detail: "confirmed merged".into(),
2225 empty: false,
2226 });
2227 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
2229 source.status = status;
2230 assert!(
2231 crate::graph::fold_run(&mut source, true, &f.home)
2232 .await
2233 .is_err()
2234 );
2235 assert!(source.candidates[0].worktree.exists());
2236 }
2237 git::git(
2238 &f.repo,
2239 &["merge", "--ff-only", &source.candidates[0].branch],
2240 )
2241 .await
2242 .unwrap();
2243 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2244 .await
2245 .unwrap();
2246 source.status = RunStatus::Merged;
2247 source.save_under(&f.home).unwrap();
2248 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
2249 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2250 assert!(
2251 crate::graph::fold_run(&mut source, true, &f.home)
2252 .await
2253 .unwrap_err()
2254 .to_string()
2255 .contains("explicitly retained")
2256 );
2257 std::fs::remove_file(
2260 f.home
2261 .join("runs")
2262 .join(&source.id)
2263 .join("retain-recovery-sources"),
2264 )
2265 .unwrap();
2266 let (folded, unreadable) = crate::clean::fold_due(
2267 &f.home.join("runs"),
2268 &f.home,
2269 &f.bay,
2270 &Fx::disk(),
2271 Fx::now(),
2272 )
2273 .await
2274 .unwrap();
2275 assert_eq!((folded, unreadable), (1, 0));
2276 assert!(!source.candidates[0].worktree.exists());
2277 let evidence = f
2278 .home
2279 .join("runs")
2280 .join(&source.id)
2281 .join("artifacts/fold-evidence");
2282 let names: Vec<_> = std::fs::read_dir(evidence)
2283 .unwrap()
2284 .map(|entry| entry.unwrap().file_name())
2285 .collect();
2286 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
2287 assert_eq!(
2288 queue.get(&task.id).unwrap().status,
2289 crate::queue::TaskStatus::Held
2290 );
2291 }
2292
2293 #[tokio::test]
2294 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
2295 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
2296 let f = Fx::new().await;
2297 let mut state = f.run("cc06", true).await;
2298 let candidate = state.candidates[0].clone();
2299 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2300 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
2301 git::rescue_commit(&candidate.worktree, "second candidate change")
2302 .await
2303 .unwrap();
2304 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2305 state.tally = Some(crate::run::Tally {
2306 first_choice: Default::default(),
2307 borda: Default::default(),
2308 winner: 'A',
2309 rankings: 0,
2310 unanimous_initial: true,
2311 deliberated: false,
2312 changed_votes: 0,
2313 unanimous_final: true,
2314 tie_break: None,
2315 judges: 0,
2316 present: 0,
2317 quorum: 0,
2318 met_quorum: true,
2319 uncontested: Some("fixture".into()),
2320 });
2321 let reviewer = state.worktree_root().join("review-1");
2324 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
2325 .await
2326 .unwrap();
2327 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
2330 git::rescue_commit(&f.repo, "advance before squash")
2331 .await
2332 .unwrap();
2333 let published = if variant == "partial" {
2334 &intermediate
2335 } else {
2336 &head
2337 };
2338 git::git(&f.repo, &["merge", "--squash", published])
2339 .await
2340 .unwrap();
2341 git::git(&f.repo, &["commit", "-m", "squash candidate"])
2342 .await
2343 .unwrap();
2344 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
2345 assert!(
2346 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
2347 .await
2348 .unwrap()
2349 .ok()
2350 );
2351 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
2352 git::rescue_commit(&f.repo, "advance after squash")
2353 .await
2354 .unwrap();
2355 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2356 .await
2357 .unwrap();
2358 if matches!(variant, "checkpoint" | "reviewer") {
2359 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
2361 git::rescue_commit(&reviewer, "unpublished checkpoint")
2362 .await
2363 .unwrap();
2364 if variant == "checkpoint" {
2365 git::git(&reviewer, &["reset", "--hard", &intermediate])
2366 .await
2367 .unwrap();
2368 }
2369 }
2370 if variant == "loser" {
2371 let mut loser = candidate.clone();
2372 loser.label = 'B';
2373 loser.branch = "magi/cc06/B".into();
2374 loser.worktree = state.worktree_root().join("cand-B");
2375 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
2376 .await
2377 .unwrap();
2378 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
2379 git::rescue_commit(&loser.worktree, "independent loser")
2380 .await
2381 .unwrap();
2382 state.candidates.push(loser);
2383 }
2384 state.status = RunStatus::Merged;
2385 state.merge = Some(crate::run::MergeOutcome {
2386 mode: crate::config::MergeMode::Pr,
2387 ok: true,
2388 detail: "candidate A squash merged".into(),
2389 empty: false,
2390 });
2391 state.save_under(&f.home).unwrap();
2392 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
2393 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
2394 result.unwrap();
2395 assert!(
2396 !candidate.worktree.exists() && !reviewer.exists(),
2397 "{variant}"
2398 );
2399 assert!(
2400 !git::branch_exists(&f.repo, &candidate.branch)
2401 .await
2402 .unwrap()
2403 );
2404 let artifacts = f
2405 .home
2406 .join("runs")
2407 .join(&state.id)
2408 .join("artifacts/fold-evidence");
2409 assert_eq!(
2410 artifacts.join(format!("{head}.patch")).exists(),
2411 variant == "partial",
2412 "only a partially published winner needs evidence: {variant}"
2413 );
2414 assert_eq!(
2415 artifacts.join(format!("{intermediate}.patch")).exists(),
2416 variant == "partial",
2417 "squash proof also covers intermediate reviewer tips: {variant}"
2418 );
2419 let patches: Vec<_> = std::fs::read_dir(&artifacts)
2420 .unwrap()
2421 .map(|entry| entry.unwrap().path())
2422 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
2423 .collect();
2424 let expected = match variant {
2425 "partial" => 2,
2426 "loser" | "reviewer" => 1,
2427 _ => 0,
2428 };
2429 assert_eq!(patches.len(), expected, "{variant}");
2430 if variant == "loser" {
2431 assert!(
2432 std::fs::read_to_string(&patches[0])
2433 .unwrap()
2434 .contains("losing candidate")
2435 );
2436 }
2437 assert!(artifacts.join("manifest.json").exists());
2438 if variant == "reviewer" {
2439 assert!(
2440 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
2441 .unwrap()
2442 .contains("reviewer checkpoint")
2443 );
2444 }
2445 crate::graph::fold_run(&mut state, true, &f.home)
2446 .await
2447 .unwrap();
2448 }
2449 }
2450
2451 #[tokio::test]
2452 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
2453 for missing in [true, false] {
2454 let f = Fx::new().await;
2455 let mut state = f.run("cc05", true).await;
2456 git::git(
2459 &f.repo,
2460 &["update-ref", "refs/remotes/origin/main", &f.base],
2461 )
2462 .await
2463 .unwrap();
2464 let candidate = state.candidates[0].clone();
2465 if !missing {
2466 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
2467 git::rescue_commit(&f.repo, "advance base before rebase")
2468 .await
2469 .unwrap();
2470 git::git(&candidate.worktree, &["rebase", "main"])
2471 .await
2472 .unwrap();
2473 git::sync_to_head(&candidate.worktree).await.unwrap();
2476 }
2477 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2478 state.reviewed_commits = Some(Vec::new());
2480 state.candidates[0].worktree = state.worktree_root().join("under-review");
2481 git::git(
2482 &f.repo,
2483 &[
2484 "worktree",
2485 "move",
2486 candidate.worktree.to_str().unwrap(),
2487 state.candidates[0].worktree.to_str().unwrap(),
2488 ],
2489 )
2490 .await
2491 .unwrap();
2492 let reviewer = state.worktree_root().join("review-1");
2493 git::worktree_add_detached(&f.repo, &reviewer, &head)
2494 .await
2495 .unwrap();
2496 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
2499 git::rescue_commit(&reviewer, "old review history")
2500 .await
2501 .unwrap();
2502 git::git(&reviewer, &["reset", "--hard", &head])
2503 .await
2504 .unwrap();
2505 if missing {
2506 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
2507 }
2508 let run_dir = f.home.join("runs").join(&state.id);
2509 let artifacts = run_dir.join("artifacts");
2510 std::fs::create_dir_all(&artifacts).unwrap();
2511 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
2512 if missing {
2513 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
2514 }
2515 state.status = RunStatus::Merged;
2516 state.merge = Some(crate::run::MergeOutcome {
2517 mode: crate::config::MergeMode::Pr,
2518 ok: true,
2519 detail: "confirmed squash merge".into(),
2520 empty: false,
2521 });
2522 state.save_under(&f.home).unwrap();
2523 let owned = state.candidates[0].worktree.clone();
2525 state.candidates[0].worktree = f.home.join("missing-external");
2526 assert!(
2527 crate::graph::fold_run(&mut state, true, &f.home)
2528 .await
2529 .unwrap_err()
2530 .to_string()
2531 .contains("outside the inventoried")
2532 );
2533 state.candidates[0].worktree = owned;
2534 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
2536 assert!(
2537 crate::graph::fold_run(&mut state, true, &f.home)
2538 .await
2539 .is_err()
2540 );
2541 assert!(reviewer.exists());
2542 drop(guard);
2543 let evidence = artifacts.join("fold-evidence");
2545 std::fs::write(&evidence, "blocked").unwrap();
2546 assert!(
2547 crate::graph::fold_run(&mut state, true, &f.home)
2548 .await
2549 .is_err()
2550 );
2551 assert!(reviewer.exists());
2552 std::fs::remove_file(&evidence).unwrap();
2553 std::fs::create_dir_all(&evidence).unwrap();
2554 let oversized = evidence.join("capacity-fixture");
2555 std::fs::File::create(&oversized)
2556 .unwrap()
2557 .set_len(64 * 1024 * 1024)
2558 .unwrap();
2559 assert!(
2560 crate::graph::fold_run(&mut state, true, &f.home)
2561 .await
2562 .is_err()
2563 );
2564 assert!(reviewer.exists());
2565 std::fs::remove_file(oversized).unwrap();
2566 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
2569 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
2570 assert!(
2571 crate::graph::fold_run(&mut state, true, &f.home)
2572 .await
2573 .is_err()
2574 );
2575 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
2576 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
2577 .await
2578 .unwrap();
2579 std::fs::write(&checkpoint, &head).unwrap();
2580 assert!(
2581 crate::graph::fold_run(&mut state, true, &f.home)
2582 .await
2583 .is_err()
2584 );
2585 std::fs::remove_file(checkpoint).unwrap();
2586 let (folded, unreadable) = crate::clean::fold_due(
2587 &f.home.join("runs"),
2588 &f.home,
2589 &f.bay,
2590 &Fx::disk(),
2591 Fx::now(),
2592 )
2593 .await
2594 .unwrap();
2595 if folded != 1 {
2596 panic!(
2597 "normal cleanup failed: {:#}",
2598 crate::graph::fold_run(&mut state, true, &f.home)
2599 .await
2600 .unwrap_err()
2601 );
2602 }
2603 assert_eq!((folded, unreadable), (1, 0));
2604 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
2605 state = RunState::load_under(&state.id, &f.home).unwrap();
2606 assert!(state.candidates[0].folded);
2607 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
2608 crate::graph::fold_run(&mut state, true, &f.home)
2609 .await
2610 .unwrap();
2611 assert_eq!(
2612 std::fs::read(evidence.join("manifest.json")).unwrap(),
2613 manifest
2614 );
2615 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
2616 .await
2617 .unwrap();
2618 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
2619 assert!(
2620 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
2621 .await
2622 .unwrap()
2623 .ok()
2624 );
2625 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
2628 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
2629 .as_str()
2630 .unwrap();
2631 let recovered = f.home.join("recovered");
2632 std::fs::create_dir_all(&recovered).unwrap();
2633 git::git(&recovered, &["init"]).await.unwrap();
2634 std::fs::write(
2635 recovered.join("source"),
2636 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
2637 )
2638 .unwrap();
2639 git::git(
2640 &recovered,
2641 &[
2642 "apply",
2643 evidence.join(format!("{head}.patch")).to_str().unwrap(),
2644 ],
2645 )
2646 .await
2647 .unwrap();
2648 assert_eq!(
2649 std::fs::read_to_string(recovered.join("source")).unwrap(),
2650 "cc05"
2651 );
2652 assert!(
2653 !std::fs::read(evidence.join(format!("{head}.patch")))
2654 .unwrap()
2655 .is_empty()
2656 );
2657 assert_eq!(
2658 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
2659 "review findings"
2660 );
2661 if missing {
2662 assert_eq!(
2663 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
2664 "original snapshot"
2665 );
2666 } else {
2667 assert!(!artifacts.join("cand-A.patch").exists());
2668 }
2669 assert!(run_dir.join("run.json").exists());
2670 }
2671 }
2672
2673 #[tokio::test]
2674 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
2675 use std::os::unix::fs::PermissionsExt as _;
2676 let f = Fx::new().await;
2677 let source = f.run("dd03", true).await;
2678 let empty = f.run("dd04", false).await;
2679 let task = f.task(&[source, empty.clone()]);
2680 let entered = f.repo.join("proof-entered");
2681 let release = f.repo.join("proof-release");
2682 let hook = f.repo.join("proof-monitor");
2683 std::fs::write(&hook, format!(
2684 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
2685 entered.display(), release.display())).unwrap();
2686 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
2687 git::git(
2688 &f.repo,
2689 &["config", "core.fsmonitor", hook.to_str().unwrap()],
2690 )
2691 .await
2692 .unwrap();
2693 struct ReleaseProof(PathBuf);
2694 impl Drop for ReleaseProof {
2695 fn drop(&mut self) {
2696 let _ = std::fs::write(&self.0, "continue");
2697 }
2698 }
2699 let _release_on_panic = ReleaseProof(release.clone());
2700 let home = f.home.clone();
2701 let sweep = tokio::spawn(async move {
2702 let mut report = Report::default();
2703 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
2704 report
2705 });
2706 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2707 while !entered.exists() {
2708 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2709 }
2710 })
2711 .await
2712 .unwrap();
2713 let queue = crate::queue::Queue::at(f.home.join("queue"));
2714 let mut selected = queue.get(&task.id).unwrap();
2715 selected.resume_override = Some(crate::queue::OperatorResume {
2716 question_id: "during-slow-proof".into(),
2717 at: Fx::now(),
2718 conductor_rehold: None,
2719 forced: false,
2720 pinned_run: Some(empty.id.clone()),
2721 });
2722 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
2723 let quick = match tokio::time::timeout(std::time::Duration::from_secs(2), &mut update).await
2724 {
2725 Ok(result) => {
2726 result.unwrap().unwrap();
2727 true
2728 }
2729 Err(_) => {
2730 std::fs::write(&release, "continue").unwrap();
2731 update.await.unwrap().unwrap();
2732 false
2733 }
2734 };
2735 std::fs::write(release, "continue").unwrap();
2736 let report = tokio::time::timeout(std::time::Duration::from_secs(8), sweep)
2737 .await
2738 .unwrap()
2739 .unwrap();
2740 assert!(
2741 quick,
2742 "task writers must not wait behind Git retirement proof"
2743 );
2744 assert_eq!(report.retired, 0);
2745 assert!(report.retained.iter().any(|r| r.contains("task changed")));
2746 assert!(empty.candidates[0].worktree.exists());
2747 assert!(
2748 RunState::load_under(&empty.id, &f.home)
2749 .unwrap()
2750 .retired_to
2751 .is_none()
2752 );
2753 }
2754
2755 #[tokio::test]
2756 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
2757 let f = Fx::new().await;
2758 let source = f.run("dd01", true).await;
2759 let mut empty = f.run("dd02", false).await;
2760 let expected = f.task(&[source.clone(), empty.clone()]);
2761 let queue = crate::queue::Queue::at(f.home.join("queue"));
2762 let _claim = queue.claim(&expected.id).unwrap();
2763 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
2765 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
2766 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
2767 let mut selected = queue.get(&expected.id).unwrap();
2768 selected.resume_override = Some(crate::queue::OperatorResume {
2769 question_id: "new-owner-decision".into(),
2770 at: Fx::now(),
2771 conductor_rehold: None,
2772 forced: false,
2773 pinned_run: Some(empty.id.clone()),
2774 });
2775 queue.put(&mut selected).unwrap();
2776 let error = commit_retirement(
2777 &f.home,
2778 &queue,
2779 &expected,
2780 &mut empty,
2781 &source.id,
2782 &expected.runs,
2783 Fx::now(),
2784 )
2785 .unwrap_err();
2786 assert!(error.to_string().contains("task changed"));
2787 assert!(
2788 RunState::load_under(&empty.id, &f.home)
2789 .unwrap()
2790 .retired_to
2791 .is_none()
2792 );
2793 assert!(empty.candidates[0].worktree.exists());
2794 assert_eq!(
2795 queue
2796 .get(&expected.id)
2797 .unwrap()
2798 .resume_override
2799 .unwrap()
2800 .pinned_run,
2801 Some(empty.id.clone())
2802 );
2803 assert!(
2804 !f.home
2805 .join("recovery-lineages")
2806 .join(format!("{}.json", expected.id))
2807 .exists()
2808 );
2809 assert!(
2811 commit_retirement(
2812 &f.home,
2813 &queue,
2814 &selected,
2815 &mut empty,
2816 &source.id,
2817 &selected.runs,
2818 Fx::now()
2819 )
2820 .unwrap_err()
2821 .to_string()
2822 .contains("selected this recovery source")
2823 );
2824 }
2825
2826 #[tokio::test]
2827 async fn unknown_targets_cargo_owners_and_driver_resumes_are_protected() {
2828 let f = Fx::new().await;
2829 let state = f.run("cccc", false).await;
2830 let wt = &state.candidates[0].worktree;
2831 Fx::target(wt);
2832 std::fs::write(wt.join("target/notes"), "user file").unwrap();
2833 assert!(cargo_target(wt).await.is_err());
2834 std::fs::remove_file(wt.join("target/notes")).unwrap();
2835 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
2836 assert!(
2837 !declared_outputs(&wt.join("target"))
2838 .unwrap()
2839 .contains(&wt.join("target/debug/link"))
2840 );
2841 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
2842 let cargo = cargo_locks(&wt.join("target")).unwrap();
2843 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2844 assert_eq!(out.freed_bytes, 0);
2845 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
2846 drop(cargo);
2847 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2848 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2849 assert_eq!(out.freed_bytes, 0);
2850 drop(driver);
2851 let exclusive = OutputLease::take(wt, true).unwrap();
2852 assert!(
2853 OutputLease::borrow(wt).is_err(),
2854 "resume cannot start a writer during cleanup"
2855 );
2856 drop(exclusive);
2857 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
2858 }
2859
2860 #[tokio::test]
2861 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
2862 let f = Fx::new().await;
2863 let source = f.run("0000", true).await;
2864 let empty = f.run("0001", false).await;
2865 let changed = f.run("0002", false).await;
2866 let unknown = f.run("0003", false).await;
2867 let unproven = f.run("0004", false).await;
2868 f.task(&[
2869 source.clone(),
2870 empty.clone(),
2871 changed.clone(),
2872 unknown.clone(),
2873 unproven.clone(),
2874 ]);
2875 for run in [&empty, &changed, &unknown] {
2876 Fx::target(&run.candidates[0].worktree);
2877 }
2878 let reviewer = empty.worktree_root().join("review-1");
2879 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2880 .await
2881 .unwrap();
2882 Fx::target(&reviewer);
2883 let mut disk = Fx::disk();
2884 disk.auto_fold = false;
2885 let report = sweep(&f.home, &disk, Fx::now()).await;
2886 assert_eq!(report.freed_bytes, 800, "{report:?}");
2887 let wt = &empty.candidates[0].worktree;
2888 assert!(wt.join("target/debug/output.d").exists());
2889 assert!(!wt.join("target/debug/output").exists());
2890 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
2891 let proof: CargoResidue =
2892 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
2893 assert_eq!(
2894 proof.files[Path::new("debug/output.d")],
2895 std::fs::read(wt.join("target/debug/output.d")).unwrap()
2896 );
2897 std::fs::write(
2898 changed.candidates[0].worktree.join("target/debug/output.d"),
2899 "owner notes",
2900 )
2901 .unwrap();
2902 std::fs::write(
2903 unknown.candidates[0].worktree.join("target/debug/notes"),
2904 "keep",
2905 )
2906 .unwrap();
2907 Fx::target(&unproven.candidates[0].worktree);
2908 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
2909 disk.auto_fold = true;
2910 let locks = cargo_locks(&wt.join("target")).unwrap();
2911 let mut report = Report::default();
2912 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2913 assert_eq!(report.retired, 0, "{report:?}");
2914 assert!(
2915 report
2916 .retained
2917 .iter()
2918 .any(|reason| reason.contains("Cargo build"))
2919 );
2920 drop(locks);
2921 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2922 assert_eq!(report.retired, 1, "{report:?}");
2923 assert!(!wt.exists());
2924 assert!(!reviewer.exists());
2925 assert!(
2926 residue_proof(&f.home, &empty.id, &reviewer)
2927 .unwrap()
2928 .exists()
2929 );
2930 assert!(
2931 proof_path.exists(),
2932 "metadata evidence outlives the worktree"
2933 );
2934 assert_eq!(
2935 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
2936 Some(source.id)
2937 );
2938 for run in [&changed, &unknown, &unproven] {
2939 assert!(run.candidates[0].worktree.exists());
2940 assert!(
2941 RunState::load_under(&run.id, &f.home)
2942 .unwrap()
2943 .retired_to
2944 .is_none()
2945 );
2946 }
2947 }
2948
2949 #[tokio::test]
2950 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
2951 let f = Fx::new().await;
2952 let source = f.run("0000", true).await;
2953 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
2954 .await
2955 .unwrap();
2956 let mut runs = vec![source.clone()];
2957 for n in 1..=12 {
2958 runs.push(f.run(&format!("{n:04}"), false).await);
2959 }
2960 let dirty = runs[1].candidates[0].worktree.clone();
2961 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
2962 let ignored = runs[2].candidates[0].worktree.clone();
2963 std::fs::create_dir(ignored.join("scratch")).unwrap();
2964 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
2965 runs[1].candidates.clear();
2968 runs[1].save_under(&f.home).unwrap();
2969 let task = f.task(&runs);
2970 let mut total = 0;
2971 for _ in 0..4 {
2972 let mut report = Report::default();
2973 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2974 assert!(report.retired <= 5);
2975 total += report.retired;
2976 }
2977 assert_eq!(total, 10);
2978 assert!(dirty.exists());
2979 assert!(ignored.exists());
2980 assert_eq!(
2981 git::rev_parse(&f.repo, &source.candidates[0].branch)
2982 .await
2983 .unwrap(),
2984 source_head
2985 );
2986 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
2987 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
2988 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
2989 let mut released = task.clone();
2990 released.release();
2991 assert_eq!(
2992 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
2993 RunState::load_under(id, &f.home)
2994 }),
2995 Some(source.id.clone())
2996 );
2997
2998 assert!(
2999 f.home
3000 .join("runs")
3001 .join(&retired.id)
3002 .join("run.json")
3003 .exists()
3004 );
3005 assert!(
3006 f.home
3007 .join("recovery-lineages")
3008 .join(format!("{}.json", task.id))
3009 .exists()
3010 );
3011 }
3012
3013 #[tokio::test]
3014 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
3015 let f = Fx::new().await;
3016 let a = f.run("dddd", true).await;
3017 let b = f.run("eeee", true).await;
3018 let empty = f.run("ffff", false).await;
3019 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
3020 assert!(
3021 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
3022 "unfinished queue history protects older source branches from ordinary folding"
3023 );
3024 let mut prior = a.clone();
3025 assert!(
3026 crate::graph::fold_run(&mut prior, true, &f.home)
3027 .await
3028 .unwrap_err()
3029 .to_string()
3030 .contains(&task.id)
3031 );
3032 let mut report = Report::default();
3033 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3034 assert_eq!(report.retired, 0);
3035 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
3036 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3037 Fx::target(&a.candidates[0].worktree);
3038 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3039 assert!(
3040 report.freed_bytes > 0,
3041 "source retention does not retain regenerable outputs"
3042 );
3043 assert!(a.candidates[0].worktree.exists());
3044 assert!(b.candidates[0].worktree.exists());
3045 assert!(empty.candidates[0].worktree.exists());
3046 }
3047
3048 #[tokio::test]
3049 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
3050 let f = Fx::new().await;
3051 let authority = f.run("p000", true).await;
3052 let question_run = f.run("p001", false).await;
3053 let mut checkpoint = f.run("p002", false).await;
3054 checkpoint.land_approval = Some(crate::run::LandApproval {
3055 question: "pending".into(),
3056 head: f.base.clone(),
3057 });
3058 checkpoint.save_under(&f.home).unwrap();
3059 let retained = f.run("p003", false).await;
3060 std::fs::write(
3061 f.home
3062 .join("runs")
3063 .join(&retained.id)
3064 .join("retain-recovery-sources"),
3065 "owner",
3066 )
3067 .unwrap();
3068 let mut partial = f.run("p004", false).await;
3069 let reviewer = partial.worktree_root().join("review-1");
3070 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
3071 .await
3072 .unwrap();
3073 partial.retired_to = Some(authority.id.clone());
3075 partial.save_under(&f.home).unwrap();
3076 assert!(
3077 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
3078 .await
3079 .unwrap()
3080 );
3081 let git_checkpoint = f.run("p005", false).await;
3082 let location = git::git(
3083 &git_checkpoint.candidates[0].worktree,
3084 &[
3085 "rev-parse",
3086 "--path-format=absolute",
3087 "--git-path",
3088 "sequencer",
3089 ],
3090 )
3091 .await
3092 .unwrap();
3093 std::fs::create_dir(location.trim()).unwrap();
3094 let mut preservation = f.run("p006", false).await;
3095 preservation
3096 .candidate_preservation
3097 .insert('A', "pending preservation".into());
3098 preservation.save_under(&f.home).unwrap();
3099 let task = f.task(&[
3100 preservation.clone(),
3101 git_checkpoint.clone(),
3102 authority.clone(),
3103 question_run.clone(),
3104 checkpoint.clone(),
3105 retained.clone(),
3106 partial.clone(),
3107 ]);
3108 let questions = crate::ask::Questions::at(f.home.join("questions"));
3109 let mut question = crate::ask::Question::new(
3110 question_run.id.clone(),
3111 "implement".into(),
3112 "fixture".into(),
3113 "owner decision".into(),
3114 String::new(),
3115 vec!["keep".into()],
3116 );
3117 questions.put(&mut question).unwrap();
3118 let mut report = Report::default();
3119 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3120 assert_eq!(report.retired, 1, "{report:?}");
3121 assert!(!reviewer.exists());
3122 for state in [
3123 &question_run,
3124 &checkpoint,
3125 &retained,
3126 &git_checkpoint,
3127 &preservation,
3128 ] {
3129 assert!(state.candidates[0].worktree.exists());
3130 assert!(
3131 RunState::load_under(&state.id, &f.home)
3132 .unwrap()
3133 .retired_to
3134 .is_none()
3135 );
3136 }
3137 assert!(questions.get(&question.id).unwrap().status.open());
3138 assert_eq!(
3139 crate::queue::Queue::at(f.home.join("queue"))
3140 .get(&task.id)
3141 .unwrap()
3142 .status,
3143 task.status
3144 );
3145 }
3146
3147 #[tokio::test]
3148 async fn preservation_schema_migration_remains_distinct_from_retirement() {
3149 let f = Fx::new().await;
3150 let mut state = f.run("p007", false).await;
3151 state.schema = 15;
3152 state.save_under(&f.home).unwrap();
3153 let native = RunState::load_under(&state.id, &f.home).unwrap();
3154 assert_eq!(native.schema, crate::run::SCHEMA);
3155 assert!(native.candidate_preservation.is_empty());
3156 assert!(native.retired_to.is_none());
3157 state.schema = 14;
3158 state.save_under(&f.home).unwrap();
3159 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
3160 assert!(legacy.candidate_preservation.contains_key(&'A'));
3161 legacy.save_under(&f.home).unwrap();
3162 let restored = RunState::load_under(&state.id, &f.home).unwrap();
3163 assert_eq!(
3164 restored.candidate_preservation,
3165 legacy.candidate_preservation
3166 );
3167 }
3168
3169 #[tokio::test]
3170 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
3171 let f = Fx::new().await;
3172 let cache = f.repo.join("cache");
3173 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
3174 std::fs::create_dir_all(&cache).unwrap();
3175 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3176 assert!(
3177 crate::cache::maintenance_prune(&f.home, &cache, 1)
3178 .unwrap()
3179 .is_none()
3180 );
3181 drop(guard);
3182 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3183 }
3184
3185 #[tokio::test]
3186 async fn launcher_death_does_not_release_live_tool_output_reservations() {
3187 let f = Fx::new().await;
3188 let run = f.run("abcd", false).await;
3189 let wt = &run.candidates[0].worktree;
3190 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
3191 helper
3192 .quiet()
3193 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
3194 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
3195 .env("MAGI_TEST_WORKTREE", wt)
3196 .stdin(std::process::Stdio::null())
3197 .stdout(std::process::Stdio::null())
3198 .stderr(std::process::Stdio::null())
3199 .kill_on_drop(true);
3200 assert!(
3201 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
3202 .await
3203 .unwrap()
3204 .unwrap()
3205 .success()
3206 );
3207 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
3208 .unwrap()
3209 .trim()
3210 .parse()
3211 .unwrap();
3212 struct KillTool(i32);
3213 impl Drop for KillTool {
3214 fn drop(&mut self) {
3215 unsafe { libc::kill(self.0, libc::SIGKILL) };
3217 }
3218 }
3219 let tool = KillTool(pid);
3220 assert!(OutputLease::take(wt, true).is_err());
3221 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
3222 drop(tool);
3223 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3224 loop {
3225 if OutputLease::take(wt, true).is_ok()
3226 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
3227 {
3228 break;
3229 }
3230 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3231 }
3232 })
3233 .await
3234 .unwrap();
3235 }
3236
3237 #[tokio::test]
3238 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
3239 let f = Fx::new().await;
3240 let run = f.run("abcd", false).await;
3241 let wt = &run.candidates[0].worktree;
3242 let cache = f.repo.join("cache");
3243 std::fs::create_dir(&cache).unwrap();
3244 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
3245 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
3246 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
3247 cmd.quiet()
3248 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
3249 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
3250 .stdin(std::process::Stdio::null())
3251 .stdout(std::process::Stdio::null())
3252 .stderr(std::process::Stdio::null())
3253 .kill_on_drop(true);
3254 if clear_owner {
3255 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
3256 } else {
3257 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
3258 }
3259 let mut running = crate::seat_process::Running::spawn(
3260 cmd,
3261 None,
3262 vec![
3263 OutputLease::borrow(wt).unwrap().unwrap(),
3264 OutputLease::cache(&f.home, &cache, false).unwrap(),
3265 ],
3266 )
3267 .unwrap();
3268 assert!(
3269 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
3270 .await
3271 .unwrap()
3272 .unwrap()
3273 .success()
3274 );
3275 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
3276 struct KillTool(Option<i32>);
3277 impl Drop for KillTool {
3278 fn drop(&mut self) {
3279 if let Some(pid) = self.0 {
3281 unsafe { libc::kill(pid, libc::SIGKILL) };
3282 }
3283 }
3284 }
3285 let mut tool = KillTool(Some(pid));
3286 assert!(OutputLease::take(wt, true).is_err());
3290 let finish = if cancel {
3291 drop(running);
3292 None
3293 } else {
3294 Some(tokio::spawn(async move { running.finish().await }))
3295 };
3296 if clear_owner {
3297 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3298 assert!(crate::proc::pid_alive(pid as u32));
3299 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
3300 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3301 assert!(
3302 crate::cache::maintenance_prune(&f.home, &cache, 0)
3303 .unwrap()
3304 .is_none()
3305 );
3306 if let Some(finish) = &finish {
3307 assert!(!finish.is_finished());
3308 }
3309 unsafe { libc::kill(pid, libc::SIGKILL) };
3311 }
3312 if let Some(finish) = finish {
3313 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
3314 .await
3315 .unwrap()
3316 .unwrap();
3317 }
3318 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3319 loop {
3320 if OutputLease::take(wt, true).is_ok()
3321 && OutputLease::cache(&f.home, &cache, true).is_ok()
3322 {
3323 if !clear_owner {
3325 assert!(!crate::proc::pid_alive(pid as u32));
3326 }
3327 break;
3328 }
3329 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3330 }
3331 })
3332 .await
3333 .unwrap();
3334 tool.0 = None;
3335 }
3336 }
3337
3338 #[tokio::test]
3339 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
3340 let f = Fx::new().await;
3341 let run = f.run("abcd", false).await;
3342 let wt = &run.candidates[0].worktree;
3343 let cache = f.repo.join("cache");
3344 std::fs::create_dir(&cache).unwrap();
3345 let pid_file = f.repo.join("child-pid");
3346 let mut cmd = tokio::process::Command::new("sh");
3347 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
3348 .quiet()
3349 .env("CHILD_PID_FILE", &pid_file)
3350 .process_group(0)
3351 .kill_on_drop(true);
3352 let running = crate::seat_process::Running::spawn(
3353 cmd,
3354 None,
3355 vec![
3356 OutputLease::borrow(wt).unwrap().unwrap(),
3357 OutputLease::cache(&f.home, &cache, false).unwrap(),
3358 ],
3359 )
3360 .unwrap();
3361 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3362 while !pid_file.exists() {
3363 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3364 }
3365 })
3366 .await
3367 .unwrap();
3368 assert!(OutputLease::take(wt, true).is_err());
3369 assert!(
3370 crate::cache::maintenance_prune(&f.home, &cache, 0)
3371 .unwrap()
3372 .is_none()
3373 );
3374 let pid: u32 = std::fs::read_to_string(pid_file)
3375 .unwrap()
3376 .trim()
3377 .parse()
3378 .unwrap();
3379 drop(running);
3380 tokio::time::timeout(std::time::Duration::from_secs(10), async {
3381 loop {
3382 if let Ok(guard) = OutputLease::take(wt, true) {
3383 assert!(
3384 !crate::proc::pid_alive(pid),
3385 "output lock released before tool death"
3386 );
3387 drop(guard);
3388 break;
3389 }
3390 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3391 }
3392 })
3393 .await
3394 .unwrap();
3395 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3396 }
3397
3398 #[tokio::test]
3399 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
3400 let f = Fx::new().await;
3401 let reason = tokio::time::timeout(
3402 std::time::Duration::from_secs(2),
3403 emergency(vec![f.repo.clone()], u64::MAX),
3404 )
3405 .await
3406 .unwrap();
3407 assert!(reason.contains("disk pressure"));
3408 assert!(
3409 tokio::time::timeout(
3410 std::time::Duration::from_millis(10),
3411 emergency(vec![f.repo.clone()], 0)
3412 )
3413 .await
3414 .is_err()
3415 );
3416 assert_eq!(
3417 std::fs::read_to_string(f.repo.join("source")).unwrap(),
3418 "base\n"
3419 );
3420 }
3421
3422 #[tokio::test]
3423 async fn automatic_requeues_cannot_refund_failed_input_generation() {
3424 let f = Fx::new().await;
3425 let mut run = f.run("1234", false).await;
3426 run.prep_failure = Some("cherry-pick conflict".into());
3427 let mut task = f.task(std::slice::from_ref(&run));
3428 let inputs = crate::queue::recovery_inputs(&run.sources);
3429 for _ in 0..108 {
3430 task.recovery_failed(&run);
3431 task.requeue();
3432 assert_eq!(task.attempts, 0);
3433 }
3434 let bytes = serde_json::to_vec(&task).unwrap();
3435 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
3436 assert!(
3437 restored
3438 .recovery_exhausted(&run.base_commit, &inputs, 2)
3439 .is_some()
3440 );
3441 assert!(
3442 restored
3443 .recovery_exhausted("different-base", &inputs, 2)
3444 .is_none()
3445 );
3446 let original_base = run.base_commit.clone();
3447 run.base_commit = "different-base".into();
3448 restored.recovery_failed(&run);
3449 assert!(
3450 restored
3451 .recovery_exhausted(&original_base, &inputs, 2)
3452 .is_some(),
3453 "returning to old inputs does not replenish their budget"
3454 );
3455 assert!(
3456 restored
3457 .recovery_exhausted(&run.base_commit, &inputs, 2)
3458 .is_none()
3459 );
3460 restored.release_operator("test-owner", None);
3461 assert!(
3462 restored
3463 .recovery_exhausted(&run.base_commit, &inputs, 2)
3464 .is_none()
3465 );
3466 }
3467}