1pub mod reconcile;
4
5use crate::{
6 git,
7 proc::Quiet as _,
8 run::{Liveness, RunState},
9};
10use anyhow::{Context, Result, ensure};
11use jiff::Timestamp;
12use serde::{Deserialize, Serialize};
13use std::{
14 fs::{File, OpenOptions},
15 path::{Path, PathBuf},
16};
17
18#[derive(Debug)]
22pub(crate) struct OutputLease {
23 _file: File,
24}
25
26impl OutputLease {
27 #[cfg(unix)]
28 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
29 use std::os::fd::AsRawFd;
30 self._file.as_raw_fd()
31 }
32
33 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
34 if !worktree.join(".git").is_file() {
35 return Ok(None);
36 }
37 Self::take(worktree, false).map(Some)
38 }
39 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
40 use std::hash::{Hash, Hasher};
41 let mut hash = std::collections::hash_map::DefaultHasher::new();
42 let mut normalized = cache.to_owned();
43 let mut missing = Vec::new();
44 while !normalized.exists() {
45 let Some(name) = normalized.file_name() else {
46 break;
47 };
48 missing.push(name.to_owned());
49 if !normalized.pop() {
50 break;
51 }
52 }
53 normalized = normalized.canonicalize().unwrap_or(normalized);
54 for part in missing.into_iter().rev() {
55 normalized.push(part);
56 }
57 normalized.hash(&mut hash);
58 let dir = home.join("cache-leases");
59 std::fs::create_dir_all(&dir)?;
60 Self::lock(
61 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
62 exclusive,
63 )
64 }
65 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
66 let out = std::process::Command::new("git")
67 .args(["rev-parse", "--absolute-git-dir"])
68 .current_dir(worktree)
69 .quiet()
70 .output()?;
71 ensure!(
72 out.status.success(),
73 "cannot identify output lease directory"
74 );
75 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
76 Self::lock(&dir.join("magi-output.lock"), exclusive)
77 }
78 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
79 let file = OpenOptions::new()
80 .create(true)
81 .truncate(false)
82 .read(true)
83 .write(true)
84 .open(path)?;
85 #[cfg(unix)]
86 {
87 use std::os::fd::AsRawFd;
88 let mode = if exclusive {
89 libc::LOCK_EX
90 } else {
91 libc::LOCK_SH
92 };
93 ensure!(
95 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
96 "worktree outputs are in use or being reclaimed"
97 );
98 }
99 #[cfg(not(unix))]
100 let _ = exclusive;
101 Ok(Self { _file: file })
102 }
103}
104#[cfg(unix)]
105impl Drop for OutputLease {
106 fn drop(&mut self) {
107 use std::os::fd::AsRawFd;
108 unsafe {
110 libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
111 }
112 }
113}
114
115#[derive(Default, Debug, Serialize, Deserialize)]
116#[serde(default)]
117pub(crate) struct Report {
118 pub checked_at: Option<Timestamp>,
119 pub limit_bytes: u64,
120 pub reclaim_budget_bytes: u64,
121 pub measured_bytes: u64,
122 pub freed_bytes: u64,
123 pub retired: usize,
124 pub retained: Vec<String>,
125 pub failures: Vec<String>,
126}
127
128fn plain_dir(path: &Path) -> bool {
129 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
130}
131
132fn run_paths_missing(state: &RunState) -> Result<bool> {
135 for path in [state.worktree_root(), state.repo.clone()] {
136 match std::fs::metadata(&path) {
137 Ok(_) => {}
138 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
139 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
140 }
141 }
142 Ok(false)
143}
144
145async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
147 let root = state.worktree_root();
148 if !plain_dir(&root) {
149 return Ok(Vec::new());
150 }
151 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
152 registered_worktrees(state, &list)
153}
154
155fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
156 let root = state.worktree_root();
157 if !plain_dir(&root) {
158 return Ok(Vec::new());
159 }
160 let root = root.canonicalize()?;
161 let mut paths = Vec::new();
162 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
163 let path = PathBuf::from(path);
164 if plain_dir(&path)
165 && path.parent() == Some(root.as_path())
166 && path.canonicalize()? == path
167 && path.join(".git").is_file()
168 {
169 paths.push(path);
170 }
171 }
172 Ok(paths)
173}
174
175async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
178 let target = worktree.join("target");
179 ensure!(plain_dir(&target), "target absent or not a plain directory");
180 ensure!(target.canonicalize()? == target, "target path changed");
181 ensure!(
182 git::git(worktree, &["ls-files", "--", "target"])
183 .await?
184 .is_empty(),
185 "tracked target contents"
186 );
187 ensure!(
188 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
189 .await?
190 .ok(),
191 "target is not ignored"
192 );
193 for name in ["CACHEDIR.TAG", ".rustc_info.json"] {
194 let path = target.join(name);
195 ensure!(
196 std::fs::symlink_metadata(&path)?.is_file() && path.canonicalize()? == path,
197 "cache identity is not a plain file"
198 );
199 }
200 let tag = std::fs::read_to_string(target.join("CACHEDIR.TAG"))?;
201 ensure!(
202 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55d"),
203 "unknown Cargo cache tag"
204 );
205 let info: serde_json::Value =
206 serde_json::from_slice(&std::fs::read(target.join(".rustc_info.json"))?)?;
207 ensure!(
208 info.get("rustc_fingerprint").is_some(),
209 "unknown Cargo identity"
210 );
211 for entry in std::fs::read_dir(&target)? {
212 let entry = entry?;
213 let name = entry.file_name();
214 let name = name.to_str().context("non-UTF8 output")?;
215 ensure!(
216 matches!(
217 name,
218 "debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json"
219 ),
220 "unrecognized target entry {name}"
221 );
222 if matches!(name, "debug" | "release") {
223 ensure!(
224 plain_dir(&entry.path()) && entry.path().canonicalize()? == entry.path(),
225 "profile is not a plain directory"
226 );
227 }
228 }
229 Ok(target)
230}
231
232fn artifact_signature(path: &Path) -> Result<bool> {
233 use std::io::Read as _;
234 let mut header = [0; 8];
235 let n = File::open(path)?.read(&mut header)?;
236 let header = &header[..n];
237 Ok(header.starts_with(b"\x7fELF")
238 || header.starts_with(b"!<arch>\n")
239 || header.starts_with(b"rust\0\0\0")
240 || header.starts_with(b"MZ\x90\0")
241 || [
242 [0xfe, 0xed, 0xfa, 0xce],
243 [0xce, 0xfa, 0xed, 0xfe],
244 [0xfe, 0xed, 0xfa, 0xcf],
245 [0xcf, 0xfa, 0xed, 0xfe],
246 [0xca, 0xfe, 0xba, 0xbe],
247 [0xca, 0xfe, 0xba, 0xbf],
248 ]
249 .iter()
250 .any(|magic| header.starts_with(magic)))
251}
252
253fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
257 let mut outputs = std::collections::BTreeSet::new();
258 for profile in ["debug", "release"] {
259 for dir in [target.join(profile), target.join(profile).join("deps")] {
260 if !plain_dir(&dir) || dir.canonicalize()? != dir {
261 continue;
262 }
263 for entry in std::fs::read_dir(&dir)? {
264 let path = entry?.path();
265 if path.extension().is_none_or(|ext| ext != "d")
266 || !std::fs::symlink_metadata(&path)?.is_file()
267 || std::fs::metadata(&path)?.len() > 1024 * 1024
268 || path.canonicalize()? != path
269 {
270 continue;
271 }
272 let body = std::fs::read_to_string(&path)?;
273 for line in body.lines() {
274 let Some((names, dependencies)) = line.split_once(": ") else {
275 continue;
276 };
277 if dependencies.trim().is_empty() {
278 continue;
279 }
280 let mut words = Vec::new();
283 let mut word = String::new();
284 let mut escaped = false;
285 for c in names.chars() {
286 if escaped {
287 word.push(c);
288 escaped = false;
289 } else if c == '\\' {
290 escaped = true;
291 } else if c.is_whitespace() {
292 if !word.is_empty() {
293 words.push(std::mem::take(&mut word));
294 }
295 } else {
296 word.push(c);
297 }
298 }
299 if escaped || word.contains(['$', '%']) {
300 continue;
301 }
302 if !word.is_empty() {
303 words.push(word);
304 }
305 for name in words {
306 let output = PathBuf::from(name);
307 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
308 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
309 let artifact_name =
310 output_stem == stem || output_stem == format!("lib{stem}");
311 let artifact_kind = output.extension().is_none_or(|ext| {
312 matches!(
313 ext.to_str(),
314 Some(
315 "rlib"
316 | "rmeta"
317 | "so"
318 | "dylib"
319 | "a"
320 | "lib"
321 | "dll"
322 | "exe"
323 | "pdb"
324 )
325 )
326 });
327 if artifact_name
328 && artifact_kind
329 && output.parent() == Some(dir.as_path())
330 && output.extension().is_none_or(|ext| ext != "d")
331 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
332 && output.canonicalize().is_ok_and(|p| p == output)
333 && artifact_signature(&output)?
334 {
335 outputs.insert(output);
336 }
337 }
338 }
339 }
340 }
341 }
342 Ok(outputs.into_iter().collect())
343}
344
345#[derive(Serialize, Deserialize)]
346struct CargoResidue {
347 target: PathBuf,
348 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
349}
350
351fn residue_files(
354 target: &Path,
355 outputs: &[PathBuf],
356) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
357 let mut files = std::collections::BTreeMap::new();
358 let mut dirs = vec![target.to_owned()];
359 let mut bytes = 0usize;
360 let mut entries = 0usize;
361 while let Some(dir) = dirs.pop() {
362 for entry in std::fs::read_dir(dir)? {
363 let path = entry?.path();
364 entries += 1;
365 ensure!(entries <= 4096, "residue inventory too large");
366 let metadata = std::fs::symlink_metadata(&path)?;
367 ensure!(
368 !metadata.file_type().is_symlink(),
369 "symlink in Cargo residue"
370 );
371 if metadata.is_dir() {
372 dirs.push(path);
373 } else {
374 ensure!(metadata.is_file(), "non-file in Cargo residue");
375 if outputs.contains(&path) {
376 continue;
377 }
378 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
379 let body = std::fs::read(&path)?;
380 bytes = bytes.saturating_add(body.len());
381 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
382 files.insert(path.strip_prefix(target)?.to_owned(), body);
383 }
384 }
385 }
386 Ok(files)
387}
388
389fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
390 Ok(home
391 .join("runs")
392 .join(id)
393 .join("build-residue")
394 .join(worktree.file_name().context("missing worktree name")?))
395}
396
397fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
398 let files = residue_files(target, outputs)?;
399 for (relative, body) in &files {
400 let path = target.join(relative);
401 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
402 let lock = matches!(
403 relative.to_str(),
404 Some("debug/.cargo-lock" | "release/.cargo-lock")
405 ) && body.is_empty();
406 let dep_info = path.extension().is_some_and(|ext| ext == "d")
407 && outputs.iter().any(|output| {
408 output.parent() == path.parent()
409 && (output.file_stem() == path.file_stem()
410 || output.file_stem().is_some_and(|stem| {
411 stem.to_string_lossy()
412 == format!(
413 "lib{}",
414 path.file_stem().unwrap_or_default().to_string_lossy()
415 )
416 }))
417 });
418 ensure!(identity || lock || dep_info, "unproven Cargo residue");
419 }
420 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
421 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
422 let tmp = proof.with_extension("tmp");
423 std::fs::write(
424 &tmp,
425 serde_json::to_vec(&CargoResidue {
426 target: target.to_owned(),
427 files,
428 })?,
429 )?;
430 std::fs::rename(tmp, proof)?;
431 Ok(())
432}
433
434pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
435 let target = cargo_target(worktree).await?;
436 let locks = cargo_locks(&target)?;
437 let proof: CargoResidue =
438 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
439 ensure!(
440 proof.target == target && proof.files == residue_files(&target, &[])?,
441 "Cargo residue changed or contains unproven files"
442 );
443 Ok(locks)
444}
445
446pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
448 let mut held = Vec::new();
449 for profile in ["debug", "release"] {
450 let lock = target.join(profile).join(".cargo-lock");
451 if lock.exists() {
452 ensure!(
453 std::fs::symlink_metadata(&lock)?.is_file() && lock.canonicalize()? == lock,
454 "Cargo lock is not a plain file"
455 );
456 let file = OpenOptions::new().read(true).write(true).open(lock)?;
457 crate::pr_head::try_lock_file(&file).context("Cargo build is active")?;
458 held.push(file);
459 }
460 }
461 Ok(held)
462}
463
464pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
467 let home = home.to_owned();
468 let disk = disk.clone();
469 let runtime = tokio::runtime::Handle::current();
470 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
471 .await
472 .unwrap_or_else(|e| Report {
473 failures: vec![format!("retention worker failed: {e}")],
474 ..Report::default()
475 })
476}
477
478#[derive(Default, Serialize, Deserialize)]
479struct Measurement {
480 at: Option<Timestamp>,
481 bytes: u64,
482}
483
484async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
485 let mut report = Report {
486 checked_at: Some(now),
487 limit_bytes: disk.worktree_limit_bytes,
488 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
489 ..Report::default()
490 };
491 if !cfg!(unix) {
492 report
493 .failures
494 .push("Automatic output reclamation requires Unix process containment".into());
495 return report;
496 }
497 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
498 return report;
499 };
500 let mut targets = Vec::new();
501 let mut registrations = std::collections::HashMap::new();
502 let measurements_path = home.join("output-measurements.json");
503 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
504 std::fs::read(&measurements_path)
505 .ok()
506 .and_then(|b| serde_json::from_slice(&b).ok())
507 .unwrap_or_default();
508 let mut observed = std::collections::BTreeSet::new();
509 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
510 .into_iter()
511 .flatten()
512 .flatten()
513 .map(|e| e.file_name().to_string_lossy().into_owned())
514 .collect();
515 ids.sort();
516 for id in ids {
517 if !crate::run::is_run_id(&id) {
518 continue;
519 }
520 let state = match RunState::load_under(&id, home) {
521 Ok(state) => state,
522 Err(e) => {
523 report
524 .retained
525 .push(format!("{id}: unreadable run ownership: {e:#}"));
526 continue;
527 }
528 };
529 if disk.worktree_limit_bytes == 0 {
530 continue;
531 }
532 match run_paths_missing(&state) {
533 Ok(true) => continue,
534 Ok(false) => {}
535 Err(e) => {
536 report.failures.push(format!("{id}: {e:#}"));
537 continue;
538 }
539 }
540 let list = match registrations.entry(state.repo.clone()) {
541 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
542 std::collections::hash_map::Entry::Vacant(entry) => {
543 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
544 Ok(list) => entry.insert(list),
545 Err(e) => {
546 if matches!(run_paths_missing(&state), Ok(true)) {
549 continue;
550 }
551 report.failures.push(format!("{id}: {e:#}"));
552 continue;
553 }
554 }
555 }
556 };
557 let paths = match registered_worktrees(&state, list) {
558 Ok(paths) => paths,
559 Err(e) => {
560 report.failures.push(format!("{id}: {e:#}"));
561 continue;
562 }
563 };
564 for path in paths {
565 if !path.join("target").exists() {
566 continue;
567 }
568 let target = path.join("target");
569 observed.insert(target.clone());
570 let pressure =
571 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
572 let measurement = measurements.entry(target).or_default();
573 if pressure
574 || measurement
575 .at
576 .is_none_or(|at| crate::clean::due(now, at, 300))
577 {
578 measurement.bytes = crate::disk::dir_size(&path.join("target"));
579 measurement.at = Some(now);
580 }
581 let size = measurement.bytes;
582 report.measured_bytes = report.measured_bytes.saturating_add(size);
583 targets.push((state.updated_at, id.clone(), path, size));
584 }
585 }
586 targets.sort_by_key(|t| t.0);
587 let mut remaining = report.measured_bytes;
588 for (updated, id, path, size) in targets {
589 if disk.worktree_limit_bytes == 0
590 || remaining <= disk.worktree_limit_bytes
591 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
592 {
593 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
594 continue;
595 }
596 if report.freed_bytes.saturating_add(size) > disk.worktree_reclaim_bytes {
597 report.retained.push(format!(
598 "{id}: target exceeds remaining per-pass byte budget"
599 ));
600 continue;
601 }
602 let result = async {
603 ensure!(
604 crate::clean::due(now, updated, disk.fold_grace_secs),
605 "within grace period"
606 );
607 let _owner = crate::pr_head::owner_lock(home, &id)?;
608 let state = RunState::load_under(&id, home)?;
609 ensure!(
610 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
611 == Liveness::Dead,
612 "run live or ownership unknown"
613 );
614 ensure!(crate::clean::due(now, state.updated_at, disk.fold_grace_secs), "recently used after inventory");
615 let _driver = crate::driver::maintenance(home, &id)?;
616 let _outputs = OutputLease::take(&path, true)?;
617 ensure!(
618 worktrees(&state).await?.contains(&path),
619 "worktree registration changed"
620 );
621 let target = cargo_target(&path).await?;
622 let _cargo = cargo_locks(&target)?;
623 let outputs = declared_outputs(&target)?;
624 ensure!(!outputs.is_empty(), "no dep-info-proven outputs; unknown files retained");
625 let current_size = outputs.iter().try_fold(0u64, |bytes, p| {
626 Ok::<_, std::io::Error>(bytes.saturating_add(std::fs::metadata(p)?.len()))
627 })?;
628 ensure!(report.freed_bytes.saturating_add(current_size) <= disk.worktree_reclaim_bytes,
629 "target grew beyond remaining per-pass budget");
630 let ledger = home.join("runs").join(&id).join("build-outputs.json");
633 std::fs::write(
634 ledger,
635 serde_json::to_vec(
636 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","files":outputs}),
637 )?,
638 )?;
639 if let Err(error) = record_residue(home, &id, &target, &outputs) {
642 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
643 }
644 for output in &outputs {
645 std::fs::remove_file(output)?;
646 }
647 measurements.remove(&target);
648 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
649 Ok::<_, anyhow::Error>(current_size)
650 }
651 .await;
652 match result {
653 Ok(freed) => {
654 report.freed_bytes = report.freed_bytes.saturating_add(freed);
655 remaining = remaining.saturating_sub(freed);
656 }
657 Err(e) => report.retained.push(format!(
658 "{id}/{}: {e:#}",
659 path.file_name().unwrap_or_default().to_string_lossy()
660 )),
661 }
662 }
663 measurements.retain(|path, _| observed.contains(path));
664 if let Ok(body) = serde_json::to_vec(&measurements)
665 && let Err(e) = std::fs::write(&measurements_path, body)
666 {
667 report.failures.push(format!("output measurements: {e}"));
668 }
669 retire_empty(home, disk, now, &mut report).await;
670 if !report.failures.is_empty()
671 || disk.worktree_limit_bytes > 0
672 && remaining > disk.worktree_limit_bytes
673 && report.freed_bytes == 0
674 {
675 crate::notices::raise_in(
676 home,
677 crate::notices::Notice::warn(
678 "disk:retention",
679 "Retained build outputs could not be brought under the disk cap. Inspect the cleanup report for active owners, retained recovery sources and failures.",
680 ),
681 );
682 }
683 let store = home.join("retention.json");
684 if let Ok(body) = serde_json::to_vec_pretty(&report) {
685 let tmp = home.join("retention.json.tmp");
686 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
687 tracing::warn!("retention report could not be saved: {e}");
688 }
689 }
690 tracing::info!(
691 measured_bytes = report.measured_bytes,
692 freed_bytes = report.freed_bytes,
693 retired = report.retired,
694 retained = report.retained.len(),
695 failures = report.failures.len(),
696 "retained worktree cleanup"
697 );
698 for reason in report.retained.iter().chain(&report.failures) {
699 tracing::debug!(%reason, "worktree retained");
700 }
701 report
702}
703
704async fn retire_empty(
705 home: &Path,
706 disk: &crate::config::Disk,
707 now: Timestamp,
708 report: &mut Report,
709) {
710 if !disk.auto_fold {
711 return;
712 }
713 let queue = crate::queue::Queue::at(home.join("queue"));
714 let questions = crate::ask::Questions::at(home.join("questions"));
715 let open = questions.list();
716 for listed in queue.list() {
717 let Ok(_claim) = queue.claim(&listed.id) else {
718 continue;
719 };
720 let Ok(task) = queue.get(&listed.id) else {
721 continue;
722 };
723 if open
724 .iter()
725 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
726 {
727 report.retained.push(format!(
728 "{}: task has an unanswered owner decision",
729 task.id
730 ));
731 continue;
732 }
733 let sources: Vec<_> = task
734 .runs
735 .iter()
736 .collect::<std::collections::BTreeSet<_>>()
737 .into_iter()
738 .cloned()
739 .collect();
740 let mut useful = Vec::new();
741 let mut selected = Vec::new();
742 let mut public = Vec::new();
743 let mut unreadable = false;
744 for id in &sources {
745 let Ok(state) = RunState::load_under(id, home) else {
746 unreadable = true;
747 continue;
748 };
749 if state.released() || state.retired_to.is_some() {
750 continue;
751 }
752 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
753 selected.push(state.clone());
754 }
755 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
756 public.push(state.clone());
757 }
758 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
759 useful.push(state);
760 }
761 }
762 let pool = if !selected.is_empty() {
766 selected
767 } else if !public.is_empty() {
768 public
769 } else {
770 useful
771 };
772 if unreadable || pool.len() != 1 {
773 report.retained.push(format!(
774 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
775 task.id,
776 sources.len()
777 ));
778 continue;
779 }
780 let authority = &pool[0];
781 if authority.candidates.is_empty() {
782 continue;
783 }
784 let latest = &authority.id;
785 for id in &sources {
786 if id == latest || report.retired >= 5 {
787 continue;
788 }
789 if let Ok(state) = RunState::load_under(id, home)
790 && state.retired_to.as_ref() == Some(latest)
791 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
792 {
793 continue;
794 }
795 let result = async {
796 ensure!(
797 !task
798 .resume_override
799 .as_ref()
800 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
801 "operator-pinned recovery run"
802 );
803 ensure!(
804 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
805 "selected replacement recovery source"
806 );
807 let _owner = crate::pr_head::owner_lock(home, id)?;
808 let mut state = RunState::load_under(id, home)?;
809 ensure!(
810 state.repo == authority.repo,
811 "repository differs from recovery authority"
812 );
813 ensure!(
814 !crate::pr_recovery::explicit_sources_retained(home, id),
815 "explicitly retained recovery source"
816 );
817 ensure!(
818 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
819 == Liveness::Dead,
820 "run live or ownership unknown"
821 );
822 let _driver = crate::driver::maintenance(home, id)?;
823 ensure!(
824 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
825 "within grace period"
826 );
827 ensure!(
828 !open.iter().any(|q| q.run == *id && q.status.open()),
829 "unanswered owner decision"
830 );
831 ensure!(
832 state.pr.is_none()
833 && state.pr_head.is_none()
834 && state.pr_recovery.is_none()
835 && state.land_approval.is_none()
836 && state.merge_decision.is_none()
837 && state.reviews.is_empty()
838 && state.recovery.is_none()
839 && state.candidate_preservation.is_empty()
840 && state.interrupted_implementations.is_empty()
841 && !state.released(),
842 "PR ownership or recovery checkpoint"
843 );
844 let proof = prove_empty(home, &state, latest).await?;
845 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
846 for path in &proof.paths {
849 ensure!(
850 git::worktree_remove_clean(&state.repo, path).await?,
851 "Git refused clean worktree removal"
852 );
853 }
854 Ok::<_, anyhow::Error>(())
855 }
856 .await;
857 match result {
858 Ok(()) => report.retired += 1,
859 Err(e) => report.retained.push(format!("{id}: {e:#}")),
860 }
861 }
862 }
863}
864
865struct EmptyProof {
866 paths: Vec<PathBuf>,
867 _outputs: Vec<OutputLease>,
868 _cargo: Vec<File>,
869}
870
871async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
874 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
875 let mut files = Vec::new();
876 let mut blobs = Vec::new();
877 for record in tree.split('\0').filter(|s| !s.is_empty()) {
878 let (header, name) = record.split_once('\t').context("unknown tree record")?;
879 let parts: Vec<_> = header.split_whitespace().collect();
880 ensure!(
881 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
882 "unsupported tracked source kind"
883 );
884 let path = worktree.join(name);
885 let meta = std::fs::symlink_metadata(&path)?;
886 ensure!(
887 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
888 "noncanonical tracked source"
889 );
890 #[cfg(unix)]
891 {
892 use std::os::unix::fs::PermissionsExt as _;
893 ensure!(
894 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
895 "tracked executable mode differs from original base"
896 );
897 }
898 files.push(path.to_str().context("non UTF-8 source")?.to_owned());
899 blobs.push(parts[2].to_owned());
900 }
901 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
902 let mut args = vec!["hash-object", "--no-filters", "--"];
903 args.extend(paths.iter().map(String::as_str));
904 let actual = git::git(worktree, &args).await?;
905 ensure!(
906 actual.lines().eq(expected.iter().map(String::as_str)),
907 "tracked bytes differ from original base"
908 );
909 }
910 Ok(())
911}
912
913async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
914 let id = &state.id;
915 let paths = worktrees(state).await?;
916 let mut guards = Vec::new();
917 let mut cargo_guards = Vec::new();
918 for path in &paths {
919 guards.push(OutputLease::take(path, true)?);
920 ensure!(
921 git::rev_parse(path, "HEAD").await? == state.base_commit,
922 "unique or unrecorded commits"
923 );
924 ensure!(
925 git::git(
926 path,
927 &[
928 "--no-optional-locks",
929 "status",
930 "--porcelain",
931 "--untracked-files=all"
932 ]
933 )
934 .await?
935 .is_empty(),
936 "tracked or untracked source changes"
937 );
938 ensure!(
939 git::git(
940 path,
941 &[
942 "--no-optional-locks",
943 "status",
944 "--porcelain",
945 "--untracked-files=all",
946 "--ignored",
947 "--",
948 ".",
949 ":(exclude)target"
950 ]
951 )
952 .await?
953 .is_empty(),
954 "source changes, ignored files or build outputs"
955 );
956 tracked_base(path, &state.base_commit).await?;
957 match std::fs::symlink_metadata(path.join("target")) {
958 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
959 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
960 Err(error) => return Err(error.into()),
961 }
962 for checkpoint in [
963 "rebase-merge",
964 "rebase-apply",
965 "sequencer",
966 "BISECT_LOG",
967 "REVERT_HEAD",
968 "AUTO_MERGE",
969 "MERGE_MSG",
970 "CHERRY_PICK_HEAD",
971 "MERGE_HEAD",
972 "REBASE_HEAD",
973 ] {
974 let location = git::git(
975 path,
976 &[
977 "rev-parse",
978 "--path-format=absolute",
979 "--git-path",
980 checkpoint,
981 ],
982 )
983 .await?;
984 ensure!(
985 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
986 "retained Git recovery metadata: {checkpoint}"
987 );
988 }
989 let origin_path = git::git(
992 path,
993 &[
994 "rev-parse",
995 "--path-format=absolute",
996 "--git-path",
997 "ORIG_HEAD",
998 ],
999 )
1000 .await?;
1001 match std::fs::symlink_metadata(origin_path.trim()) {
1002 Ok(meta) => {
1003 ensure!(
1004 meta.is_file() && !meta.file_type().is_symlink(),
1005 "unknown ORIG_HEAD artifact"
1006 );
1007 git::rev_parse(path, "ORIG_HEAD")
1008 .await
1009 .context("unreadable ORIG_HEAD")?;
1010 }
1011 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1012 Err(e) => return Err(e.into()),
1013 }
1014 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1015 ensure!(
1016 git::git_raw(
1017 path,
1018 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1019 )
1020 .await?
1021 .ok(),
1022 "ORIG_HEAD retains commits outside the original base"
1023 );
1024 }
1025 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1026 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1027 ensure!(
1028 git::git_raw(
1029 path,
1030 &["merge-base", "--is-ancestor", head, &state.base_commit]
1031 )
1032 .await?
1033 .ok(),
1034 "worktree reflog retains commits outside the original base"
1035 );
1036 }
1037 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1038 ensure!(
1039 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1040 .await?
1041 .ok(),
1042 "unfinished Git operation"
1043 );
1044 }
1045 }
1046 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1049 return Ok(EmptyProof {
1050 paths,
1051 _outputs: guards,
1052 _cargo: cargo_guards,
1053 });
1054 }
1055 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1056 for candidate in &state.candidates {
1057 ensure!(
1058 (paths.contains(&candidate.worktree)
1059 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1060 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1061 "candidate recovery source"
1062 );
1063 }
1064 let pattern = format!("refs/heads/magi/{}/", state.short());
1065 let branches = git::git(
1066 &state.repo,
1067 &["for-each-ref", "--format=%(objectname)", &pattern],
1068 )
1069 .await?;
1070 ensure!(
1071 branches.lines().all(|head| head == state.base_commit),
1072 "unrecorded candidate branch changes"
1073 );
1074 Ok(EmptyProof {
1075 paths,
1076 _outputs: guards,
1077 _cargo: cargo_guards,
1078 })
1079}
1080
1081fn commit_retirement(
1086 home: &Path,
1087 queue: &crate::queue::Queue,
1088 expected: &crate::queue::Task,
1089 state: &mut RunState,
1090 authority: &str,
1091 sources: &[String],
1092 now: Timestamp,
1093) -> Result<()> {
1094 let _write = queue.lock_task(&expected.id)?;
1095 let current = queue.get(&expected.id)?;
1096 ensure!(
1097 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1098 "task changed during retirement proof; retaining worktrees for the current owner decision"
1099 );
1100 ensure!(
1101 !current
1102 .resume_override
1103 .as_ref()
1104 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1105 && !current
1106 .replacement
1107 .as_ref()
1108 .is_some_and(|pin| pin.run == state.id),
1109 "current owner selected this recovery source"
1110 );
1111 ensure!(
1112 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1113 "explicitly retained recovery source"
1114 );
1115 ensure!(
1116 !crate::ask::Questions::at(home.join("questions"))
1117 .list()
1118 .iter()
1119 .any(|q| q.status.open()
1120 && (q.run == state.id
1121 || q.run == current.id
1122 || current.blocked_by.contains(&q.id))),
1123 "owner decision arrived during retirement proof"
1124 );
1125 let dir = home.join("recovery-lineages");
1126 std::fs::create_dir_all(&dir)?;
1127 let tmp = dir.join(format!("{}.tmp", current.id));
1128 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1129 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1130 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1131 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1132 state.retired_to = Some(authority.to_owned());
1133 state.event(
1134 "retention",
1135 format!(
1136 "empty attempt retired to {authority}; recovery sources: {}",
1137 sources.join(", ")
1138 ),
1139 );
1140 state.save_under(home)?;
1141 Ok(())
1143}
1144
1145pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1149 if min_free == 0 {
1150 return std::future::pending().await;
1151 }
1152 loop {
1153 for path in &paths {
1154 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1155 match crate::disk::free_bytes(existing) {
1156 Ok(free) if free >= min_free / 2 => {}
1157 Ok(free) => {
1158 return format!(
1159 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1160 );
1161 }
1162 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1163 }
1164 }
1165 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1166 }
1167}
1168
1169#[cfg(all(test, unix))]
1170mod tests {
1171 use super::*;
1172 use crate::{
1173 config::{Config, Disk},
1174 queue::{Source, Task},
1175 run::{Candidate, RunStatus},
1176 };
1177
1178 struct Fx {
1179 _temp: tempfile::TempDir,
1180 home: PathBuf,
1181 repo: PathBuf,
1182 bay: PathBuf,
1183 base: String,
1184 }
1185 impl Fx {
1186 async fn new() -> Self {
1187 let temp = tempfile::tempdir().unwrap();
1188 let root = temp.path().canonicalize().unwrap();
1189 let home = root.join("home");
1190 let repo = root.join("repo");
1191 let bay = root.join("worktrees");
1192 std::fs::create_dir_all(&repo).unwrap();
1193 std::fs::create_dir_all(home.join("runs")).unwrap();
1194 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1195 git::git(&repo, &["config", "user.name", "Fixture"])
1196 .await
1197 .unwrap();
1198 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1199 .await
1200 .unwrap();
1201 git::git(&repo, &["config", "commit.gpgsign", "false"])
1202 .await
1203 .unwrap();
1204 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1205 std::fs::write(repo.join("source"), "base\n").unwrap();
1206 git::git(&repo, &["add", "."]).await.unwrap();
1207 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1208 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1209 Self {
1210 _temp: temp,
1211 home,
1212 repo,
1213 bay,
1214 base,
1215 }
1216 }
1217 async fn run(&self, tag: &str, useful: bool) -> RunState {
1218 let mut cfg = Config::default();
1219 cfg.graph.worktree_root = Some(self.bay.clone());
1220 let mut state = RunState::new(
1221 self.repo.clone(),
1222 "main".into(),
1223 self.base.clone(),
1224 "task".into(),
1225 cfg,
1226 );
1227 state.id = format!("20260101-000000-{tag}");
1228 state.status = RunStatus::Blocked;
1229 state.driver_exited = true;
1230 let wt = state.worktree_root().join("cand-A");
1231 let branch = format!("magi/{tag}/A");
1232 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1233 .await
1234 .unwrap();
1235 if useful {
1236 std::fs::write(wt.join("source"), tag).unwrap();
1237 git::git(&wt, &["add", "."]).await.unwrap();
1238 git::git(&wt, &["commit", "-m", "candidate change"])
1239 .await
1240 .unwrap();
1241 }
1242 state.candidates.push(Candidate {
1243 index: 0,
1244 label: 'A',
1245 agent: "fixture".into(),
1246 branch,
1247 worktree: wt,
1248 summary: String::new(),
1249 stat: String::new(),
1250 files: usize::from(useful),
1251 commits: usize::from(useful),
1252 empty: !useful,
1253 failed: None,
1254 verified_noop: None,
1255 duration_ms: 0,
1256 folded: false,
1257 });
1258 state.save_under(&self.home).unwrap();
1259 state
1260 }
1261 fn disk() -> Disk {
1262 Disk {
1263 worktree_limit_bytes: 1,
1264 worktree_reclaim_bytes: 1024 * 1024,
1265 fold_grace_secs: 0,
1266 min_free_bytes: 0,
1267 ..Disk::default()
1268 }
1269 }
1270 fn target(wt: &Path) {
1271 let target = wt.join("target");
1272 std::fs::create_dir_all(target.join("debug")).unwrap();
1273 let mut artifact = [1; 200];
1274 artifact[..4].copy_from_slice(b"\x7fELF");
1275 std::fs::write(target.join("debug/output"), artifact).unwrap();
1276 let output = target
1277 .join("debug/output")
1278 .display()
1279 .to_string()
1280 .replace('\\', "\\\\")
1281 .replace(' ', "\\ ");
1282 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1283 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1284 std::fs::write(
1285 target.join("CACHEDIR.TAG"),
1286 "Signature: 8a477f597d28d172789f06886806bc55d\n",
1287 )
1288 .unwrap();
1289 std::fs::write(
1290 target.join(".rustc_info.json"),
1291 "{\"rustc_fingerprint\":123}",
1292 )
1293 .unwrap();
1294 }
1295 fn now() -> Timestamp {
1296 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1297 }
1298 fn task(&self, runs: &[RunState]) -> Task {
1299 let mut task = Task::new(
1300 "fixture".into(),
1301 "task".into(),
1302 self.repo.clone(),
1303 Source::Human,
1304 );
1305 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1306 crate::queue::Queue::at(self.home.join("queue"))
1307 .put(&mut task)
1308 .unwrap();
1309 task
1310 }
1311 }
1312
1313 #[tokio::test]
1314 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
1315 for remove_root in [true, false] {
1316 let f = Fx::new().await;
1317 let stale = Fx::new().await;
1318 let mut missing = stale.run("aa00", false).await;
1319 Fx::target(&missing.candidates[0].worktree);
1320 missing.save_under(&f.home).unwrap();
1321 if remove_root {
1322 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
1323 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
1326 } else {
1327 std::fs::remove_dir_all(&stale.repo).unwrap();
1328 }
1329 let healthy = f.run("aa01", false).await;
1330 Fx::target(&healthy.candidates[0].worktree);
1331
1332 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1333 assert!(report.failures.is_empty(), "{report:?}");
1334 assert_eq!(report.freed_bytes, 200, "{report:?}");
1335 assert!(
1336 !healthy.candidates[0]
1337 .worktree
1338 .join("target/debug/output")
1339 .exists()
1340 );
1341 if !remove_root {
1342 assert!(
1344 missing.candidates[0]
1345 .worktree
1346 .join("target/debug/output")
1347 .exists()
1348 );
1349 }
1350 }
1351 }
1352
1353 #[tokio::test]
1354 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
1355 let f = Fx::new().await;
1356 let mut missing = f.run("aa00", false).await;
1357 Fx::target(&missing.candidates[0].worktree);
1358 missing.repo = missing.candidates[0].worktree.clone();
1359 missing.save_under(&f.home).unwrap();
1360 std::fs::remove_dir_all(&missing.repo).unwrap();
1361 assert!(missing.worktree_root().is_dir());
1362 assert!(f.repo.is_dir());
1363
1364 let healthy = f.run("aa01", false).await;
1365 Fx::target(&healthy.candidates[0].worktree);
1366
1367 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1368 assert!(report.failures.is_empty(), "{report:?}");
1369 assert_eq!(report.freed_bytes, 200, "{report:?}");
1370 assert!(
1371 !healthy.candidates[0]
1372 .worktree
1373 .join("target/debug/output")
1374 .exists()
1375 );
1376 }
1377
1378 #[tokio::test]
1379 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
1380 let f = Fx::new().await;
1381 let broken = Fx::new().await;
1382 let mut invalid = broken.run("aa00", false).await;
1383 Fx::target(&invalid.candidates[0].worktree);
1384 invalid.save_under(&f.home).unwrap();
1385 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
1386 let healthy = f.run("aa01", false).await;
1387 Fx::target(&healthy.candidates[0].worktree);
1388
1389 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1390 assert_eq!(report.failures.len(), 1, "{report:?}");
1391 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
1392 assert_eq!(report.freed_bytes, 200, "{report:?}");
1393 assert!(
1394 !healthy.candidates[0]
1395 .worktree
1396 .join("target/debug/output")
1397 .exists()
1398 );
1399 assert!(
1400 invalid.candidates[0]
1401 .worktree
1402 .join("target/debug/output")
1403 .exists()
1404 );
1405 }
1406
1407 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
1408 let path = reconcile::run(&f.home, &task.id, &source.id, None)
1409 .await
1410 .unwrap();
1411 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
1412 }
1413
1414 #[tokio::test]
1415 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
1416 use crate::{
1417 chat_control::{Operation, dispatch},
1418 talk::{self, Talks},
1419 };
1420 let f = Fx::new().await;
1421 let source = f.run("ae00", true).await;
1422 let clean = f.run("ae01", false).await;
1423 let queue = crate::queue::Queue::at(f.home.join("queue"));
1424 let talks = Talks::at(f.home.join("talks"));
1425 let open_talk = |id: &str| -> talk::Talk {
1426 let mut conversation = serde_json::from_value(serde_json::json!({
1427 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
1428 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
1429 "seat":crate::agent::SeatState::new("talk","worker",1)
1430 }))
1431 .unwrap();
1432 talks.put(&mut conversation).unwrap();
1433 conversation
1434 };
1435 let conversation = open_talk("talk-owner");
1436 let other = open_talk("talk-other");
1437 let task = f.task(&[source.clone(), clean.clone()]);
1439 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1440 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
1441 task: task.id.clone(),
1442 authority: source.id.clone(),
1443 apply: apply.map(str::to_owned),
1444 };
1445 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
1447 .await
1448 .unwrap();
1449 let legacy: serde_json::Value =
1450 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
1451 let legacy_id = legacy["id"].as_str().unwrap();
1452 assert!(
1453 dispatch(
1454 &queue,
1455 &f.home,
1456 &talks,
1457 &conversation.id,
1458 operation(Some(legacy_id))
1459 )
1460 .await
1461 .is_err()
1462 );
1463 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
1464 .await
1465 .unwrap();
1466 assert!(summary.contains("Complete review summary"));
1467 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
1468 let root = f.home.join("retention-reconciliations");
1469 let displayed: serde_json::Value =
1470 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
1471 let id = displayed["plan"].as_str().unwrap();
1472 let dir = root.join(id);
1473 let plan: serde_json::Value =
1474 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
1475 assert_eq!(plan["version"], 2);
1476 assert!(clean.candidates[0].worktree.exists());
1477 assert!(!dir.join("approved.json").exists());
1478 assert!(
1480 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
1481 .await
1482 .is_err()
1483 );
1484 assert!(!dir.join("approved.json").exists());
1485 let error = reconcile::interrupt_chat_after_fence(
1488 &f.home,
1489 &task,
1490 id,
1491 reconcile::ChatRequest {
1492 talks: &talks,
1493 conversation: &conversation.id,
1494 },
1495 )
1496 .await
1497 .unwrap_err();
1498 assert!(
1499 error.to_string().contains("simulated interruption"),
1500 "{error:#}"
1501 );
1502 assert!(clean.candidates[0].worktree.exists());
1503 assert!(dir.join("approved.json").exists());
1504 dispatch(
1505 &queue,
1506 &f.home,
1507 &talks,
1508 &conversation.id,
1509 operation(Some(id)),
1510 )
1511 .await
1512 .unwrap();
1513 assert!(!clean.candidates[0].worktree.exists());
1514 assert!(source.candidates[0].worktree.exists());
1515 let audit: serde_json::Value =
1516 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
1517 assert_eq!(audit["version"], 2);
1518 assert_eq!(
1519 audit["owner"],
1520 serde_json::json!({"conversation": conversation.id})
1521 );
1522 dispatch(
1523 &queue,
1524 &f.home,
1525 &talks,
1526 &conversation.id,
1527 operation(Some(id)),
1528 )
1529 .await
1530 .unwrap();
1531 }
1532
1533 #[tokio::test]
1534 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
1535 let f = Fx::new().await;
1536 let source = f.run("af00", true).await;
1537 let mut clean = f.run("af01", false).await;
1538 let mut dirty = f.run("af02", false).await;
1539 for state in [&mut clean, &mut dirty] {
1540 state.status = RunStatus::Prep;
1541 state.prep_complete = Some(false);
1542 state.candidates[0].empty = false;
1543 state.save_under(&f.home).unwrap();
1544 }
1545 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
1546 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
1547 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1548 let plan = reconciliation_plan(&f, &task, &source).await;
1549 let selected: Vec<_> = plan["candidates"]
1550 .as_array()
1551 .unwrap()
1552 .iter()
1553 .filter(|e| e["selected"] == true)
1554 .collect();
1555 assert_eq!(selected.len(), 1, "{plan:#}");
1556 assert_eq!(selected[0]["run"], clean.id);
1557 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1558 .await
1559 .unwrap();
1560 assert!(!clean.candidates[0].worktree.exists());
1561 assert!(dirty.candidates[0].worktree.exists());
1562 assert!(
1563 RunState::load_under(&dirty.id, &f.home)
1564 .unwrap()
1565 .retired_to
1566 .is_none()
1567 );
1568 }
1569
1570 #[tokio::test]
1571 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
1572 let f = Fx::new().await;
1573 let mut source = f.run("aa00", true).await;
1574 let alternative = f.run("aa01", true).await;
1575 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
1576 .await
1577 .unwrap();
1578 source.reviewed_commits = Some(vec![head.clone()]);
1579 source.save_under(&f.home).unwrap();
1580 let mut runs = vec![source.clone(), alternative.clone()];
1581 for i in 2..14 {
1582 runs.push(f.run(&format!("aa{i:02}"), false).await);
1583 }
1584 let mut task = f.task(&runs);
1585 task.pr_recovery = Some(crate::pr_recovery::Selection {
1586 url: "https://example.invalid/pull/1".into(),
1587 branch: "selected".into(),
1588 head: head.clone(),
1589 source_head: Some(head.clone()),
1590 source_run: Some(source.id.clone()),
1591 source_proof: None,
1592 disposition: Some(crate::pr_recovery::Disposition::Land),
1593 instruction: task.instruction.clone(),
1594 approved_at: Timestamp::now(),
1595 });
1596 source.pr_recovery = task.pr_recovery.clone();
1597 source.save_under(&f.home).unwrap();
1598 runs[0] = source.clone();
1599 crate::queue::Queue::at(f.home.join("queue"))
1600 .put(&mut task)
1601 .unwrap();
1602 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1603 let task_before = serde_json::to_value(&task).unwrap();
1604 let mut report = Report::default();
1605 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
1606 assert_eq!(
1607 report.retired, 0,
1608 "automatic cleanup must still refuse blanket markers"
1609 );
1610 assert!(
1611 report
1612 .retained
1613 .iter()
1614 .any(|reason| reason.contains("explicitly retained"))
1615 );
1616 let mut total = 0;
1617 for expected in [5, 5, 2, 0] {
1618 let plan = reconciliation_plan(&f, &task, &source).await;
1619 let selected: Vec<_> = plan["candidates"]
1620 .as_array()
1621 .unwrap()
1622 .iter()
1623 .filter(|e| e["selected"] == true)
1624 .collect();
1625 assert_eq!(selected.len(), expected, "{plan:#}");
1626 for e in &selected {
1627 assert!(
1628 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
1629 .unwrap()
1630 .retired_to
1631 .is_none()
1632 );
1633 }
1634 let operation = plan["id"].as_str().unwrap();
1635 for _ in 0..2 {
1636 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
1637 .await
1638 .unwrap();
1639 }
1640 total += expected;
1641 }
1642 assert_eq!(total, 12);
1643 for run in &runs {
1644 let dir = f.home.join("runs").join(&run.id);
1645 assert!(dir.join("run.json").exists());
1646 assert!(dir.join("retain-recovery-sources").exists());
1647 assert!(
1648 git::branch_exists(&f.repo, &run.candidates[0].branch)
1649 .await
1650 .unwrap()
1651 );
1652 }
1653 assert!(source.candidates[0].worktree.exists());
1654 assert!(alternative.candidates[0].worktree.exists());
1655 assert_eq!(
1656 git::rev_parse(&f.repo, &source.candidates[0].branch)
1657 .await
1658 .unwrap(),
1659 head
1660 );
1661 assert_eq!(
1662 serde_json::to_value(
1663 crate::queue::Queue::at(f.home.join("queue"))
1664 .get(&task.id)
1665 .unwrap()
1666 )
1667 .unwrap(),
1668 task_before
1669 );
1670 assert_eq!(
1671 runs.iter()
1672 .filter(|r| RunState::load_under(&r.id, &f.home)
1673 .unwrap()
1674 .retired_to
1675 .is_none())
1676 .count(),
1677 2
1678 );
1679 }
1680
1681 #[tokio::test]
1682 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
1683 let f = Fx::new().await;
1684 let source = f.run("ab00", true).await;
1685 let mut runs = vec![source.clone()];
1686 for i in 1..9 {
1687 runs.push(f.run(&format!("ab{i:02}"), false).await);
1688 }
1689 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
1690 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
1691 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
1692 std::fs::write(
1693 runs[3].candidates[0].worktree.join("scratch/source"),
1694 "ignored source",
1695 )
1696 .unwrap();
1697 std::fs::write(
1698 runs[4].worktree_root().join("unknown"),
1699 "unregistered source",
1700 )
1701 .unwrap();
1702 let checkpoint = git::git(
1703 &runs[5].candidates[0].worktree,
1704 &[
1705 "rev-parse",
1706 "--path-format=absolute",
1707 "--git-path",
1708 "sequencer",
1709 ],
1710 )
1711 .await
1712 .unwrap();
1713 std::fs::create_dir(checkpoint.trim()).unwrap();
1714 runs[6].driver_exited = false;
1715 runs[6].driver_pid = Some(std::process::id());
1716 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
1717 runs[6].save_under(&f.home).unwrap();
1718 runs[7].driver_exited = false;
1719 runs[7].driver_pid = None;
1720 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
1722 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
1723
1724 let hidden = &runs[8].candidates[0].worktree;
1725 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
1726 .await
1727 .unwrap();
1728 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
1729 assert!(
1730 git::git(hidden, &["status", "--porcelain"])
1731 .await
1732 .unwrap()
1733 .is_empty()
1734 );
1735 let task = f.task(&runs);
1736 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1737 let plan = reconciliation_plan(&f, &task, &source).await;
1738 assert!(
1739 plan["candidates"]
1740 .as_array()
1741 .unwrap()
1742 .iter()
1743 .all(|e| e["selected"] == false),
1744 "{plan:#}"
1745 );
1746 for run in &runs {
1747 assert!(run.candidates[0].worktree.exists());
1748 }
1749 }
1750
1751 #[tokio::test]
1752 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
1753 let f = Fx::new().await;
1754 let source = f.run("ac00", true).await;
1755 let empty = f.run("ac01", false).await;
1756 let task = f.task(&[source.clone(), empty.clone()]);
1757 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1758 let plan = reconciliation_plan(&f, &task, &source).await;
1759 std::fs::write(
1760 f.home.join("runs").join(&source.id).join("new-report"),
1761 "new review checkpoint",
1762 )
1763 .unwrap();
1764 assert!(
1765 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1766 .await
1767 .is_err()
1768 );
1769 assert!(empty.candidates[0].worktree.exists());
1770 assert!(
1771 RunState::load_under(&empty.id, &f.home)
1772 .unwrap()
1773 .retired_to
1774 .is_none()
1775 );
1776 let marker = f
1777 .home
1778 .join("runs")
1779 .join(&empty.id)
1780 .join("retain-recovery-sources");
1781 std::fs::write(&marker, "different owner").unwrap();
1782 let plan = reconciliation_plan(&f, &task, &source).await;
1783 assert!(
1784 plan["candidates"]
1785 .as_array()
1786 .unwrap()
1787 .iter()
1788 .all(|e| e["selected"] == false)
1789 );
1790 }
1791
1792 #[tokio::test]
1793 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
1794 for partial in [false, true] {
1795 let f = Fx::new().await;
1796 let source = f.run("ab10", true).await;
1797 let first = f.run("ab11", false).await;
1798 let second = f.run("ab12", false).await;
1799 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
1800 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1801 let plan = reconciliation_plan(&f, &task, &source).await;
1802 let id = plan["id"].as_str().unwrap();
1803 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
1804 std::fs::write(&staging, b"").unwrap();
1806 assert!(
1807 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1808 .await
1809 .is_err()
1810 );
1811 assert!(staging.exists());
1812 std::fs::remove_file(&staging).unwrap();
1813 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
1814 .await
1815 .unwrap_err();
1816 assert!(
1817 error.to_string().contains("before run state rename"),
1818 "{error:#}"
1819 );
1820 assert!(
1821 RunState::load_under(&first.id, &f.home)
1822 .unwrap()
1823 .retired_to
1824 .is_some()
1825 );
1826 assert!(
1827 RunState::load_under(&second.id, &f.home)
1828 .unwrap()
1829 .retired_to
1830 .is_none()
1831 );
1832 assert!(first.candidates[0].worktree.exists());
1833 assert!(second.candidates[0].worktree.exists());
1834 let interrupted = std::fs::read(&staging).unwrap();
1835 std::fs::write(&staging, b"unrelated recovery source").unwrap();
1837 assert!(
1838 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1839 .await
1840 .is_err()
1841 );
1842 assert_eq!(
1843 std::fs::read(&staging).unwrap(),
1844 b"unrelated recovery source"
1845 );
1846 std::fs::write(&staging, interrupted).unwrap();
1847 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1848 .await
1849 .unwrap();
1850 assert!(!staging.exists());
1851 assert!(!first.candidates[0].worktree.exists());
1852 assert!(!second.candidates[0].worktree.exists());
1853 assert_eq!(
1854 RunState::load_under(&second.id, &f.home)
1855 .unwrap()
1856 .retired_to,
1857 Some(source.id)
1858 );
1859 }
1860 }
1861
1862 #[tokio::test]
1863 async fn reconciliation_resumes_only_its_exact_durable_fence() {
1864 let f = Fx::new().await;
1865 let source = f.run("ad00", true).await;
1866 let empty = f.run("ad01", false).await;
1867 let task = f.task(&[source.clone(), empty.clone()]);
1868 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1869 let plan = reconciliation_plan(&f, &task, &source).await;
1870 let id = plan["id"].as_str().unwrap();
1871 assert!(
1872 reconcile::interrupt_after_fence(&f.home, &task, id)
1873 .await
1874 .is_err()
1875 );
1876 assert!(empty.candidates[0].worktree.exists());
1877 assert_eq!(
1878 RunState::load_under(&empty.id, &f.home)
1879 .unwrap()
1880 .retired_to
1881 .as_ref(),
1882 Some(&source.id)
1883 );
1884 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1885 .await
1886 .unwrap();
1887 assert!(!empty.candidates[0].worktree.exists());
1888 git::worktree_add_branch(
1890 &f.repo,
1891 &empty.candidates[0].worktree,
1892 "restore-empty",
1893 &f.base,
1894 )
1895 .await
1896 .unwrap();
1897 assert!(
1899 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1900 .await
1901 .is_err()
1902 );
1903 assert!(empty.candidates[0].worktree.exists());
1904 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
1905 changed.prep_failure = Some("new owner edit".into());
1906 changed.save_under(&f.home).unwrap();
1907 assert!(
1908 reconcile::run(&f.home, &task.id, &source.id, Some(id))
1909 .await
1910 .is_err()
1911 );
1912 }
1913
1914 #[tokio::test]
1915 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
1916 let f = Fx::new().await;
1917 let source = f.run("ae00", true).await;
1918 let mut prep = f.run("ae01", false).await;
1919 let candidate = prep.candidates.pop().unwrap();
1920 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
1921 prep.save_under(&f.home).unwrap();
1922 let reset = f.run("ae02", false).await;
1923 let wt = &reset.candidates[0].worktree;
1924 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
1925 git::git(wt, &["add", "."]).await.unwrap();
1926 git::git(wt, &["commit", "-m", "unique implementation"])
1927 .await
1928 .unwrap();
1929 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
1930 let unknown = f.run("ae03", false).await;
1931 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
1932 let mut raw: serde_json::Value =
1933 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
1934 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
1935 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
1936 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
1937 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1938 let plan = reconciliation_plan(&f, &task, &source).await;
1939 let selected: Vec<_> = plan["candidates"]
1940 .as_array()
1941 .unwrap()
1942 .iter()
1943 .filter(|e| e["selected"] == true)
1944 .collect();
1945 assert_eq!(selected.len(), 1, "{plan:#}");
1946 assert_eq!(selected[0]["run"], prep.id);
1947 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1948 .await
1949 .unwrap();
1950 assert!(!candidate.worktree.exists());
1951 assert!(reset.candidates[0].worktree.exists());
1952 assert!(unknown.candidates[0].worktree.exists());
1953 }
1954
1955 #[tokio::test]
1956 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
1957 let f = Fx::new().await;
1958 let mut state = f.run("aaaa", true).await;
1959 let candidate = state.candidates[0].worktree.clone();
1960 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
1961 Fx::target(&candidate);
1962 for seat in ["review-1", "review-2"] {
1963 let wt = state.worktree_root().join(seat);
1964 git::worktree_add_detached(&f.repo, &wt, &f.base)
1965 .await
1966 .unwrap();
1967 Fx::target(&wt);
1968 }
1969 state.status = RunStatus::Landing;
1970 state.save_under(&f.home).unwrap();
1971 let second = Fx::new().await;
1972 let other = second.run("bbbb", true).await;
1973 let mut imported = other.clone();
1975 imported.save_under(&f.home).unwrap();
1976 Fx::target(&other.candidates[0].worktree);
1977 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
1978 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1979 assert!(
1980 candidate.join("target/debug/output").exists(),
1981 "active writer protected"
1982 );
1983 assert!(
1984 !state
1985 .worktree_root()
1986 .join("review-1/target/debug/output")
1987 .exists()
1988 );
1989 assert!(
1990 !state
1991 .worktree_root()
1992 .join("review-2/target/debug/output")
1993 .exists()
1994 );
1995 assert!(
1996 !other.candidates[0]
1997 .worktree
1998 .join("target/debug/output")
1999 .exists()
2000 );
2001 assert!(out.freed_bytes >= 600);
2002 drop(guard);
2003 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2004 assert!(out.freed_bytes >= 200);
2005 assert_eq!(
2006 std::fs::read_to_string(candidate.join("source")).unwrap(),
2007 "uncommitted work"
2008 );
2009 assert!(
2010 git::branch_exists(&f.repo, &state.candidates[0].branch)
2011 .await
2012 .unwrap()
2013 );
2014 }
2015
2016 #[tokio::test]
2017 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
2018 let f = Fx::new().await;
2019 let state = f.run("cc01", false).await;
2020 let wt = &state.candidates[0].worktree;
2021 Fx::target(wt);
2022 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2023 let dir = wt.join("target").join(dir);
2024 std::fs::create_dir_all(&dir).unwrap();
2025 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
2026 }
2027 let manifest = wt.join("target/debug/output.d");
2029 let output = wt.join("target/debug/output");
2030 std::fs::write(
2031 &manifest,
2032 format!(
2033 "{}: {}\n",
2034 output.display(),
2035 wt.join("target/debug/notes.txt").display()
2036 ),
2037 )
2038 .unwrap();
2039 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2040 assert_eq!(report.freed_bytes, 200);
2041 assert!(!output.exists());
2042 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2043 assert_eq!(
2044 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
2045 "user evidence"
2046 );
2047 }
2048 assert!(manifest.exists());
2050 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
2051 std::fs::write(&output, "user note replacing an old binary").unwrap();
2052 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2053 assert_eq!(report.freed_bytes, 0);
2054 assert_eq!(
2055 std::fs::read_to_string(&output).unwrap(),
2056 "user note replacing an old binary"
2057 );
2058 }
2059
2060 #[tokio::test]
2061 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
2062 let f = Fx::new().await;
2063 let state = f.run("cc02", false).await;
2064 let wt = &state.candidates[0].worktree;
2065 Fx::target(wt);
2066 let mut disk = Fx::disk();
2067 disk.worktree_limit_bytes = u64::MAX;
2068 let now = Fx::now();
2069 let first = sweep(&f.home, &disk, now).await;
2070 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
2071 assert_eq!(
2072 sweep(&f.home, &disk, now).await.measured_bytes,
2073 first.measured_bytes
2074 );
2075 assert!(
2076 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
2077 .await
2078 .measured_bytes
2079 > first.measured_bytes
2080 );
2081 disk.worktree_limit_bytes = 0;
2082 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
2083 }
2084
2085 #[tokio::test]
2086 async fn unfinished_source_fold_reports_its_retention_reason() {
2087 let f = Fx::new().await;
2088 let mut source = f.run("cc03", true).await;
2089 let task = f.task(&[source.clone()]);
2090 let error = crate::graph::fold_run(&mut source, true, &f.home)
2091 .await
2092 .unwrap_err();
2093 assert!(error.to_string().contains(&task.id));
2094 assert!(source.candidates[0].worktree.exists());
2095 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2096 let error = crate::graph::fold_run(&mut source, true, &f.home)
2097 .await
2098 .unwrap_err();
2099 assert!(error.to_string().contains("explicitly retained"));
2100 }
2101
2102 #[tokio::test]
2103 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
2104 let f = Fx::new().await;
2105 let mut source = f.run("cc04", true).await;
2106 let mut task = f.task(&[source.clone()]);
2107 task.hold_manual(Some("queue settlement pending".into()));
2108 let queue = crate::queue::Queue::at(f.home.join("queue"));
2109 queue.put(&mut task).unwrap();
2110 source.merge = Some(crate::run::MergeOutcome {
2111 mode: crate::config::MergeMode::Pr,
2112 ok: true,
2113 detail: "confirmed merged".into(),
2114 empty: false,
2115 });
2116 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
2118 source.status = status;
2119 assert!(
2120 crate::graph::fold_run(&mut source, true, &f.home)
2121 .await
2122 .is_err()
2123 );
2124 assert!(source.candidates[0].worktree.exists());
2125 }
2126 git::git(
2127 &f.repo,
2128 &["merge", "--ff-only", &source.candidates[0].branch],
2129 )
2130 .await
2131 .unwrap();
2132 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2133 .await
2134 .unwrap();
2135 source.status = RunStatus::Merged;
2136 source.save_under(&f.home).unwrap();
2137 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
2138 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2139 assert!(
2140 crate::graph::fold_run(&mut source, true, &f.home)
2141 .await
2142 .unwrap_err()
2143 .to_string()
2144 .contains("explicitly retained")
2145 );
2146 std::fs::remove_file(
2149 f.home
2150 .join("runs")
2151 .join(&source.id)
2152 .join("retain-recovery-sources"),
2153 )
2154 .unwrap();
2155 let (folded, unreadable) = crate::clean::fold_due(
2156 &f.home.join("runs"),
2157 &f.home,
2158 &f.bay,
2159 &Fx::disk(),
2160 Fx::now(),
2161 )
2162 .await
2163 .unwrap();
2164 assert_eq!((folded, unreadable), (1, 0));
2165 assert!(!source.candidates[0].worktree.exists());
2166 let evidence = f
2167 .home
2168 .join("runs")
2169 .join(&source.id)
2170 .join("artifacts/fold-evidence");
2171 let names: Vec<_> = std::fs::read_dir(evidence)
2172 .unwrap()
2173 .map(|entry| entry.unwrap().file_name())
2174 .collect();
2175 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
2176 assert_eq!(
2177 queue.get(&task.id).unwrap().status,
2178 crate::queue::TaskStatus::Held
2179 );
2180 }
2181
2182 #[tokio::test]
2183 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
2184 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
2185 let f = Fx::new().await;
2186 let mut state = f.run("cc06", true).await;
2187 let candidate = state.candidates[0].clone();
2188 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2189 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
2190 git::rescue_commit(&candidate.worktree, "second candidate change")
2191 .await
2192 .unwrap();
2193 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2194 state.tally = Some(crate::run::Tally {
2195 first_choice: Default::default(),
2196 borda: Default::default(),
2197 winner: 'A',
2198 rankings: 0,
2199 unanimous_initial: true,
2200 deliberated: false,
2201 changed_votes: 0,
2202 unanimous_final: true,
2203 tie_break: None,
2204 judges: 0,
2205 present: 0,
2206 quorum: 0,
2207 met_quorum: true,
2208 uncontested: Some("fixture".into()),
2209 });
2210 let reviewer = state.worktree_root().join("review-1");
2213 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
2214 .await
2215 .unwrap();
2216 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
2219 git::rescue_commit(&f.repo, "advance before squash")
2220 .await
2221 .unwrap();
2222 let published = if variant == "partial" {
2223 &intermediate
2224 } else {
2225 &head
2226 };
2227 git::git(&f.repo, &["merge", "--squash", published])
2228 .await
2229 .unwrap();
2230 git::git(&f.repo, &["commit", "-m", "squash candidate"])
2231 .await
2232 .unwrap();
2233 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
2234 assert!(
2235 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
2236 .await
2237 .unwrap()
2238 .ok()
2239 );
2240 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
2241 git::rescue_commit(&f.repo, "advance after squash")
2242 .await
2243 .unwrap();
2244 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2245 .await
2246 .unwrap();
2247 if matches!(variant, "checkpoint" | "reviewer") {
2248 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
2250 git::rescue_commit(&reviewer, "unpublished checkpoint")
2251 .await
2252 .unwrap();
2253 if variant == "checkpoint" {
2254 git::git(&reviewer, &["reset", "--hard", &intermediate])
2255 .await
2256 .unwrap();
2257 }
2258 }
2259 if variant == "loser" {
2260 let mut loser = candidate.clone();
2261 loser.label = 'B';
2262 loser.branch = "magi/cc06/B".into();
2263 loser.worktree = state.worktree_root().join("cand-B");
2264 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
2265 .await
2266 .unwrap();
2267 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
2268 git::rescue_commit(&loser.worktree, "independent loser")
2269 .await
2270 .unwrap();
2271 state.candidates.push(loser);
2272 }
2273 state.status = RunStatus::Merged;
2274 state.merge = Some(crate::run::MergeOutcome {
2275 mode: crate::config::MergeMode::Pr,
2276 ok: true,
2277 detail: "candidate A squash merged".into(),
2278 empty: false,
2279 });
2280 state.save_under(&f.home).unwrap();
2281 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
2282 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
2283 result.unwrap();
2284 assert!(
2285 !candidate.worktree.exists() && !reviewer.exists(),
2286 "{variant}"
2287 );
2288 assert!(
2289 !git::branch_exists(&f.repo, &candidate.branch)
2290 .await
2291 .unwrap()
2292 );
2293 let artifacts = f
2294 .home
2295 .join("runs")
2296 .join(&state.id)
2297 .join("artifacts/fold-evidence");
2298 assert_eq!(
2299 artifacts.join(format!("{head}.patch")).exists(),
2300 variant == "partial",
2301 "only a partially published winner needs evidence: {variant}"
2302 );
2303 assert_eq!(
2304 artifacts.join(format!("{intermediate}.patch")).exists(),
2305 variant == "partial",
2306 "squash proof also covers intermediate reviewer tips: {variant}"
2307 );
2308 let patches: Vec<_> = std::fs::read_dir(&artifacts)
2309 .unwrap()
2310 .map(|entry| entry.unwrap().path())
2311 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
2312 .collect();
2313 let expected = match variant {
2314 "partial" => 2,
2315 "loser" | "reviewer" => 1,
2316 _ => 0,
2317 };
2318 assert_eq!(patches.len(), expected, "{variant}");
2319 if variant == "loser" {
2320 assert!(
2321 std::fs::read_to_string(&patches[0])
2322 .unwrap()
2323 .contains("losing candidate")
2324 );
2325 }
2326 assert!(artifacts.join("manifest.json").exists());
2327 if variant == "reviewer" {
2328 assert!(
2329 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
2330 .unwrap()
2331 .contains("reviewer checkpoint")
2332 );
2333 }
2334 crate::graph::fold_run(&mut state, true, &f.home)
2335 .await
2336 .unwrap();
2337 }
2338 }
2339
2340 #[tokio::test]
2341 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
2342 for missing in [true, false] {
2343 let f = Fx::new().await;
2344 let mut state = f.run("cc05", true).await;
2345 git::git(
2348 &f.repo,
2349 &["update-ref", "refs/remotes/origin/main", &f.base],
2350 )
2351 .await
2352 .unwrap();
2353 let candidate = state.candidates[0].clone();
2354 if !missing {
2355 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
2356 git::rescue_commit(&f.repo, "advance base before rebase")
2357 .await
2358 .unwrap();
2359 git::git(&candidate.worktree, &["rebase", "main"])
2360 .await
2361 .unwrap();
2362 git::sync_to_head(&candidate.worktree).await.unwrap();
2365 }
2366 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2367 state.reviewed_commits = Some(Vec::new());
2369 state.candidates[0].worktree = state.worktree_root().join("under-review");
2370 git::git(
2371 &f.repo,
2372 &[
2373 "worktree",
2374 "move",
2375 candidate.worktree.to_str().unwrap(),
2376 state.candidates[0].worktree.to_str().unwrap(),
2377 ],
2378 )
2379 .await
2380 .unwrap();
2381 let reviewer = state.worktree_root().join("review-1");
2382 git::worktree_add_detached(&f.repo, &reviewer, &head)
2383 .await
2384 .unwrap();
2385 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
2388 git::rescue_commit(&reviewer, "old review history")
2389 .await
2390 .unwrap();
2391 git::git(&reviewer, &["reset", "--hard", &head])
2392 .await
2393 .unwrap();
2394 if missing {
2395 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
2396 }
2397 let run_dir = f.home.join("runs").join(&state.id);
2398 let artifacts = run_dir.join("artifacts");
2399 std::fs::create_dir_all(&artifacts).unwrap();
2400 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
2401 if missing {
2402 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
2403 }
2404 state.status = RunStatus::Merged;
2405 state.merge = Some(crate::run::MergeOutcome {
2406 mode: crate::config::MergeMode::Pr,
2407 ok: true,
2408 detail: "confirmed squash merge".into(),
2409 empty: false,
2410 });
2411 state.save_under(&f.home).unwrap();
2412 let owned = state.candidates[0].worktree.clone();
2414 state.candidates[0].worktree = f.home.join("missing-external");
2415 assert!(
2416 crate::graph::fold_run(&mut state, true, &f.home)
2417 .await
2418 .unwrap_err()
2419 .to_string()
2420 .contains("outside the inventoried")
2421 );
2422 state.candidates[0].worktree = owned;
2423 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
2425 assert!(
2426 crate::graph::fold_run(&mut state, true, &f.home)
2427 .await
2428 .is_err()
2429 );
2430 assert!(reviewer.exists());
2431 drop(guard);
2432 let evidence = artifacts.join("fold-evidence");
2434 std::fs::write(&evidence, "blocked").unwrap();
2435 assert!(
2436 crate::graph::fold_run(&mut state, true, &f.home)
2437 .await
2438 .is_err()
2439 );
2440 assert!(reviewer.exists());
2441 std::fs::remove_file(&evidence).unwrap();
2442 std::fs::create_dir_all(&evidence).unwrap();
2443 let oversized = evidence.join("capacity-fixture");
2444 std::fs::File::create(&oversized)
2445 .unwrap()
2446 .set_len(64 * 1024 * 1024)
2447 .unwrap();
2448 assert!(
2449 crate::graph::fold_run(&mut state, true, &f.home)
2450 .await
2451 .is_err()
2452 );
2453 assert!(reviewer.exists());
2454 std::fs::remove_file(oversized).unwrap();
2455 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
2458 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
2459 assert!(
2460 crate::graph::fold_run(&mut state, true, &f.home)
2461 .await
2462 .is_err()
2463 );
2464 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
2465 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
2466 .await
2467 .unwrap();
2468 std::fs::write(&checkpoint, &head).unwrap();
2469 assert!(
2470 crate::graph::fold_run(&mut state, true, &f.home)
2471 .await
2472 .is_err()
2473 );
2474 std::fs::remove_file(checkpoint).unwrap();
2475 let (folded, unreadable) = crate::clean::fold_due(
2476 &f.home.join("runs"),
2477 &f.home,
2478 &f.bay,
2479 &Fx::disk(),
2480 Fx::now(),
2481 )
2482 .await
2483 .unwrap();
2484 if folded != 1 {
2485 panic!(
2486 "normal cleanup failed: {:#}",
2487 crate::graph::fold_run(&mut state, true, &f.home)
2488 .await
2489 .unwrap_err()
2490 );
2491 }
2492 assert_eq!((folded, unreadable), (1, 0));
2493 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
2494 state = RunState::load_under(&state.id, &f.home).unwrap();
2495 assert!(state.candidates[0].folded);
2496 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
2497 crate::graph::fold_run(&mut state, true, &f.home)
2498 .await
2499 .unwrap();
2500 assert_eq!(
2501 std::fs::read(evidence.join("manifest.json")).unwrap(),
2502 manifest
2503 );
2504 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
2505 .await
2506 .unwrap();
2507 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
2508 assert!(
2509 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
2510 .await
2511 .unwrap()
2512 .ok()
2513 );
2514 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
2517 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
2518 .as_str()
2519 .unwrap();
2520 let recovered = f.home.join("recovered");
2521 std::fs::create_dir_all(&recovered).unwrap();
2522 git::git(&recovered, &["init"]).await.unwrap();
2523 std::fs::write(
2524 recovered.join("source"),
2525 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
2526 )
2527 .unwrap();
2528 git::git(
2529 &recovered,
2530 &[
2531 "apply",
2532 evidence.join(format!("{head}.patch")).to_str().unwrap(),
2533 ],
2534 )
2535 .await
2536 .unwrap();
2537 assert_eq!(
2538 std::fs::read_to_string(recovered.join("source")).unwrap(),
2539 "cc05"
2540 );
2541 assert!(
2542 !std::fs::read(evidence.join(format!("{head}.patch")))
2543 .unwrap()
2544 .is_empty()
2545 );
2546 assert_eq!(
2547 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
2548 "review findings"
2549 );
2550 if missing {
2551 assert_eq!(
2552 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
2553 "original snapshot"
2554 );
2555 } else {
2556 assert!(!artifacts.join("cand-A.patch").exists());
2557 }
2558 assert!(run_dir.join("run.json").exists());
2559 }
2560 }
2561
2562 #[tokio::test]
2563 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
2564 use std::os::unix::fs::PermissionsExt as _;
2565 let f = Fx::new().await;
2566 let source = f.run("dd03", true).await;
2567 let empty = f.run("dd04", false).await;
2568 let task = f.task(&[source, empty.clone()]);
2569 let entered = f.repo.join("proof-entered");
2570 let release = f.repo.join("proof-release");
2571 let hook = f.repo.join("proof-monitor");
2572 std::fs::write(&hook, format!(
2573 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
2574 entered.display(), release.display())).unwrap();
2575 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
2576 git::git(
2577 &f.repo,
2578 &["config", "core.fsmonitor", hook.to_str().unwrap()],
2579 )
2580 .await
2581 .unwrap();
2582 struct ReleaseProof(PathBuf);
2583 impl Drop for ReleaseProof {
2584 fn drop(&mut self) {
2585 let _ = std::fs::write(&self.0, "continue");
2586 }
2587 }
2588 let _release_on_panic = ReleaseProof(release.clone());
2589 let home = f.home.clone();
2590 let sweep = tokio::spawn(async move {
2591 let mut report = Report::default();
2592 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
2593 report
2594 });
2595 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2596 while !entered.exists() {
2597 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2598 }
2599 })
2600 .await
2601 .unwrap();
2602 let queue = crate::queue::Queue::at(f.home.join("queue"));
2603 let mut selected = queue.get(&task.id).unwrap();
2604 selected.resume_override = Some(crate::queue::OperatorResume {
2605 question_id: "during-slow-proof".into(),
2606 at: Fx::now(),
2607 conductor_rehold: None,
2608 forced: false,
2609 pinned_run: Some(empty.id.clone()),
2610 });
2611 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
2612 let quick = match tokio::time::timeout(std::time::Duration::from_secs(2), &mut update).await
2613 {
2614 Ok(result) => {
2615 result.unwrap().unwrap();
2616 true
2617 }
2618 Err(_) => {
2619 std::fs::write(&release, "continue").unwrap();
2620 update.await.unwrap().unwrap();
2621 false
2622 }
2623 };
2624 std::fs::write(release, "continue").unwrap();
2625 let report = tokio::time::timeout(std::time::Duration::from_secs(8), sweep)
2626 .await
2627 .unwrap()
2628 .unwrap();
2629 assert!(
2630 quick,
2631 "task writers must not wait behind Git retirement proof"
2632 );
2633 assert_eq!(report.retired, 0);
2634 assert!(report.retained.iter().any(|r| r.contains("task changed")));
2635 assert!(empty.candidates[0].worktree.exists());
2636 assert!(
2637 RunState::load_under(&empty.id, &f.home)
2638 .unwrap()
2639 .retired_to
2640 .is_none()
2641 );
2642 }
2643
2644 #[tokio::test]
2645 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
2646 let f = Fx::new().await;
2647 let source = f.run("dd01", true).await;
2648 let mut empty = f.run("dd02", false).await;
2649 let expected = f.task(&[source.clone(), empty.clone()]);
2650 let queue = crate::queue::Queue::at(f.home.join("queue"));
2651 let _claim = queue.claim(&expected.id).unwrap();
2652 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
2654 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
2655 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
2656 let mut selected = queue.get(&expected.id).unwrap();
2657 selected.resume_override = Some(crate::queue::OperatorResume {
2658 question_id: "new-owner-decision".into(),
2659 at: Fx::now(),
2660 conductor_rehold: None,
2661 forced: false,
2662 pinned_run: Some(empty.id.clone()),
2663 });
2664 queue.put(&mut selected).unwrap();
2665 let error = commit_retirement(
2666 &f.home,
2667 &queue,
2668 &expected,
2669 &mut empty,
2670 &source.id,
2671 &expected.runs,
2672 Fx::now(),
2673 )
2674 .unwrap_err();
2675 assert!(error.to_string().contains("task changed"));
2676 assert!(
2677 RunState::load_under(&empty.id, &f.home)
2678 .unwrap()
2679 .retired_to
2680 .is_none()
2681 );
2682 assert!(empty.candidates[0].worktree.exists());
2683 assert_eq!(
2684 queue
2685 .get(&expected.id)
2686 .unwrap()
2687 .resume_override
2688 .unwrap()
2689 .pinned_run,
2690 Some(empty.id.clone())
2691 );
2692 assert!(
2693 !f.home
2694 .join("recovery-lineages")
2695 .join(format!("{}.json", expected.id))
2696 .exists()
2697 );
2698 assert!(
2700 commit_retirement(
2701 &f.home,
2702 &queue,
2703 &selected,
2704 &mut empty,
2705 &source.id,
2706 &selected.runs,
2707 Fx::now()
2708 )
2709 .unwrap_err()
2710 .to_string()
2711 .contains("selected this recovery source")
2712 );
2713 }
2714
2715 #[tokio::test]
2716 async fn unknown_targets_cargo_owners_and_driver_resumes_are_protected() {
2717 let f = Fx::new().await;
2718 let state = f.run("cccc", false).await;
2719 let wt = &state.candidates[0].worktree;
2720 Fx::target(wt);
2721 std::fs::write(wt.join("target/notes"), "user file").unwrap();
2722 assert!(cargo_target(wt).await.is_err());
2723 std::fs::remove_file(wt.join("target/notes")).unwrap();
2724 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
2725 assert!(
2726 !declared_outputs(&wt.join("target"))
2727 .unwrap()
2728 .contains(&wt.join("target/debug/link"))
2729 );
2730 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
2731 let cargo = cargo_locks(&wt.join("target")).unwrap();
2732 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2733 assert_eq!(out.freed_bytes, 0);
2734 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
2735 drop(cargo);
2736 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2737 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2738 assert_eq!(out.freed_bytes, 0);
2739 drop(driver);
2740 let exclusive = OutputLease::take(wt, true).unwrap();
2741 assert!(
2742 OutputLease::borrow(wt).is_err(),
2743 "resume cannot start a writer during cleanup"
2744 );
2745 drop(exclusive);
2746 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
2747 }
2748
2749 #[tokio::test]
2750 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
2751 let f = Fx::new().await;
2752 let source = f.run("0000", true).await;
2753 let empty = f.run("0001", false).await;
2754 let changed = f.run("0002", false).await;
2755 let unknown = f.run("0003", false).await;
2756 let unproven = f.run("0004", false).await;
2757 f.task(&[
2758 source.clone(),
2759 empty.clone(),
2760 changed.clone(),
2761 unknown.clone(),
2762 unproven.clone(),
2763 ]);
2764 for run in [&empty, &changed, &unknown] {
2765 Fx::target(&run.candidates[0].worktree);
2766 }
2767 let reviewer = empty.worktree_root().join("review-1");
2768 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2769 .await
2770 .unwrap();
2771 Fx::target(&reviewer);
2772 let mut disk = Fx::disk();
2773 disk.auto_fold = false;
2774 let report = sweep(&f.home, &disk, Fx::now()).await;
2775 assert_eq!(report.freed_bytes, 800, "{report:?}");
2776 let wt = &empty.candidates[0].worktree;
2777 assert!(wt.join("target/debug/output.d").exists());
2778 assert!(!wt.join("target/debug/output").exists());
2779 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
2780 let proof: CargoResidue =
2781 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
2782 assert_eq!(
2783 proof.files[Path::new("debug/output.d")],
2784 std::fs::read(wt.join("target/debug/output.d")).unwrap()
2785 );
2786 std::fs::write(
2787 changed.candidates[0].worktree.join("target/debug/output.d"),
2788 "owner notes",
2789 )
2790 .unwrap();
2791 std::fs::write(
2792 unknown.candidates[0].worktree.join("target/debug/notes"),
2793 "keep",
2794 )
2795 .unwrap();
2796 Fx::target(&unproven.candidates[0].worktree);
2797 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
2798 disk.auto_fold = true;
2799 let locks = cargo_locks(&wt.join("target")).unwrap();
2800 let mut report = Report::default();
2801 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2802 assert_eq!(report.retired, 0, "{report:?}");
2803 assert!(
2804 report
2805 .retained
2806 .iter()
2807 .any(|reason| reason.contains("Cargo build"))
2808 );
2809 drop(locks);
2810 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2811 assert_eq!(report.retired, 1, "{report:?}");
2812 assert!(!wt.exists());
2813 assert!(!reviewer.exists());
2814 assert!(
2815 residue_proof(&f.home, &empty.id, &reviewer)
2816 .unwrap()
2817 .exists()
2818 );
2819 assert!(
2820 proof_path.exists(),
2821 "metadata evidence outlives the worktree"
2822 );
2823 assert_eq!(
2824 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
2825 Some(source.id)
2826 );
2827 for run in [&changed, &unknown, &unproven] {
2828 assert!(run.candidates[0].worktree.exists());
2829 assert!(
2830 RunState::load_under(&run.id, &f.home)
2831 .unwrap()
2832 .retired_to
2833 .is_none()
2834 );
2835 }
2836 }
2837
2838 #[tokio::test]
2839 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
2840 let f = Fx::new().await;
2841 let source = f.run("0000", true).await;
2842 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
2843 .await
2844 .unwrap();
2845 let mut runs = vec![source.clone()];
2846 for n in 1..=12 {
2847 runs.push(f.run(&format!("{n:04}"), false).await);
2848 }
2849 let dirty = runs[1].candidates[0].worktree.clone();
2850 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
2851 let ignored = runs[2].candidates[0].worktree.clone();
2852 std::fs::create_dir(ignored.join("scratch")).unwrap();
2853 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
2854 runs[1].candidates.clear();
2857 runs[1].save_under(&f.home).unwrap();
2858 let task = f.task(&runs);
2859 let mut total = 0;
2860 for _ in 0..4 {
2861 let mut report = Report::default();
2862 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2863 assert!(report.retired <= 5);
2864 total += report.retired;
2865 }
2866 assert_eq!(total, 10);
2867 assert!(dirty.exists());
2868 assert!(ignored.exists());
2869 assert_eq!(
2870 git::rev_parse(&f.repo, &source.candidates[0].branch)
2871 .await
2872 .unwrap(),
2873 source_head
2874 );
2875 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
2876 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
2877 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
2878 let mut released = task.clone();
2879 released.release();
2880 assert_eq!(
2881 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
2882 RunState::load_under(id, &f.home)
2883 }),
2884 Some(source.id.clone())
2885 );
2886
2887 assert!(
2888 f.home
2889 .join("runs")
2890 .join(&retired.id)
2891 .join("run.json")
2892 .exists()
2893 );
2894 assert!(
2895 f.home
2896 .join("recovery-lineages")
2897 .join(format!("{}.json", task.id))
2898 .exists()
2899 );
2900 }
2901
2902 #[tokio::test]
2903 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
2904 let f = Fx::new().await;
2905 let a = f.run("dddd", true).await;
2906 let b = f.run("eeee", true).await;
2907 let empty = f.run("ffff", false).await;
2908 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
2909 assert!(
2910 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
2911 "unfinished queue history protects older source branches from ordinary folding"
2912 );
2913 let mut prior = a.clone();
2914 assert!(
2915 crate::graph::fold_run(&mut prior, true, &f.home)
2916 .await
2917 .unwrap_err()
2918 .to_string()
2919 .contains(&task.id)
2920 );
2921 let mut report = Report::default();
2922 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2923 assert_eq!(report.retired, 0);
2924 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
2925 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2926 Fx::target(&a.candidates[0].worktree);
2927 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2928 assert!(
2929 report.freed_bytes > 0,
2930 "source retention does not retain regenerable outputs"
2931 );
2932 assert!(a.candidates[0].worktree.exists());
2933 assert!(b.candidates[0].worktree.exists());
2934 assert!(empty.candidates[0].worktree.exists());
2935 }
2936
2937 #[tokio::test]
2938 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
2939 let f = Fx::new().await;
2940 let authority = f.run("p000", true).await;
2941 let question_run = f.run("p001", false).await;
2942 let mut checkpoint = f.run("p002", false).await;
2943 checkpoint.land_approval = Some(crate::run::LandApproval {
2944 question: "pending".into(),
2945 head: f.base.clone(),
2946 });
2947 checkpoint.save_under(&f.home).unwrap();
2948 let retained = f.run("p003", false).await;
2949 std::fs::write(
2950 f.home
2951 .join("runs")
2952 .join(&retained.id)
2953 .join("retain-recovery-sources"),
2954 "owner",
2955 )
2956 .unwrap();
2957 let mut partial = f.run("p004", false).await;
2958 let reviewer = partial.worktree_root().join("review-1");
2959 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2960 .await
2961 .unwrap();
2962 partial.retired_to = Some(authority.id.clone());
2964 partial.save_under(&f.home).unwrap();
2965 assert!(
2966 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
2967 .await
2968 .unwrap()
2969 );
2970 let git_checkpoint = f.run("p005", false).await;
2971 let location = git::git(
2972 &git_checkpoint.candidates[0].worktree,
2973 &[
2974 "rev-parse",
2975 "--path-format=absolute",
2976 "--git-path",
2977 "sequencer",
2978 ],
2979 )
2980 .await
2981 .unwrap();
2982 std::fs::create_dir(location.trim()).unwrap();
2983 let mut preservation = f.run("p006", false).await;
2984 preservation
2985 .candidate_preservation
2986 .insert('A', "pending preservation".into());
2987 preservation.save_under(&f.home).unwrap();
2988 let task = f.task(&[
2989 preservation.clone(),
2990 git_checkpoint.clone(),
2991 authority.clone(),
2992 question_run.clone(),
2993 checkpoint.clone(),
2994 retained.clone(),
2995 partial.clone(),
2996 ]);
2997 let questions = crate::ask::Questions::at(f.home.join("questions"));
2998 let mut question = crate::ask::Question::new(
2999 question_run.id.clone(),
3000 "implement".into(),
3001 "fixture".into(),
3002 "owner decision".into(),
3003 String::new(),
3004 vec!["keep".into()],
3005 );
3006 questions.put(&mut question).unwrap();
3007 let mut report = Report::default();
3008 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3009 assert_eq!(report.retired, 1, "{report:?}");
3010 assert!(!reviewer.exists());
3011 for state in [
3012 &question_run,
3013 &checkpoint,
3014 &retained,
3015 &git_checkpoint,
3016 &preservation,
3017 ] {
3018 assert!(state.candidates[0].worktree.exists());
3019 assert!(
3020 RunState::load_under(&state.id, &f.home)
3021 .unwrap()
3022 .retired_to
3023 .is_none()
3024 );
3025 }
3026 assert!(questions.get(&question.id).unwrap().status.open());
3027 assert_eq!(
3028 crate::queue::Queue::at(f.home.join("queue"))
3029 .get(&task.id)
3030 .unwrap()
3031 .status,
3032 task.status
3033 );
3034 }
3035
3036 #[tokio::test]
3037 async fn preservation_schema_migration_remains_distinct_from_retirement() {
3038 let f = Fx::new().await;
3039 let mut state = f.run("p007", false).await;
3040 state.schema = 15;
3041 state.save_under(&f.home).unwrap();
3042 let native = RunState::load_under(&state.id, &f.home).unwrap();
3043 assert_eq!(native.schema, crate::run::SCHEMA);
3044 assert!(native.candidate_preservation.is_empty());
3045 assert!(native.retired_to.is_none());
3046 state.schema = 14;
3047 state.save_under(&f.home).unwrap();
3048 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
3049 assert!(legacy.candidate_preservation.contains_key(&'A'));
3050 legacy.save_under(&f.home).unwrap();
3051 let restored = RunState::load_under(&state.id, &f.home).unwrap();
3052 assert_eq!(
3053 restored.candidate_preservation,
3054 legacy.candidate_preservation
3055 );
3056 }
3057
3058 #[tokio::test]
3059 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
3060 let f = Fx::new().await;
3061 let cache = f.repo.join("cache");
3062 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
3063 std::fs::create_dir_all(&cache).unwrap();
3064 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3065 assert!(
3066 crate::cache::maintenance_prune(&f.home, &cache, 1)
3067 .unwrap()
3068 .is_none()
3069 );
3070 drop(guard);
3071 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3072 }
3073
3074 #[tokio::test]
3075 async fn launcher_death_does_not_release_live_tool_output_reservations() {
3076 let f = Fx::new().await;
3077 let run = f.run("abcd", false).await;
3078 let wt = &run.candidates[0].worktree;
3079 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
3080 helper
3081 .quiet()
3082 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
3083 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
3084 .env("MAGI_TEST_WORKTREE", wt)
3085 .stdin(std::process::Stdio::null())
3086 .stdout(std::process::Stdio::null())
3087 .stderr(std::process::Stdio::null())
3088 .kill_on_drop(true);
3089 assert!(
3090 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
3091 .await
3092 .unwrap()
3093 .unwrap()
3094 .success()
3095 );
3096 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
3097 .unwrap()
3098 .trim()
3099 .parse()
3100 .unwrap();
3101 struct KillTool(i32);
3102 impl Drop for KillTool {
3103 fn drop(&mut self) {
3104 unsafe { libc::kill(self.0, libc::SIGKILL) };
3106 }
3107 }
3108 let tool = KillTool(pid);
3109 assert!(OutputLease::take(wt, true).is_err());
3110 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
3111 drop(tool);
3112 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3113 loop {
3114 if OutputLease::take(wt, true).is_ok()
3115 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
3116 {
3117 break;
3118 }
3119 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3120 }
3121 })
3122 .await
3123 .unwrap();
3124 }
3125
3126 #[tokio::test]
3127 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
3128 let f = Fx::new().await;
3129 let run = f.run("abcd", false).await;
3130 let wt = &run.candidates[0].worktree;
3131 let cache = f.repo.join("cache");
3132 std::fs::create_dir(&cache).unwrap();
3133 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
3134 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
3135 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
3136 cmd.quiet()
3137 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
3138 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
3139 .stdin(std::process::Stdio::null())
3140 .stdout(std::process::Stdio::null())
3141 .stderr(std::process::Stdio::null())
3142 .kill_on_drop(true);
3143 if clear_owner {
3144 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
3145 } else {
3146 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
3147 }
3148 let mut running = crate::seat_process::Running::spawn(
3149 cmd,
3150 None,
3151 vec![
3152 OutputLease::borrow(wt).unwrap().unwrap(),
3153 OutputLease::cache(&f.home, &cache, false).unwrap(),
3154 ],
3155 )
3156 .unwrap();
3157 assert!(
3158 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
3159 .await
3160 .unwrap()
3161 .unwrap()
3162 .success()
3163 );
3164 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
3165 struct KillTool(Option<i32>);
3166 impl Drop for KillTool {
3167 fn drop(&mut self) {
3168 if let Some(pid) = self.0 {
3170 unsafe { libc::kill(pid, libc::SIGKILL) };
3171 }
3172 }
3173 }
3174 let mut tool = KillTool(Some(pid));
3175 assert!(OutputLease::take(wt, true).is_err());
3179 let finish = if cancel {
3180 drop(running);
3181 None
3182 } else {
3183 Some(tokio::spawn(async move { running.finish().await }))
3184 };
3185 if clear_owner {
3186 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3187 assert!(crate::proc::pid_alive(pid as u32));
3188 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
3189 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3190 assert!(
3191 crate::cache::maintenance_prune(&f.home, &cache, 0)
3192 .unwrap()
3193 .is_none()
3194 );
3195 if let Some(finish) = &finish {
3196 assert!(!finish.is_finished());
3197 }
3198 unsafe { libc::kill(pid, libc::SIGKILL) };
3200 }
3201 if let Some(finish) = finish {
3202 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
3203 .await
3204 .unwrap()
3205 .unwrap();
3206 }
3207 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3208 loop {
3209 if OutputLease::take(wt, true).is_ok()
3210 && OutputLease::cache(&f.home, &cache, true).is_ok()
3211 {
3212 if !clear_owner {
3214 assert!(!crate::proc::pid_alive(pid as u32));
3215 }
3216 break;
3217 }
3218 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3219 }
3220 })
3221 .await
3222 .unwrap();
3223 tool.0 = None;
3224 }
3225 }
3226
3227 #[tokio::test]
3228 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
3229 let f = Fx::new().await;
3230 let run = f.run("abcd", false).await;
3231 let wt = &run.candidates[0].worktree;
3232 let cache = f.repo.join("cache");
3233 std::fs::create_dir(&cache).unwrap();
3234 let pid_file = f.repo.join("child-pid");
3235 let mut cmd = tokio::process::Command::new("sh");
3236 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
3237 .quiet()
3238 .env("CHILD_PID_FILE", &pid_file)
3239 .process_group(0)
3240 .kill_on_drop(true);
3241 let running = crate::seat_process::Running::spawn(
3242 cmd,
3243 None,
3244 vec![
3245 OutputLease::borrow(wt).unwrap().unwrap(),
3246 OutputLease::cache(&f.home, &cache, false).unwrap(),
3247 ],
3248 )
3249 .unwrap();
3250 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3251 while !pid_file.exists() {
3252 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3253 }
3254 })
3255 .await
3256 .unwrap();
3257 assert!(OutputLease::take(wt, true).is_err());
3258 assert!(
3259 crate::cache::maintenance_prune(&f.home, &cache, 0)
3260 .unwrap()
3261 .is_none()
3262 );
3263 let pid: u32 = std::fs::read_to_string(pid_file)
3264 .unwrap()
3265 .trim()
3266 .parse()
3267 .unwrap();
3268 drop(running);
3269 tokio::time::timeout(std::time::Duration::from_secs(10), async {
3270 loop {
3271 if let Ok(guard) = OutputLease::take(wt, true) {
3272 assert!(
3273 !crate::proc::pid_alive(pid),
3274 "output lock released before tool death"
3275 );
3276 drop(guard);
3277 break;
3278 }
3279 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3280 }
3281 })
3282 .await
3283 .unwrap();
3284 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3285 }
3286
3287 #[tokio::test]
3288 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
3289 let f = Fx::new().await;
3290 let reason = tokio::time::timeout(
3291 std::time::Duration::from_secs(2),
3292 emergency(vec![f.repo.clone()], u64::MAX),
3293 )
3294 .await
3295 .unwrap();
3296 assert!(reason.contains("disk pressure"));
3297 assert!(
3298 tokio::time::timeout(
3299 std::time::Duration::from_millis(10),
3300 emergency(vec![f.repo.clone()], 0)
3301 )
3302 .await
3303 .is_err()
3304 );
3305 assert_eq!(
3306 std::fs::read_to_string(f.repo.join("source")).unwrap(),
3307 "base\n"
3308 );
3309 }
3310
3311 #[tokio::test]
3312 async fn automatic_requeues_cannot_refund_failed_input_generation() {
3313 let f = Fx::new().await;
3314 let mut run = f.run("1234", false).await;
3315 run.prep_failure = Some("cherry-pick conflict".into());
3316 let mut task = f.task(std::slice::from_ref(&run));
3317 let inputs = crate::queue::recovery_inputs(&run.sources);
3318 for _ in 0..108 {
3319 task.recovery_failed(&run);
3320 task.requeue();
3321 assert_eq!(task.attempts, 0);
3322 }
3323 let bytes = serde_json::to_vec(&task).unwrap();
3324 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
3325 assert!(
3326 restored
3327 .recovery_exhausted(&run.base_commit, &inputs, 2)
3328 .is_some()
3329 );
3330 assert!(
3331 restored
3332 .recovery_exhausted("different-base", &inputs, 2)
3333 .is_none()
3334 );
3335 let original_base = run.base_commit.clone();
3336 run.base_commit = "different-base".into();
3337 restored.recovery_failed(&run);
3338 assert!(
3339 restored
3340 .recovery_exhausted(&original_base, &inputs, 2)
3341 .is_some(),
3342 "returning to old inputs does not replenish their budget"
3343 );
3344 assert!(
3345 restored
3346 .recovery_exhausted(&run.base_commit, &inputs, 2)
3347 .is_none()
3348 );
3349 restored.release_operator("test-owner", None);
3350 assert!(
3351 restored
3352 .recovery_exhausted(&run.base_commit, &inputs, 2)
3353 .is_none()
3354 );
3355 }
3356}