Skip to main content

orch/engine/
cleanup.rs

1//! Worktree cleanup and merged-PR detection.
2//!
3//! Contains the post-merge cleanup pipeline: removing git worktrees,
4//! deleting local/remote branches, pulling main, and detecting
5//! already-merged PRs so their tasks can be marked done.
6
7use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
8use crate::cmd::CommandErrorContext;
9use crate::engine::ci_bootstrap;
10use crate::engine::tasks::{TaskManager, store_task_to_external};
11use crate::store;
12use crate::store::TaskStatus;
13use crate::store::TaskStore;
14use crate::store::store_log_activity;
15use dashmap::{DashMap, DashSet};
16use futures::future;
17use std::collections::HashMap;
18use std::collections::HashSet;
19use std::env;
20use std::fs as std_fs;
21use std::io::ErrorKind;
22use std::path::Path;
23use std::path::PathBuf;
24use std::process::Output;
25use std::sync::Arc;
26use std::sync::LazyLock;
27use std::time::SystemTime;
28use std::time::{Duration, Instant};
29use tokio::fs;
30use tokio::process::Command;
31use tokio::time::sleep;
32use tokio::time::timeout;
33
34use super::sync::ReviewTaskSnapshot;
35use crate::config;
36use crate::engine;
37use crate::engine::auto_merge;
38use crate::engine::runner::cow;
39use crate::engine::runner::git_ops;
40use crate::engine::runner::screenshots::SCREENSHOTS_BRANCH;
41use crate::engine::runner::worktree;
42use crate::github::http::GhHttp;
43use crate::home;
44
45/// Reconciliation calls can be slow on repos with many issues or during GitHub instability.
46/// Cleanup runs in a background task (tokio::spawn) so it does not block the main tick loop.
47/// 30 s gives enough headroom for paginated list_all_tasks calls while still bounding the
48/// overall background-task duration.
49const RECONCILIATION_LIST_TIMEOUT: Duration = Duration::from_secs(30);
50/// Closed-issue reconciliation is a best-effort cleanup fallback, not a per-sync requirement.
51/// Run it on a slower cadence so repeated GitHub slowness does not consume 30 s every sync.
52const RECONCILIATION_INTERVAL: Duration = Duration::from_mins(15);
53
54const RUN_OUTPUT_RETENTION_DAYS: i32 = 30;
55const RUN_PRUNE_INTERVAL_SECS: i64 = 24 * 60 * 60;
56const RUN_PRUNE_KV_KEY: &str = "task_runs_pruned_at";
57
58/// Per-repo in-flight guard for closed-issue reconciliation/cleanup.
59/// Prevents overlapping cleanup runs from piling up across sync cycles.
60static CLEANUP_IN_FLIGHT: LazyLock<DashSet<String>> = LazyLock::new(DashSet::new);
61/// Per-repo reconciliation cadence tracker.
62static RECONCILIATION_LAST_RUN: LazyLock<DashMap<String, Instant>> = LazyLock::new(DashMap::new);
63
64struct CleanupInFlightGuard {
65    repo: String,
66}
67
68impl CleanupInFlightGuard {
69    fn try_acquire(repo: &str) -> Option<Self> {
70        if CLEANUP_IN_FLIGHT.insert(repo.to_string()) {
71            Some(Self {
72                repo: repo.to_string(),
73            })
74        } else {
75            None
76        }
77    }
78}
79
80impl Drop for CleanupInFlightGuard {
81    fn drop(&mut self) {
82        CLEANUP_IN_FLIGHT.remove(&self.repo);
83    }
84}
85
86fn should_run_reconciliation(repo: &str) -> bool {
87    let now = Instant::now();
88    if let Some(mut last_run) = RECONCILIATION_LAST_RUN.get_mut(repo) {
89        if last_run.elapsed() < RECONCILIATION_INTERVAL {
90            return false;
91        }
92        *last_run = now;
93        true
94    } else {
95        RECONCILIATION_LAST_RUN.insert(repo.to_string(), now);
96        true
97    }
98}
99
100/// Options controlling the worktree janitor.
101#[derive(Debug, Clone)]
102pub struct JanitorOptions {
103    /// Minimum age (in hours) of the worktree directory before it is eligible for removal.
104    ///
105    /// A worktree whose filesystem mtime is more recent than this threshold is skipped
106    /// even if the task is already in a terminal state. This provides a safety buffer
107    /// against races (e.g. review agent still writing when the janitor runs).
108    ///
109    /// Default: 24.  Configurable via `workflow.worktree_janitor_ttl_hours` in config.
110    pub ttl_hours: u64,
111
112    /// When `true`, log what would be done but skip all destructive operations.
113    ///
114    /// Configurable via `workflow.worktree_janitor_dry_run: true` in config.
115    pub dry_run: bool,
116}
117
118impl Default for JanitorOptions {
119    fn default() -> Self {
120        Self {
121            ttl_hours: 24,
122            dry_run: false,
123        }
124    }
125}
126
127impl JanitorOptions {
128    /// Read options from the global config, falling back to defaults.
129    pub fn from_config() -> Self {
130        let ttl_hours = config::get("workflow.worktree_janitor_ttl_hours")
131            .ok()
132            .and_then(|v| v.parse::<u64>().ok())
133            .unwrap_or(24);
134
135        let dry_run = config::get("workflow.worktree_janitor_dry_run").is_ok_and(|v| v == "true");
136
137        Self { ttl_hours, dry_run }
138    }
139}
140
141/// Cleanup worktrees for completed tasks.
142///
143/// Targets tasks that are finished: status:done OR closed on GitHub
144/// (e.g. when a PR merge auto-closes the issue via "Fixes #N").
145///
146/// Note: This is a fallback. Primary cleanup happens inline in
147/// `auto_merge_pr` via `cleanup_task_worktree`. This catches edge cases
148/// where the inline cleanup missed (e.g., manual merges, auto-closed issues).
149pub(crate) async fn cleanup_done_worktrees(
150    backend: &Arc<dyn ExternalBackend>,
151    repo: &str,
152    task_manager: &Arc<TaskManager>,
153    store: &Arc<TaskStore>,
154) -> anyhow::Result<()> {
155    let opts = JanitorOptions::from_config();
156    cleanup_done_worktrees_with_opts(backend, repo, task_manager, store, &opts).await
157}
158
159/// Clear the stored output of runs for tasks done longer than `RUN_OUTPUT_RETENTION_DAYS`, once a day
160pub(crate) async fn prune_old_run_output(store: &TaskStore) -> anyhow::Result<()> {
161    let now = chrono::Utc::now().timestamp();
162    let last = store
163        .kv_get(RUN_PRUNE_KV_KEY)
164        .await?
165        .and_then(|v| v.parse::<i64>().ok())
166        .unwrap_or(0);
167    if now - last < RUN_PRUNE_INTERVAL_SECS {
168        return Ok(());
169    }
170    store.kv_set(RUN_PRUNE_KV_KEY, &now.to_string()).await?;
171
172    let pruned = store.prune_old_runs(RUN_OUTPUT_RETENTION_DAYS).await?;
173    if pruned > 0 {
174        tracing::info!(pruned, "cleared the output of old task runs");
175    }
176    Ok(())
177}
178
179async fn reconcile_closed_tasks(
180    task_ids: &mut Vec<String>,
181    all_tasks: &[ExternalTask],
182    task_manager: &TaskManager,
183    store: &Arc<TaskStore>,
184    repo: &str,
185) {
186    const MAX_RECONCILE_PER_CYCLE: usize = 10;
187
188    let done_set: HashSet<String> = task_ids.iter().cloned().collect();
189    let mut closed_not_in_store = Vec::new();
190    for task in all_tasks {
191        if task.state != "closed" || done_set.contains(&task.id.0) {
192            continue;
193        }
194        if ci_bootstrap::hold_after_merge(task, store, repo).await {
195            continue;
196        }
197        closed_not_in_store.push(task);
198    }
199
200    // Split: issues that already have status:done label just need worktree
201    // cleanup (no API call needed), vs those needing label reconciliation.
202    let mut already_labeled: Vec<String> = Vec::new();
203    let mut needs_reconcile: Vec<&ExternalTask> = Vec::new();
204    for task in &closed_not_in_store {
205        if task.labels.iter().any(|l| l == "status:done") {
206            already_labeled.push(task.id.0.clone());
207        } else {
208            needs_reconcile.push(task);
209        }
210    }
211
212    // Add already-labeled issues directly (no API call).
213    if !already_labeled.is_empty() {
214        tracing::debug!(
215            count = already_labeled.len(),
216            "closed issues already labeled status:done — adding to cleanup set"
217        );
218        task_ids.extend(already_labeled);
219    }
220
221    // Reconcile issues missing the done label — cap to avoid rate-limit
222    // exhaustion. Remaining issues will be picked up in subsequent cycles.
223    if needs_reconcile.len() > MAX_RECONCILE_PER_CYCLE {
224        tracing::info!(
225            total = needs_reconcile.len(),
226            batch = MAX_RECONCILE_PER_CYCLE,
227            "capping closed-issue reconciliation to avoid rate-limit exhaustion"
228        );
229    }
230    let mut futs = Vec::new();
231    for task in needs_reconcile.iter().take(MAX_RECONCILE_PER_CYCLE) {
232        let id = task.id.clone();
233        let task_id_str = task.id.0.clone();
234        let labels = task.labels.clone();
235        futs.push(async move {
236            tracing::info!(
237                task_id = task_id_str,
238                labels = ?labels,
239                "closed issue with stale status label — reconciling to done and cleaning up"
240            );
241            match task_manager.update_task_status(&id, Status::Done).await {
242                Ok(()) => Some(task_id_str),
243                Err(e) => {
244                    tracing::warn!(
245                        task_id = task_id_str,
246                        err = %e,
247                        "failed to reconcile closed issue status to done"
248                    );
249                    None
250                }
251            }
252        });
253    }
254    let results = future::join_all(futs).await;
255    task_ids.extend(results.into_iter().flatten());
256}
257
258/// Cleanup worktrees for completed tasks with explicit options.
259///
260/// Separated from `cleanup_done_worktrees` so that integration tests can
261/// inject specific options (e.g. `ttl_hours: 0`, `dry_run: true`) without
262/// touching the global config.
263pub(crate) async fn cleanup_done_worktrees_with_opts(
264    backend: &Arc<dyn ExternalBackend>,
265    repo: &str,
266    task_manager: &Arc<TaskManager>,
267    store: &Arc<TaskStore>,
268    opts: &JanitorOptions,
269) -> anyhow::Result<()> {
270    let Some(_guard) = CleanupInFlightGuard::try_acquire(repo) else {
271        tracing::debug!(
272            repo = %repo,
273            "cleanup already in progress for repo — skipping overlapping run"
274        );
275        return Ok(());
276    };
277
278    // Read done tasks from the store first; fall back to backend before first sync.
279    let done_tasks = task_manager.list_all_by_status(Status::Done).await?;
280    tracing::debug!(count = done_tasks.len(), "checking done tasks for cleanup");
281
282    // Collect task IDs from external done tasks.
283    let mut task_ids: Vec<String> = done_tasks.iter().map(|t| t.id.0.clone()).collect();
284
285    // Also include closed-but-not-done external tasks.
286    // When a PR merge auto-closes a GitHub issue (via "Fixes #N"), the issue
287    // state becomes "closed" but orch never updates the status label to "done".
288    // These orphaned worktrees accumulate unless we catch them here.
289    // Note: This still queries the backend because GitHub `state` (open/closed)
290    // is backend-specific and not tracked in the store.
291    //
292    // Use list_reconciliation_candidates() (closed issues only, bounded recent window) directly.
293    // list_all_tasks() fetches every issue ever created (state=all, all pages) and
294    // consistently times out on repos with many issues — it is not appropriate for
295    // a cleanup fallback path.
296    if should_run_reconciliation(repo) {
297        match timeout(
298            RECONCILIATION_LIST_TIMEOUT,
299            backend.list_reconciliation_candidates(),
300        )
301        .await
302        {
303            Ok(Ok(candidates)) if !candidates.is_empty() => {
304                tracing::debug!(
305                    count = candidates.len(),
306                    "reconciling closed issues from candidates"
307                );
308                reconcile_closed_tasks(&mut task_ids, &candidates, task_manager, store, repo).await;
309            }
310            Ok(Ok(_)) => {
311                tracing::debug!("no reconciliation candidates found");
312            }
313            Ok(Err(e)) => {
314                tracing::warn!(err = %e, "failed to list reconciliation candidates");
315            }
316            Err(_) => {
317                tracing::warn!(
318                    timeout_secs = RECONCILIATION_LIST_TIMEOUT.as_secs(),
319                    retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
320                    "timed out listing reconciliation candidates"
321                );
322            }
323        }
324    } else {
325        tracing::debug!(
326            retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
327            "skipping closed-issue reconciliation until interval elapses"
328        );
329    }
330
331    // Also include internal done tasks.
332    if let Ok(internal_done) = task_manager.list_internal_by_status(TaskStatus::Done).await {
333        task_ids.extend(internal_done.iter().map(|t| t.id.0.clone()));
334    }
335
336    tracing::debug!(
337        count = task_ids.len(),
338        "checking all terminal tasks for cleanup"
339    );
340
341    if let Err(e) = auto_merge::sweep_ci_timer_keys(store, repo, &task_ids).await {
342        tracing::warn!(err = %e, "failed to sweep CI timer keys");
343    }
344
345    // Prefetch all cleaned external IDs in one query instead of N+1 per-task lookups.
346    // NOTE: This is now primarily a hint to skip processing, NOT a correctness
347    // guarantee — tasks with external_id NOT in the store at all (or mismatched
348    // repo) will still be cleaned by iterating all worktree directories.
349    let already_cleaned = store.cleaned_external_ids(repo).await.unwrap_or_default();
350
351    let mut cleaned_any = false;
352    for task_id in &task_ids {
353        // Skip if already cleaned (checked via prefetched set).
354        if already_cleaned.contains(task_id) {
355            continue;
356        }
357
358        match cleanup_task_worktree_with_opts(task_id, repo, store, opts).await {
359            Ok(true) => cleaned_any = true,
360            Ok(false) => {
361                // Skipped (TTL guard, tmux guard, or nothing to remove) — do not
362                // pull main; we did not change the repo state.
363            }
364            Err(e) => {
365                tracing::warn!(task_id, err = %e, "worktree cleanup failed for task");
366            }
367        }
368    }
369
370    // ── Orphaned worktree scanner ─────────────────────────────────────────────
371    //
372    // Even if every task in our list was already cleaned (or had no worktree),
373    // there may be worktree directories on disk that are not owned by any task.
374    // This happens when a task was ingested into the store but its worktree_cleaned
375    // flag was never set — e.g. because the task was never in `list_all_by_status`
376    // (unknown repo / missing from store at cleanup time), or the cleanup skipped
377    // due to TTL guard and never retried.
378    //
379    // We scan all worktree directories in this repo's worktrees subdirectory and
380    // remove any that are not referenced by an existing task. This catches the
381    // "done but worktree not cleaned" case that `orch service doctor` reports.
382    let worktrees_base = match home::worktrees_dir() {
383        Ok(base) => Some(base),
384        Err(e) => {
385            tracing::debug!(err = %e, "worktrees_dir() unavailable — skipping orphan scan");
386            None
387        }
388    };
389
390    if let Some(base) = worktrees_base {
391        let project = repo
392            .rsplit('/')
393            .next()
394            .unwrap_or(repo)
395            .trim_end_matches(".git");
396        let repo_worktrees = base.join(project);
397
398        // Build set of all known worktree paths for this repo (all statuses).
399        let mut combined: HashSet<String> = HashSet::new();
400        if let Ok(tasks) = store.list_all(repo).await {
401            for t in tasks {
402                if !t.worktree.is_empty() {
403                    combined.insert(t.worktree);
404                }
405            }
406        }
407
408        // Also include active worktrees (in case task is in-memory but has a dir).
409        if let Ok(project_dir) = resolve_project_dir_for_repo(repo)
410            && let Ok(active_wts) = worktree::list_project_worktrees(&project_dir).await
411        {
412            for wt in active_wts {
413                combined.insert(wt.to_string_lossy().to_string());
414            }
415        }
416
417        // Scan all branch directories and remove orphaned ones.
418        if let Ok(mut entries) = fs::read_dir(&repo_worktrees).await {
419            while let Ok(Some(entry)) = entries.next_entry().await {
420                let Ok(entry_meta) = entry.metadata().await else {
421                    continue;
422                };
423                if !entry_meta.is_dir() {
424                    continue;
425                }
426                let wt_path = entry.path().to_string_lossy().to_string();
427                if combined.contains(&wt_path) {
428                    continue;
429                }
430                tracing::info!(worktree = %wt_path, "found orphaned worktree — attempting cleanup");
431                match resolve_repo_root_for_orphaned_worktree(&entry.path()).await {
432                    Ok(repo_root) => {
433                        let removed = remove_worktree_and_branch(
434                            "orphan-scan",
435                            &entry.path(),
436                            None,
437                            Path::new(&repo_root),
438                            false,
439                        )
440                        .await;
441                        if removed {
442                            tracing::info!(worktree = %wt_path, "orphaned worktree removed");
443                        } else {
444                            tracing::warn!(worktree = %wt_path, "orphaned worktree could not be removed");
445                        }
446                    }
447                    Err(e) => {
448                        tracing::warn!(worktree = %wt_path, err = %e, "could not resolve repo root for orphaned worktree");
449                    }
450                }
451            }
452        }
453    }
454
455    // Prune stale worktree metadata and pull main after cleanup.
456    if cleaned_any
457        && !opts.dry_run
458        && let Ok(repo_root) = resolve_repo_root(repo).await
459    {
460        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
461
462        // Prune stale .git/worktrees/ entries whose directories no longer exist.
463        let prune_result = timeout(
464            GIT_TIMEOUT,
465            Command::new("git")
466                .args(["-C", &repo_root, "worktree", "prune"])
467                .output_with_context(),
468        )
469        .await;
470        match prune_result {
471            Ok(Ok(output)) if output.status.success() => {
472                tracing::debug!(%repo, "pruned stale worktree metadata");
473            }
474            Ok(Ok(output)) => {
475                let stderr = String::from_utf8_lossy(&output.stderr);
476                tracing::warn!(%repo, err = %stderr, "git worktree prune failed");
477            }
478            Ok(Err(e)) => {
479                tracing::warn!(%repo, err = %e, "git worktree prune failed");
480            }
481            Err(_) => {
482                tracing::warn!(%repo, "git worktree prune timed out after 60s");
483            }
484        }
485
486        // Fetch instead of pull: refresh `origin/*` refs without touching
487        // the operator's checked-out branch or working tree. `pull` would
488        // operate on whatever HEAD happens to be (often a feature branch,
489        // not main) and would silently no-op on divergence.
490        let fetch_result = timeout(
491            GIT_TIMEOUT,
492            git_ops::git()
493                .await
494                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
495                .output_with_context(),
496        )
497        .await;
498        match fetch_result {
499            Ok(Ok(output)) if output.status.success() => {
500                tracing::info!(%repo, "fetched origin after worktree cleanup");
501            }
502            Ok(Ok(output)) => {
503                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
504                tracing::debug!(%repo, err = %stderr, "git fetch skipped after cleanup");
505            }
506            Ok(Err(e)) => {
507                tracing::debug!(%repo, err = %e, "git fetch failed after cleanup");
508            }
509            Err(_) => {
510                tracing::debug!(%repo, "git fetch timed out after 60s");
511            }
512        }
513    }
514
515    Ok(())
516}
517
518/// Cleanup a single task's worktree and branches.
519///
520/// Removes the git worktree, deletes local + remote branches,
521/// fetches origin to refresh remote tracking refs, and marks the task as cleaned.
522///
523/// This function is used for inline post-merge cleanup (called from
524/// auto-merge flows) and must attempt immediate removal — it constructs
525/// janitor options with `ttl_hours = 0` so the janitor age guard does not
526/// postpone removing freshly-created worktrees.
527pub(crate) async fn cleanup_task_worktree(
528    task_id: &str,
529    repo: &str,
530    store: &Arc<TaskStore>,
531) -> anyhow::Result<()> {
532    cleanup_task_worktree_impl(task_id, repo, store, false).await
533}
534
535/// Cleanup after a merged pull request, including while `hold_after_merge` keeps the task approved
536pub(crate) async fn cleanup_merged_pr_worktree(
537    task_id: &str,
538    repo: &str,
539    store: &Arc<TaskStore>,
540) -> anyhow::Result<()> {
541    cleanup_task_worktree_impl(task_id, repo, store, true).await
542}
543
544async fn cleanup_task_worktree_impl(
545    task_id: &str,
546    repo: &str,
547    store: &Arc<TaskStore>,
548    known_pr_merged: bool,
549) -> anyhow::Result<()> {
550    let opts = JanitorOptions {
551        ttl_hours: 0,
552        ..Default::default()
553    };
554    let cleaned =
555        cleanup_task_worktree_with_opts_impl(task_id, repo, store, &opts, known_pr_merged).await?;
556
557    // Fetch origin once after inline cleanup (post-merge single-task path).
558    // Only fetch if cleanup actually happened (not skipped due to active session).
559    // Fetch (not pull) so the operator's checked-out branch and working tree
560    // are never touched — see the matching comment in cleanup_orphaned_worktrees.
561    if cleaned && let Ok(repo_root) = resolve_repo_root(repo).await {
562        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
563        let fetch_result = timeout(
564            GIT_TIMEOUT,
565            git_ops::git()
566                .await
567                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
568                .output_with_context(),
569        )
570        .await;
571        match fetch_result {
572            Ok(Ok(output)) if output.status.success() => {
573                tracing::debug!(%repo, "fetched origin after task cleanup");
574            }
575            Ok(Ok(output)) => {
576                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
577                tracing::debug!(%repo, err = %stderr, "git fetch skipped after task cleanup");
578            }
579            Ok(Err(e)) => {
580                tracing::debug!(%repo, err = %e, "git fetch failed after task cleanup");
581            }
582            Err(_) => {
583                tracing::debug!(%repo, "git fetch timed out after 60s");
584            }
585        }
586    }
587
588    Ok(())
589}
590
591/// Cleanup a single task's worktree and branches with explicit janitor options.
592///
593/// Returns `Ok(true)` when cleanup was actually performed and the task was
594/// marked as cleaned.  Returns `Ok(false)` when cleanup was intentionally
595/// skipped (TTL guard, active tmux session guard, or nothing on disk to remove).
596/// Returns `Err` on unexpected failures (e.g. cannot resolve the repo root).
597pub(crate) async fn cleanup_task_worktree_with_opts(
598    task_id: &str,
599    repo: &str,
600    store: &Arc<TaskStore>,
601    opts: &JanitorOptions,
602) -> anyhow::Result<bool> {
603    cleanup_task_worktree_with_opts_impl(task_id, repo, store, opts, false).await
604}
605
606async fn cleanup_task_worktree_with_opts_impl(
607    task_id: &str,
608    repo: &str,
609    store: &Arc<TaskStore>,
610    opts: &JanitorOptions,
611    known_pr_merged: bool,
612) -> anyhow::Result<bool> {
613    let (worktree, branch, keep_remote_branch, merged, external_id, pr_merged) =
614        store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
615            .await
616            .map_or((None, None, false, false, None, false), |t| {
617                // Blocked tasks with a PR should keep their remote branch so the
618                // PR stays open for human review.
619                let keep = t.status == TaskStatus::Blocked && t.pr_number.is_some();
620                let merged = t.status == TaskStatus::Done || known_pr_merged;
621                (
622                    Some(t.worktree).filter(|w| !w.is_empty()),
623                    Some(t.branch).filter(|b| !b.is_empty()),
624                    keep,
625                    merged,
626                    t.external_id,
627                    // A squash leaves the branch commits on no ref, so skip rescue when the PR merged
628                    known_pr_merged || (t.status == TaskStatus::Done && t.pr_number.is_some()),
629                )
630            });
631
632    let worktree_path = worktree.as_ref().map(PathBuf::from);
633
634    // Get worktrees base path
635    let worktrees_base = home::worktrees_dir().unwrap_or_else(|_| PathBuf::from(".orch/worktrees"));
636
637    // Determine which directory to remove.
638    //
639    // Priority:
640    //   1. stored "worktree" path, if it exists on disk.
641    //   2. Construct from worktrees_base + project name + branch, if it exists.
642    let worktree_to_remove = if let Some(ref wt) = worktree_path {
643        if match fs::try_exists(wt).await {
644            Ok(true) => true,
645            Ok(false) => false,
646            Err(e) => {
647                tracing::warn!(task_id, worktree = %wt.display(), err = %e, "try_exists failed on stored worktree path — trying branch-based fallback");
648                false
649            }
650        } {
651            Some(wt.clone())
652        } else {
653            // The recorded worktree path doesn't exist — try reconstructing from branch.
654            if let Some(ref b) = branch {
655                let project = repo
656                    .rsplit('/')
657                    .next()
658                    .unwrap_or(repo)
659                    .trim_end_matches(".git");
660                let candidate = worktrees_base.join(project).join(b);
661                if match fs::try_exists(&candidate).await {
662                    Ok(true) => true,
663                    Ok(false) => false,
664                    Err(e) => {
665                        tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
666                        false
667                    }
668                } {
669                    Some(candidate)
670                } else {
671                    None
672                }
673            } else {
674                None
675            }
676        }
677    } else if let Some(ref b) = branch {
678        // No worktree path in store — try branch-based path.
679        let project = repo
680            .rsplit('/')
681            .next()
682            .unwrap_or(repo)
683            .trim_end_matches(".git");
684        let candidate = worktrees_base.join(project).join(b);
685        if match fs::try_exists(&candidate).await {
686            Ok(true) => true,
687            Ok(false) => false,
688            Err(e) => {
689                tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
690                false
691            }
692        } {
693            Some(candidate)
694        } else {
695            None
696        }
697    } else {
698        None
699    };
700
701    let mut did_clean = false;
702
703    // If there's nothing to remove (no worktree path and no branch),
704    // treat as a no-op and return Ok(false) without attempting to
705    // resolve the repo root. Resolving the repo root can fail for
706    // projects not registered in config and that should not make
707    // cleanup a hard error when there is nothing to do.
708    let branch_nonempty = branch.as_ref().filter(|b| !b.is_empty());
709    if worktree_to_remove.is_none() && branch_nonempty.is_none() {
710        // If the stored worktree path is non-empty but doesn't exist on disk,
711        // the worktree is already gone — mark it cleaned so we stop retrying.
712        if worktree_path
713            .as_ref()
714            .is_some_and(|p| !p.as_os_str().is_empty())
715        {
716            tracing::debug!(
717                task_id,
718                worktree = ?worktree_path,
719                "stored worktree path no longer exists on disk — marking cleaned"
720            );
721            if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
722                && let Err(e) = store.mark_cleaned(store_id).await
723            {
724                tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned in store — will retry");
725            }
726        }
727        return Ok(false);
728    }
729
730    if let Some(wt) = worktree_to_remove {
731        // A task directory sits at least two levels under the base, never the base or a project dir
732        if let Ok(rel) = wt.strip_prefix(&worktrees_base)
733            && rel.components().count() < 2
734        {
735            tracing::warn!(
736                task_id,
737                worktree = %wt.display(),
738                "refusing to clean the worktrees root or a project directory"
739            );
740            return Ok(false);
741        }
742
743        // TTL guard: skip if the worktree directory is too young.
744        if let Some(age_hours) = worktree_age_hours(&wt).await
745            && age_hours < opts.ttl_hours
746        {
747            tracing::debug!(
748                task_id,
749                worktree = %wt.display(),
750                age_hours,
751                ttl_hours = opts.ttl_hours,
752                "worktree too young for cleanup, skipping"
753            );
754            return Ok(false);
755        }
756
757        // Tmux guard: skip if any active pane still has its cwd inside this worktree.
758        if is_worktree_in_active_session(&wt).await {
759            tracing::warn!(
760                task_id,
761                worktree = %wt.display(),
762                "worktree is referenced by an active tmux session — skipping cleanup"
763            );
764            return Ok(false);
765        }
766
767        // Validate git metadata before attempting any git operations.
768        // If the .git file points to a deleted gitdir the worktree is
769        // unrecoverable — force-remove the directory and skip git commands.
770        let gitdir_valid = worktree::validate_worktree_gitdir(&wt).await;
771
772        if opts.dry_run {
773            tracing::info!(
774                task_id,
775                worktree = %wt.display(),
776                branch = ?branch,
777                gitdir_valid,
778                "[dry-run] would remove worktree and branch"
779            );
780        } else if !gitdir_valid {
781            tracing::warn!(
782                task_id,
783                worktree = %wt.display(),
784                "worktree has broken git metadata — force-removing directory without git operations"
785            );
786            match fs::remove_dir_all(&wt).await {
787                Ok(()) => {
788                    did_clean = true;
789                }
790                Err(e) => {
791                    tracing::warn!(
792                        task_id,
793                        worktree = %wt.display(),
794                        err = %e,
795                        "failed to force-remove broken worktree directory"
796                    );
797                }
798            }
799        } else {
800            tracing::info!(task_id, worktree = %wt.display(), "removing worktree");
801            // Derive repo root from the worktree itself (for cross-project support).
802            // If the worktree belongs to a different project than the current repo
803            // context (e.g., internal task with worktree in another project), we can
804            // still find the repo root by examining the worktree's .git file.
805            let repo_root = match resolve_repo_root_from_worktree(&wt).await {
806                Ok(root) => root,
807                Err(_) => resolve_repo_root(repo).await?,
808            };
809            if let Some(br) = branch.as_deref().filter(|b| !b.is_empty())
810                && !pr_merged
811                && !rescue_and_notify(
812                    task_id,
813                    repo,
814                    external_id.as_deref(),
815                    Some(&wt),
816                    br,
817                    Path::new(&repo_root),
818                )
819                .await
820            {
821                return Ok(false);
822            }
823            if merged {
824                cow::warm_from_merged(
825                    Path::new(&repo_root),
826                    &worktree::project_name(Path::new(&repo_root)),
827                    &wt,
828                )
829                .await;
830            }
831            let removed = remove_worktree_and_branch_unchecked(
832                task_id,
833                &wt,
834                branch.as_deref(),
835                Path::new(&repo_root),
836                keep_remote_branch,
837            )
838            .await;
839            if removed {
840                did_clean = true;
841            } else {
842                tracing::warn!(
843                    task_id,
844                    worktree = %wt.display(),
845                    "worktree directory still exists after removal attempt — not marking as cleaned"
846                );
847            }
848        }
849    } else if let Some(ref br) = branch {
850        // Worktree directory is already gone, but the branch may still exist
851        // on the remote. Delete it to avoid orphaned branches.
852        if !opts.dry_run && !keep_remote_branch {
853            tracing::debug!(task_id, branch = %br, "no worktree on disk, cleaning up branch only");
854            // Resolve repo root lazily — branch cleanup needs it.
855            let repo_root = resolve_repo_root(repo).await?;
856            if !pr_merged
857                && !rescue_and_notify(
858                    task_id,
859                    repo,
860                    external_id.as_deref(),
861                    None,
862                    br,
863                    Path::new(&repo_root),
864                )
865                .await
866            {
867                return Ok(false);
868            }
869            delete_branches(task_id, br, Path::new(&repo_root)).await;
870            did_clean = true;
871        }
872    }
873
874    if did_clean {
875        // Mark as cleaned in store so we don't retry next tick.
876        // If resolve_task_id returns None the task is not (or no longer) in
877        // the store; that is fine — we already did the on-disk work.
878        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
879            && let Err(e) = store.mark_cleaned(store_id).await
880        {
881            tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned — will retry");
882            // Don't log activity; retry next tick will do the actual cleanup
883            return Ok(did_clean);
884        }
885        store_log_activity(
886            &Some(Arc::clone(store)),
887            repo,
888            task_id,
889            "branch_delete",
890            None,
891            None,
892            None,
893            None,
894            Some(&serde_json::json!({
895                "worktree": worktree_path.map(|p| p.display().to_string()),
896                "branch": branch,
897                "keep_remote_branch": keep_remote_branch,
898            })),
899        )
900        .await;
901    }
902
903    Ok(did_clean)
904}
905
906/// Result of saving a task branch's unpushed work before cleanup deletes it
907#[derive(Debug, PartialEq, Eq)]
908pub(crate) enum Rescue {
909    /// Every commit is on a remote ref already
910    Clean,
911    /// The commits were pushed to `rescue/<branch>`
912    Pushed {
913        branch: String,
914        commits: Vec<String>,
915    },
916    /// Git failed, so the worktree and branch must stay
917    Failed,
918}
919
920async fn git_in(dir: &Path, args: &[&str]) -> Option<Output> {
921    git_ops::git()
922        .await
923        .arg("-C")
924        .arg(dir)
925        .args(args)
926        .output_with_context()
927        .await
928        .ok()
929        .map(git_ops::sanitize_git_output)
930}
931
932/**
933 * Cleanup deletes the worktree and the local branch, which can be the only refs to commits the
934 * agent made before its run ended without a push (#3999). Uncommitted changes are committed
935 * first, then everything not on a remote ref goes to `rescue/<branch>`
936 **/
937pub(crate) async fn rescue_unpushed_work(
938    task_id: &str,
939    wt: Option<&Path>,
940    br: &str,
941    repo_root: &Path,
942) -> Rescue {
943    if let Some(wt) = wt
944        && fs::try_exists(wt.join(".git")).await.unwrap_or(false)
945    {
946        let Some(status) = git_in(wt, &["status", "--porcelain"]).await else {
947            tracing::warn!(task_id, branch = %br, "cannot read worktree status, keeping worktree");
948            return Rescue::Failed;
949        };
950        if !status.status.success() {
951            tracing::warn!(task_id, branch = %br, "git status failed, keeping worktree");
952            return Rescue::Failed;
953        }
954        if !status.stdout.is_empty() {
955            let committed = git_in(wt, &["add", "-A"])
956                .await
957                .is_some_and(|o| o.status.success())
958                && git_in(
959                    wt,
960                    &[
961                        "-c",
962                        "user.name=orch",
963                        "-c",
964                        "user.email=orch@localhost",
965                        "commit",
966                        "-m",
967                        "rescue: uncommitted changes saved by orch cleanup",
968                    ],
969                )
970                .await
971                .is_some_and(|o| o.status.success());
972            if !committed {
973                tracing::warn!(task_id, branch = %br, "cannot commit uncommitted changes, keeping worktree");
974                return Rescue::Failed;
975            }
976        }
977    }
978
979    let local_ref = format!("refs/heads/{br}");
980    let exists = git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
981        .await
982        .is_some_and(|o| o.status.success());
983    if !exists {
984        return Rescue::Clean;
985    }
986    // Commits on the local default branch survive the branch delete, so they do not count
987    let default_ref = format!(
988        "refs/heads/{}",
989        worktree::detect_default_branch(repo_root).await
990    );
991    // A bare clone has no remote-tracking refs, so ask the remote for the branch tip
992    let remote_tip = git_in(repo_root, &["ls-remote", "origin", &local_ref])
993        .await
994        .filter(|o| o.status.success())
995        .and_then(|o| {
996            String::from_utf8_lossy(&o.stdout)
997                .split_whitespace()
998                .next()
999                .map(str::to_string)
1000        });
1001    // A tip someone else pushed may be missing locally, and rev-list fails on an unknown object
1002    let remote_tip = match remote_tip {
1003        Some(tip) => {
1004            let known = git_in(repo_root, &["cat-file", "-e", &format!("{tip}^{{commit}}")])
1005                .await
1006                .is_some_and(|o| o.status.success());
1007            known.then_some(tip)
1008        }
1009        None => None,
1010    };
1011    let mut args = vec!["rev-list", "--oneline", &local_ref, "--not", "--remotes"];
1012    if let Some(tip) = remote_tip.as_deref() {
1013        args.push(tip);
1014    }
1015    let default_exists = default_ref != local_ref
1016        && git_in(
1017            repo_root,
1018            &["rev-parse", "--verify", "--quiet", &default_ref],
1019        )
1020        .await
1021        .is_some_and(|o| o.status.success());
1022    if default_exists {
1023        args.push(&default_ref);
1024    }
1025    let Some(list) = git_in(repo_root, &args)
1026        .await
1027        .filter(|o| o.status.success())
1028    else {
1029        tracing::warn!(task_id, branch = %br, "cannot list unpushed commits, keeping worktree and branch");
1030        return Rescue::Failed;
1031    };
1032    let commits: Vec<String> = String::from_utf8_lossy(&list.stdout)
1033        .lines()
1034        .map(str::to_string)
1035        .collect();
1036    if commits.is_empty() {
1037        return Rescue::Clean;
1038    }
1039
1040    let rescue = format!("rescue/{br}");
1041    let refspec = format!("{local_ref}:refs/heads/{rescue}");
1042    let pushed = git_in(
1043        repo_root,
1044        &["-c", "lfs.locksverify=false", "push", "origin", &refspec],
1045    )
1046    .await;
1047    if pushed.is_some_and(|o| o.status.success()) {
1048        // A bare clone has no fetchspec, so push does not create the tracking ref
1049        // Point that ref at the tip we just sent so delete_local_branch can see it
1050        let tracking = format!("refs/remotes/origin/{rescue}");
1051        if let Some(tip) = git_in(repo_root, &["rev-parse", &local_ref])
1052            .await
1053            .filter(|o| o.status.success())
1054        {
1055            let sha = String::from_utf8_lossy(&tip.stdout).trim().to_string();
1056            if !sha.is_empty() {
1057                let _ = git_in(repo_root, &["update-ref", &tracking, &sha]).await;
1058            }
1059        }
1060        tracing::warn!(task_id, branch = %br, rescue = %rescue, commits = commits.len(), "pushed unpushed commits to rescue branch");
1061        Rescue::Pushed {
1062            branch: rescue,
1063            commits,
1064        }
1065    } else {
1066        tracing::warn!(task_id, branch = %br, rescue = %rescue, "rescue push failed, keeping worktree and branch");
1067        Rescue::Failed
1068    }
1069}
1070
1071fn rescue_comment(rescue_branch: &str, commits: &[String]) -> String {
1072    let list = commits
1073        .iter()
1074        .map(|c| format!("- `{c}`"))
1075        .collect::<Vec<_>>()
1076        .join("\n");
1077    format!(
1078        "Cleanup found commits on this task's branch that were never pushed. They are saved on `{rescue_branch}`:\n\n{list}"
1079    )
1080}
1081
1082/// Save unpushed work and tell the task about it. Returns false when cleanup must not go on
1083async fn rescue_and_notify(
1084    task_id: &str,
1085    repo: &str,
1086    external_id: Option<&str>,
1087    wt: Option<&Path>,
1088    br: &str,
1089    repo_root: &Path,
1090) -> bool {
1091    match rescue_unpushed_work(task_id, wt, br, repo_root).await {
1092        Rescue::Clean => true,
1093        Rescue::Failed => false,
1094        Rescue::Pushed { branch, commits } => {
1095            if let Some(number) = external_id.filter(|n| n.parse::<u64>().is_ok()) {
1096                match GhHttp::new() {
1097                    Ok(gh) => {
1098                        if let Err(e) = gh
1099                            .add_comment(repo, number, &rescue_comment(&branch, &commits))
1100                            .await
1101                        {
1102                            tracing::warn!(task_id, err = %e, "failed to comment on rescue branch");
1103                        }
1104                    }
1105                    Err(e) => {
1106                        tracing::warn!(task_id, err = %e, "cannot comment on rescue branch");
1107                    }
1108                }
1109            }
1110            true
1111        }
1112    }
1113}
1114
1115/// Remove a git worktree directory and its local + remote branches.
1116/// When `keep_remote_branch` is true, only the worktree and local branch are
1117/// removed — the remote branch stays so that any linked PR remains open
1118/// (used for blocked tasks awaiting human review).
1119/// Returns `true` if the worktree directory is gone after the call
1120/// (either we removed it or it was already absent).
1121pub(crate) async fn remove_worktree_and_branch(
1122    task_id: &str,
1123    wt: &Path,
1124    branch: Option<&str>,
1125    repo_root: &Path,
1126    keep_remote_branch: bool,
1127) -> bool {
1128    remove_worktree_and_branch_for(task_id, wt, branch, repo_root, keep_remote_branch, false).await
1129}
1130
1131/// Same as `remove_worktree_and_branch`, but a merged PR skips the rescue. A squash leaves the
1132/// branch commits on no ref, so the rescue would push a branch for work that already landed
1133pub(crate) async fn remove_worktree_and_branch_for(
1134    task_id: &str,
1135    wt: &Path,
1136    branch: Option<&str>,
1137    repo_root: &Path,
1138    keep_remote_branch: bool,
1139    pr_merged: bool,
1140) -> bool {
1141    if !pr_merged
1142        && let Some(br) = branch.filter(|b| !b.is_empty())
1143        && rescue_unpushed_work(task_id, Some(wt), br, repo_root).await == Rescue::Failed
1144    {
1145        return false;
1146    }
1147
1148    remove_worktree_and_branch_unchecked(task_id, wt, branch, repo_root, keep_remote_branch).await
1149}
1150
1151/// The caller has already rescued the branch or knows its PR merged
1152async fn remove_worktree_and_branch_unchecked(
1153    task_id: &str,
1154    wt: &Path,
1155    branch: Option<&str>,
1156    repo_root: &Path,
1157    keep_remote_branch: bool,
1158) -> bool {
1159    let wt_str = wt.to_string_lossy().to_string();
1160    let repo_root_str = repo_root.to_string_lossy();
1161
1162    // Remove worktree FIRST, then delete the branch.
1163    // Git refuses to remove a worktree if its branch is already deleted.
1164    let remove_result = Command::new("git")
1165        .args([
1166            "-C",
1167            repo_root_str.as_ref(),
1168            "worktree",
1169            "remove",
1170            &wt_str,
1171            "--force",
1172        ])
1173        .output_with_context()
1174        .await;
1175
1176    match remove_result {
1177        Ok(output) if output.status.success() => {
1178            const MAX_RETRIES: usize = 3;
1179            const RETRY_DELAY_MS: u64 = 2000;
1180            tracing::info!(task_id, "worktree removed (git metadata)");
1181            // `git worktree remove` only removes git metadata — the physical directory
1182            // with working files may remain. Clean it up with retry logic, since macOS
1183            // sometimes needs a moment to release file handles from a recently-exited
1184            // process. Retrying with a delay addresses "Directory not empty" errors.
1185            let mut removed = false;
1186            for attempt in 0..MAX_RETRIES {
1187                match fs::remove_dir_all(wt).await {
1188                    Ok(()) => {
1189                        removed = true;
1190                        break;
1191                    }
1192                    Err(e) if e.kind() == ErrorKind::NotFound => {
1193                        // Directory was already removed by git or another cleanup pass.
1194                        removed = true;
1195                        break;
1196                    }
1197                    Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => {
1198                        if attempt < MAX_RETRIES - 1 {
1199                            tracing::debug!(
1200                                task_id,
1201                                path = %wt.display(),
1202                                attempt = attempt + 1,
1203                                "worktree directory not empty — retrying after delay"
1204                            );
1205                            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
1206                        } else {
1207                            tracing::warn!(
1208                                task_id,
1209                                path = %wt.display(),
1210                                err = %e,
1211                                "failed to remove worktree directory after {} attempts (Directory not empty)",
1212                                MAX_RETRIES
1213                            );
1214                        }
1215                    }
1216                    Err(e) => {
1217                        tracing::warn!(task_id, path = %wt.display(), err = %e, "failed to remove worktree directory");
1218                        break;
1219                    }
1220                }
1221            }
1222            if removed {
1223                tracing::info!(task_id, path = %wt.display(), "worktree physical directory removed");
1224            } else {
1225                tracing::warn!(
1226                    task_id,
1227                    path = %wt.display(),
1228                    "worktree directory still present after git worktree remove"
1229                );
1230            }
1231        }
1232        Ok(output) => {
1233            let stderr = String::from_utf8_lossy(&output.stderr);
1234            // "is not a working tree" means git's metadata is stale — the path
1235            // exists on disk but is no longer registered as a worktree. Treat
1236            // this as an already-gone worktree: physically remove the directory
1237            // and prune stale git metadata. Downgrade to debug — this is an
1238            // expected idempotency case, not a real error.
1239            if stderr.contains("is not a working tree") {
1240                tracing::debug!(
1241                    task_id,
1242                    path = %wt.display(),
1243                    "path is not a registered worktree (stale metadata) — removing directory and pruning"
1244                );
1245                if let Err(e) = fs::remove_dir_all(wt).await {
1246                    // Only warn if the directory actually still exists after removal attempt.
1247                    if match fs::try_exists(wt).await {
1248                        Ok(true) => true,
1249                        Ok(false) => false,
1250                        Err(e) => {
1251                            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree does not exist");
1252                            false
1253                        }
1254                    } {
1255                        tracing::warn!(
1256                            task_id,
1257                            path = %wt.display(),
1258                            err = %e,
1259                            "failed to remove stale worktree directory"
1260                        );
1261                    }
1262                } else {
1263                    tracing::debug!(task_id, path = %wt.display(), "stale worktree directory removed");
1264                }
1265                // Prune stale worktree entries from git metadata so the warning
1266                // stops recurring on future `git worktree list` / cleanup cycles.
1267                if let Err(e) = Command::new("git")
1268                    .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1269                    .output_with_context()
1270                    .await
1271                {
1272                    tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1273                }
1274            } else {
1275                tracing::warn!(task_id, err = %stderr, "failed to remove worktree");
1276            }
1277        }
1278        Err(e) => {
1279            tracing::warn!(task_id, err = %e, "failed to remove worktree");
1280        }
1281    }
1282
1283    // Prune stale .git/worktrees/ entries left behind after removal.
1284    if let Err(e) = Command::new("git")
1285        .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1286        .output_with_context()
1287        .await
1288    {
1289        tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1290    }
1291
1292    // Delete local and remote branch from the main repo root (worktree is already gone).
1293    // When keep_remote_branch is set, only delete the local branch — the remote stays
1294    // so any linked PR remains open for human review.
1295    if let Some(br) = branch {
1296        if keep_remote_branch {
1297            delete_local_branch(task_id, br, repo_root).await;
1298        } else {
1299            delete_branches(task_id, br, repo_root).await;
1300        }
1301    }
1302
1303    // Definitively check: is the worktree directory actually gone?
1304    match fs::try_exists(wt).await {
1305        Ok(false) => true, // confirmed gone
1306        Ok(true) => false, // still there
1307        Err(e) => {
1308            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree still present");
1309            false // conservative: don't claim success
1310        }
1311    }
1312}
1313
1314/// Whether `refs/heads/{br}` still exists in `repo_root`
1315pub(crate) async fn local_branch_exists(repo_root: &Path, br: &str) -> bool {
1316    let local_ref = format!("refs/heads/{br}");
1317    git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
1318        .await
1319        .is_some_and(|o| o.status.success())
1320}
1321
1322/// Commits reachable from `rev` and from none of `others`. `None` when git cannot answer
1323async fn commits_outside(repo_root: &Path, rev: &str, others: &[&str]) -> Option<u64> {
1324    let mut args = vec![
1325        "rev-list".to_string(),
1326        "--count".to_string(),
1327        rev.to_string(),
1328    ];
1329    // One `--not` then every other ref, a `--not` per ref would toggle and count the wrong set
1330    let mut exclude = Vec::new();
1331    for other in others {
1332        let exists = Command::new("git")
1333            .args([
1334                "-C",
1335                &repo_root.to_string_lossy(),
1336                "rev-parse",
1337                "--verify",
1338                "--quiet",
1339                other,
1340            ])
1341            .output_with_context()
1342            .await
1343            .is_ok_and(|o| o.status.success());
1344        if exists {
1345            exclude.push(other.to_string());
1346        }
1347    }
1348    if !exclude.is_empty() {
1349        args.push("--not".to_string());
1350        args.extend(exclude);
1351    }
1352    let output = Command::new("git")
1353        .arg("-C")
1354        .arg(repo_root)
1355        .args(&args)
1356        .output_with_context()
1357        .await
1358        .ok()?;
1359    if !output.status.success() {
1360        return None;
1361    }
1362    String::from_utf8_lossy(&output.stdout).trim().parse().ok()
1363}
1364
1365/// Delete the local branch only when origin or the default branch holds every commit on it
1366async fn delete_local_branch(task_id: &str, br: &str, repo_root: &Path) {
1367    let default_branch = worktree::detect_default_branch(repo_root).await;
1368    let remote_ref = format!("refs/remotes/origin/{br}");
1369    let default_ref = format!("refs/remotes/origin/{default_branch}");
1370    let local_ref = format!("refs/heads/{br}");
1371    let local_default_ref = format!("refs/heads/{default_branch}");
1372    let rescue_ref = format!("refs/remotes/origin/rescue/{br}");
1373    match commits_outside(
1374        repo_root,
1375        &local_ref,
1376        &[&remote_ref, &default_ref, &local_default_ref, &rescue_ref],
1377    )
1378    .await
1379    {
1380        Some(0) => {}
1381        Some(n) => {
1382            tracing::warn!(task_id, branch = %br, unpushed = n, "local branch holds commits origin does not have, keeping it");
1383            return;
1384        }
1385        None => {
1386            tracing::debug!(task_id, branch = %br, "local branch missing or unreadable, nothing to delete");
1387            return;
1388        }
1389    }
1390    let output = Command::new("git")
1391        .args(["-C", &repo_root.to_string_lossy(), "branch", "-D", br])
1392        .output_with_context()
1393        .await;
1394    match output {
1395        Ok(o) if o.status.success() => {
1396            tracing::info!(task_id, branch = %br, "local branch deleted");
1397        }
1398        Ok(o) => {
1399            let stderr = String::from_utf8_lossy(&o.stderr);
1400            tracing::debug!(task_id, err = %stderr, "local branch delete skipped (may not exist)");
1401        }
1402        Err(e) => {
1403            tracing::warn!(task_id, err = %e, "failed to delete local branch");
1404        }
1405    }
1406}
1407
1408/**
1409 * Delete local and remote branches for a task, but never the only copy of a commit.
1410 * The remote branch goes only when the default branch already contains all of it, which
1411 * covers fast-forward and merge-commit merges. A squash-merged branch stays, and GitHub's
1412 * "automatically delete head branches" removes it. Every other caller (startup rebase
1413 * failure, closed-issue reconciliation, orphan sweeps) may hold unmerged work (#3781)
1414 **/
1415pub(crate) async fn delete_branches(task_id: &str, br: &str, repo_root: &Path) {
1416    if br == SCREENSHOTS_BRANCH {
1417        return;
1418    }
1419    let default_branch = worktree::detect_default_branch(repo_root).await;
1420    let _ = git_ops::git()
1421        .await
1422        .args([
1423            "-C",
1424            &repo_root.to_string_lossy(),
1425            "fetch",
1426            "origin",
1427            &default_branch,
1428            br,
1429        ])
1430        .output_with_context()
1431        .await;
1432
1433    delete_local_branch(task_id, br, repo_root).await;
1434
1435    let remote_ref = format!("refs/remotes/origin/{br}");
1436    let default_ref = format!("refs/remotes/origin/{default_branch}");
1437    match commits_outside(repo_root, &remote_ref, &[&default_ref]).await {
1438        Some(0) => {}
1439        Some(n) => {
1440            tracing::info!(task_id, branch = %br, unmerged = n, "remote branch has commits the default branch lacks, keeping it");
1441            return;
1442        }
1443        None => {
1444            tracing::debug!(task_id, branch = %br, "remote branch missing or unreadable, nothing to delete");
1445            return;
1446        }
1447    }
1448
1449    let remote_delete = git_ops::git()
1450        .await
1451        .args([
1452            "-C",
1453            &repo_root.to_string_lossy(),
1454            "-c",
1455            "lfs.locksverify=false",
1456            "push",
1457            "origin",
1458            "--delete",
1459            br,
1460        ])
1461        .output_with_context()
1462        .await;
1463
1464    match remote_delete {
1465        Ok(output) if output.status.success() => {
1466            tracing::info!(task_id, branch = %br, "remote branch deleted");
1467            git_ops::clear_push_lease(repo_root, br).await;
1468        }
1469        Ok(output) => {
1470            let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
1471            tracing::debug!(task_id, err = %stderr, "remote branch delete skipped");
1472        }
1473        Err(e) => {
1474            tracing::warn!(task_id, err = %e, "failed to delete remote branch");
1475        }
1476    }
1477}
1478
1479/// Returns `true` if any active tmux pane has its cwd inside `worktree`.
1480///
1481/// Uses `tmux list-panes -a -F '#{pane_current_path}'` to enumerate all
1482/// pane working directories across every session. If tmux is not running
1483/// (command fails), we conservatively return `false` so cleanup can proceed.
1484async fn is_worktree_in_active_session(worktree: &Path) -> bool {
1485    let output = Command::new("tmux")
1486        .args(["list-panes", "-a", "-F", "#{pane_current_path}"])
1487        .output()
1488        .await;
1489
1490    match output {
1491        Ok(o) if o.status.success() => {
1492            let worktree_str = worktree.to_string_lossy();
1493            String::from_utf8_lossy(&o.stdout).lines().any(|line| {
1494                let pane_path = line.trim();
1495                // Match exact path or any path that starts with worktree + separator.
1496                pane_path == worktree_str.as_ref()
1497                    || pane_path.starts_with(&format!("{worktree_str}/"))
1498            })
1499        }
1500        // tmux not running or no sessions — treat as safe to clean.
1501        _ => false,
1502    }
1503}
1504
1505/// Returns how many hours old the worktree directory is, based on its mtime.
1506///
1507/// Returns `None` if the directory does not exist or the mtime cannot be read.
1508async fn worktree_age_hours(worktree: &Path) -> Option<u64> {
1509    // Use tokio::fs to avoid blocking the async runtime when reading metadata.
1510    let metadata = fs::metadata(worktree).await.ok()?;
1511    let modified = metadata.modified().ok()?;
1512    let age = SystemTime::now().duration_since(modified).ok()?;
1513    Some(age.as_secs() / 3600)
1514}
1515
1516/// Resolve the repo root for an orphaned worktree and remove it.
1517///
1518/// Used by `orch service doctor --fix` and `orch service prune` to clean up worktrees that are
1519/// no longer owned by any task. Returns `true` if the worktree directory is gone
1520/// after the call.
1521pub(crate) async fn cleanup_orphaned_worktree(task_id: &str, wt: &Path) -> bool {
1522    let repo_root = match resolve_repo_root_for_orphaned_worktree(wt).await {
1523        Ok(root) => root,
1524        Err(e) => {
1525            tracing::warn!(
1526                worktree = %wt.display(),
1527                error = %e,
1528                "cannot resolve repo root for orphaned worktree"
1529            );
1530            return false;
1531        }
1532    };
1533    remove_worktree_and_branch(task_id, wt, None, Path::new(&repo_root), false).await
1534}
1535
1536/// Resolve the main git repository root for an orphaned worktree.
1537///
1538/// First tries to parse the `.git` file's `gitdir:` pointer (two levels up from
1539/// the pointer gives the bare repo root, which is the orch convention). Falls back
1540/// to looking up registered projects by directory name when the `.git` file is
1541/// absent or unparseable.
1542pub(crate) async fn resolve_repo_root_for_orphaned_worktree(wt: &Path) -> anyhow::Result<String> {
1543    let git_file = wt.join(".git");
1544    if fs::try_exists(&git_file).await.unwrap_or(false)
1545        && let Ok(content) = fs::read_to_string(&git_file).await
1546    {
1547        for line in content.lines() {
1548            if let Some(gitdir) = line.strip_prefix("gitdir: ") {
1549                // gitdir points to: <repo>/.git/worktrees/<name> (non-bare)
1550                //                or <repo>.git/worktrees/<name>   (bare)
1551                // Two levels up reaches the .git dir / bare repo root.
1552                let gitdir_path = PathBuf::from(gitdir);
1553                if let Some(repo_root) = gitdir_path.parent().and_then(|p| p.parent()) {
1554                    return Ok(repo_root.display().to_string());
1555                }
1556            }
1557        }
1558    }
1559
1560    // Fallback: look up registered projects by the project-name component of the
1561    // worktree path (format: ~/.orch/worktrees/<project>/<branch>).
1562    let worktrees_dir = home::worktrees_dir()?;
1563    if let Ok(relative) = wt.strip_prefix(&worktrees_dir)
1564        && let Some(project_name) = relative.components().next()
1565    {
1566        let project_name = project_name.as_os_str().to_string_lossy();
1567
1568        if let Ok(paths) = config::get_project_paths() {
1569            for path_str in &paths {
1570                let path = Path::new(path_str);
1571                if let Some(name) = path.file_name()
1572                    && name.to_string_lossy() == project_name
1573                {
1574                    return Ok(path_str.clone());
1575                }
1576            }
1577        }
1578
1579        // Try bare clone at ~/.orch/projects/<owner>/<project>.git
1580        let parts: Vec<&str> = project_name.split('/').collect();
1581        let bare = if parts.len() == 2 {
1582            home::projects_dir()
1583                .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1584                .unwrap_or_default()
1585        } else {
1586            return Err(anyhow::anyhow!(
1587                "cannot determine owner for single-segment project name '{project_name}' — \
1588                     bare repo location is ambiguous; register the project with 'owner/repo' format"
1589            ));
1590        };
1591        if fs::try_exists(&bare).await.unwrap_or(false) {
1592            return Ok(bare.display().to_string());
1593        }
1594    }
1595
1596    anyhow::bail!(
1597        "cannot resolve repo root for orphaned worktree at {} — tried .git parsing and project lookup",
1598        wt.display()
1599    )
1600}
1601
1602/// Resolve the main git repository root from a worktree's .git file.
1603///
1604/// For cross-project worktree cleanup, we cannot rely on `resolve_repo_root` because
1605/// the worktree may belong to a different project than the current repo context.
1606/// Instead, we extract the repo root from the worktree's `.git` file which contains
1607/// a `gitdir:` pointer to the main repo's git directory.
1608pub async fn resolve_repo_root_from_worktree(wt: &Path) -> anyhow::Result<String> {
1609    // For worktrees, .git is a file (not a directory) containing gitdir path
1610    let git_file = wt.join(".git");
1611    if !fs::try_exists(&git_file).await.unwrap_or(false) {
1612        anyhow::bail!(".git file not found in worktree at {}", wt.display());
1613    }
1614
1615    let content = fs::read_to_string(&git_file)
1616        .await
1617        .map_err(|e| anyhow::anyhow!("failed to read .git file: {e}"))?;
1618
1619    // Parse gitdir line: "gitdir: /path/to/repo/.git/worktrees/<name>"
1620    for line in content.lines() {
1621        if let Some(gitdir) = line.strip_prefix("gitdir:") {
1622            let gitdir = gitdir.trim();
1623            // The main repo is the parent of the .git directory
1624            // gitdir points to: <repo>/.git/worktrees/<name>
1625            // We need to go up 3 levels: worktrees/<name> -> .git -> repo
1626            let gitdir_path = Path::new(gitdir);
1627            if let Some(repo_root) = gitdir_path
1628                .parent() // worktrees/<name>
1629                .and_then(|p| p.parent()) // .git
1630                .and_then(|p| p.parent())
1631            // repo root
1632            {
1633                return Ok(repo_root.display().to_string());
1634            }
1635        }
1636    }
1637
1638    anyhow::bail!(
1639        "cannot resolve repo root from worktree .git file at {}",
1640        wt.display()
1641    )
1642}
1643
1644/// Resolve the main git repository root path for a project.
1645///
1646/// Looks up the local project path from config, then verifies it's a git repo.
1647/// This avoids relying on cwd (which is undefined under launchd services).
1648pub(crate) async fn resolve_repo_root(repo: &str) -> anyhow::Result<String> {
1649    // Look up the local path from registered projects
1650    let paths = config::get_project_paths().unwrap_or_default();
1651    for path_str in &paths {
1652        let path = Path::new(path_str);
1653        let orch_yml = path.join(".orch.yml");
1654        let legacy = path.join(".orchestrator.yml");
1655        let config_file = if fs::try_exists(&orch_yml).await.unwrap_or(false) {
1656            orch_yml
1657        } else if fs::try_exists(&legacy).await.unwrap_or(false) {
1658            legacy
1659        } else {
1660            continue;
1661        };
1662        if let Ok(content) = fs::read_to_string(&config_file).await
1663            && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1664            && let Some(r) = doc
1665                .get("gh")
1666                .and_then(|gh| gh.get("repo"))
1667                .and_then(|r| r.as_str())
1668            && r == repo
1669        {
1670            return Ok(path_str.clone());
1671        }
1672    }
1673
1674    // Fallback: try bare clone in ~/.orch/projects/<owner>/<repo>.git
1675    let parts: Vec<&str> = repo.split('/').collect();
1676    let bare = if parts.len() == 2 {
1677        home::projects_dir()
1678            .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1679            .unwrap_or_default()
1680    } else {
1681        PathBuf::new()
1682    };
1683    if fs::try_exists(&bare).await.unwrap_or(false) {
1684        return Ok(bare.display().to_string());
1685    }
1686
1687    anyhow::bail!(
1688        "cannot find local path for project {repo} — \
1689         checked {} registered project(s) and bare clone at {}",
1690        paths.len(),
1691        bare.display()
1692    )
1693}
1694
1695/// Resolve the project directory for a given repo string.
1696///
1697/// Replicates the logic from `TaskRunner::resolve_project_dir()` without requiring
1698/// a Runner instance. Used by the orphan scanner to enumerate active worktrees.
1699fn resolve_project_dir_for_repo(repo: &str) -> anyhow::Result<PathBuf> {
1700    // Check PROJECT_DIR env var first
1701    if let Ok(dir) = env::var("PROJECT_DIR")
1702        && !dir.is_empty()
1703    {
1704        return Ok(PathBuf::from(dir));
1705    }
1706
1707    // Look up registered projects in config
1708    if let Ok(paths) = config::get_project_paths() {
1709        for path_str in &paths {
1710            let path = Path::new(path_str);
1711            let orch_yml = path.join(".orch.yml");
1712            let legacy = path.join(".orchestrator.yml");
1713            let config_file = if Path::new(&orch_yml).exists() {
1714                orch_yml
1715            } else if Path::new(&legacy).exists() {
1716                legacy
1717            } else {
1718                continue;
1719            };
1720            if let Ok(content) = std_fs::read_to_string(&config_file)
1721                && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1722                && let Some(r) = doc
1723                    .get("gh")
1724                    .and_then(|gh| gh.get("repo"))
1725                    .and_then(|r| r.as_str())
1726                && r == repo
1727            {
1728                return Ok(path_str.clone().into());
1729            }
1730        }
1731    }
1732
1733    anyhow::bail!("project dir not found for repo: {repo}")
1734}
1735
1736/// Check for merged PRs and update task status accordingly.
1737///
1738/// Queries status:in_review and status:needs_review tasks, checks if their PR
1739/// is merged, and updates status to done if merged.
1740pub(crate) async fn check_merged_prs(
1741    backend: &Arc<dyn ExternalBackend>,
1742    repo: &str,
1743    store: &Arc<TaskStore>,
1744    task_manager: &Arc<TaskManager>,
1745    in_review_tasks: &[ReviewTaskSnapshot],
1746    needs_review_tasks: &[ReviewTaskSnapshot],
1747) -> anyhow::Result<()> {
1748    let mut branch_map: HashMap<String, String> = HashMap::new();
1749    let all_review_tasks: Vec<&ReviewTaskSnapshot> = in_review_tasks
1750        .iter()
1751        .chain(needs_review_tasks.iter())
1752        .collect();
1753
1754    for task in &all_review_tasks {
1755        if !task.stored.branch.is_empty() {
1756            branch_map.insert(task.external.id.0.clone(), task.stored.branch.clone());
1757        }
1758    }
1759
1760    tracing::debug!(
1761        count = all_review_tasks.len(),
1762        "checking review tasks for merged PRs"
1763    );
1764
1765    // Collect (task_id, branch) pairs — skip tasks with no branch recorded.
1766    // Use the pre-built branch map; fall back to store lookup only for tasks
1767    // not in the map (e.g. internal tasks, backend fallback path).
1768    let mut task_branches: Vec<(String, String)> = Vec::new();
1769    for task in all_review_tasks {
1770        let task_id = &task.external.id.0;
1771        let branch = if let Some(b) = branch_map.get(task_id) {
1772            b.clone()
1773        } else {
1774            match store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
1775                .await
1776                .map(|t| t.branch)
1777            {
1778                Some(b) if !b.is_empty() => b,
1779                _ => {
1780                    tracing::debug!(task_id, "no branch info, skipping PR check");
1781                    continue;
1782                }
1783            }
1784        };
1785        task_branches.push((task_id.clone(), branch));
1786    }
1787
1788    if task_branches.is_empty() {
1789        return Ok(());
1790    }
1791
1792    // Single GraphQL call for all branches — N REST calls → 1 GraphQL call.
1793    let branches: Vec<String> = task_branches.iter().map(|(_, b)| b.clone()).collect();
1794    let merged_map = match backend.batch_is_pr_merged(&branches).await {
1795        Ok(map) => map,
1796        Err(e) => {
1797            tracing::warn!(
1798                err = %e,
1799                branch_count = branches.len(),
1800                "batch PR merge check failed, retrying affected branches individually"
1801            );
1802            fallback_is_pr_merged_by_branch(backend, &branches).await
1803        }
1804    };
1805
1806    for (task_id, branch) in task_branches {
1807        let is_merged = merged_map.get(&branch).copied().unwrap_or(false);
1808        if !is_merged {
1809            continue;
1810        }
1811
1812        tracing::info!(task_id, branch = %branch, "PR merged, marking task complete");
1813
1814        let stored = store::opt_store_get_task(&Some(Arc::clone(store)), repo, &task_id).await;
1815        if let Some(ref stored) = stored {
1816            let external = store_task_to_external(stored);
1817            if ci_bootstrap::hold_after_merge(&external, store, repo).await {
1818                tracing::info!(
1819                    task_id,
1820                    branch = %branch,
1821                    "CI bootstrap PR merged, holding done until the ruleset exists"
1822                );
1823                if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1824                    tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1825                }
1826                continue;
1827            }
1828        }
1829
1830        let id = ExternalId(task_id.clone());
1831        if let Err(e) = task_manager.update_task_status(&id, Status::Done).await {
1832            tracing::warn!(task_id, err = %e, "failed to update status to done");
1833            continue;
1834        }
1835
1836        if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1837            tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1838        }
1839
1840        let comment = format!("PR merged, marking task complete{}", engine::orch_footer());
1841        if let Err(e) = backend.post_comment(&id, &comment).await {
1842            tracing::warn!(task_id, err = %e, "failed to post comment");
1843        }
1844    }
1845
1846    Ok(())
1847}
1848
1849async fn fallback_is_pr_merged_by_branch(
1850    backend: &Arc<dyn ExternalBackend>,
1851    branches: &[String],
1852) -> HashMap<String, bool> {
1853    let futs: Vec<_> = branches
1854        .iter()
1855        .map(|branch| {
1856            let backend = Arc::clone(backend);
1857            let branch = branch.clone();
1858            async move {
1859                match backend.is_pr_merged(&branch).await {
1860                    Ok(is_merged) => Some((branch, is_merged)),
1861                    Err(err) => {
1862                        tracing::warn!(
1863                            branch,
1864                            err = %err,
1865                            "fallback PR merge check failed for branch"
1866                        );
1867                        None
1868                    }
1869                }
1870            }
1871        })
1872        .collect();
1873
1874    future::join_all(futs).await.into_iter().flatten().collect()
1875}
1876
1877#[cfg(test)]
1878mod tests {
1879    use super::*;
1880    use crate::engine::tasks;
1881    use crate::store::{NewTask, TaskStore};
1882    use async_trait::async_trait;
1883    use std::collections::HashMap;
1884    use std::process;
1885    use std::sync::Mutex;
1886    use std::sync::atomic::{AtomicUsize, Ordering};
1887
1888    // worktree_age_hours
1889
1890    #[tokio::test]
1891    async fn worktree_age_hours_returns_none_for_missing_dir() {
1892        let tmp = tempfile::tempdir().unwrap();
1893        let missing = tmp.path().join("does-not-exist");
1894        assert!(worktree_age_hours(&missing).await.is_none());
1895    }
1896
1897    #[tokio::test]
1898    async fn worktree_age_hours_returns_some_for_existing_dir() {
1899        let tmp = tempfile::tempdir().unwrap();
1900        // A freshly created directory should have age ~0 hours.
1901        let age = worktree_age_hours(tmp.path()).await;
1902        assert!(age.is_some());
1903        assert_eq!(age.unwrap(), 0, "freshly created dir should have age 0h");
1904    }
1905
1906    // JanitorOptions::from_config defaults
1907
1908    #[test]
1909    fn janitor_options_default_values() {
1910        let opts = JanitorOptions::default();
1911        assert_eq!(opts.ttl_hours, 24);
1912        assert!(!opts.dry_run);
1913    }
1914
1915    // cleanup_task_worktree_with_opts TTL guard (skips if git missing)
1916
1917    /// Create a minimal git repo + worktree; return (tmp, worktree_dir).
1918    fn setup_test_repo() -> Option<(tempfile::TempDir, PathBuf)> {
1919        let tmp = tempfile::tempdir().ok()?;
1920
1921        // Create a real git repo (not bare) so we can add worktrees.
1922        let repo_dir = tmp.path().join("repo.git");
1923        std_fs::create_dir_all(&repo_dir).ok()?;
1924
1925        let init = process::Command::new("git")
1926            .args([
1927                "init",
1928                "--initial-branch=main",
1929                repo_dir
1930                    .to_str()
1931                    .expect("test repo path contains non-UTF-8 characters"),
1932            ])
1933            .output()
1934            .ok()?;
1935        if !init.status.success() {
1936            // Older git may not have --initial-branch; fall back.
1937            let _ = process::Command::new("git")
1938                .args([
1939                    "init",
1940                    repo_dir
1941                        .to_str()
1942                        .expect("test repo path contains non-UTF-8 characters"),
1943                ])
1944                .output()
1945                .ok()?;
1946        }
1947
1948        // Create an initial commit so the repo has a HEAD.
1949        let readme = repo_dir.join("README.md");
1950        std_fs::write(&readme, "# test").ok()?;
1951        process::Command::new("git")
1952            .args(["-C", repo_dir.to_str()?, "add", "."])
1953            .output()
1954            .ok()?;
1955        process::Command::new("git")
1956            .args([
1957                "-C",
1958                repo_dir.to_str()?,
1959                "-c",
1960                "user.email=test@test.com",
1961                "-c",
1962                "user.name=Test",
1963                "-c",
1964                "commit.gpgsign=false",
1965                "commit",
1966                "--allow-empty",
1967                "-m",
1968                "init",
1969            ])
1970            .output()
1971            .ok()?;
1972
1973        // Create a worktree branch.
1974        let wt_dir = tmp
1975            .path()
1976            .join("worktrees")
1977            .join("repo.git")
1978            .join("gh-issue-42-test");
1979        std_fs::create_dir_all(wt_dir.parent()?).ok()?;
1980        let wt_out = process::Command::new("git")
1981            .args([
1982                "-C",
1983                repo_dir.to_str()?,
1984                "worktree",
1985                "add",
1986                "-b",
1987                "gh-issue-42-test",
1988                wt_dir.to_str()?,
1989            ])
1990            .output()
1991            .ok()?;
1992
1993        if !wt_out.status.success() {
1994            return None;
1995        }
1996
1997        Some((tmp, wt_dir))
1998    }
1999
2000    #[tokio::test]
2001    async fn janitor_skips_young_worktree() {
2002        let Some((tmp, wt_dir)) = setup_test_repo() else {
2003            eprintln!("skipping test: git not available");
2004            return;
2005        };
2006
2007        // The worktree was just created → age is 0h < ttl_hours=24 → must be skipped.
2008        assert!(wt_dir.exists(), "worktree should exist before janitor runs");
2009
2010        // Run the TTL guard check directly (without a full engine).
2011        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2012        assert!(
2013            age < 24,
2014            "freshly created worktree should be younger than 24h (age={age}h)"
2015        );
2016
2017        // Explicitly verify: with ttl_hours=24 the worktree stays.
2018        let opts = JanitorOptions {
2019            ttl_hours: 24,
2020            dry_run: true,
2021        };
2022        // Age < ttl → should skip (worktree_age_hours < opts.ttl_hours).
2023        assert!(age < opts.ttl_hours, "janitor should skip young worktree");
2024
2025        drop(tmp); // cleanup tempdir
2026    }
2027
2028    #[tokio::test]
2029    async fn janitor_eligible_when_ttl_zero() {
2030        let Some((tmp, wt_dir)) = setup_test_repo() else {
2031            eprintln!("skipping test: git not available");
2032            return;
2033        };
2034
2035        // With ttl_hours=0 even a brand-new worktree is eligible.
2036        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2037        let opts = JanitorOptions {
2038            ttl_hours: 0,
2039            dry_run: true,
2040        };
2041        assert!(
2042            age >= opts.ttl_hours,
2043            "worktree should be eligible when ttl=0 (age={age}h)"
2044        );
2045
2046        drop(tmp);
2047    }
2048
2049    #[tokio::test]
2050    async fn dry_run_does_not_remove_worktree() {
2051        // Verify that with dry_run=true the worktree directory is NOT removed.
2052        let Some((tmp, wt_dir)) = setup_test_repo() else {
2053            eprintln!("skipping test: git not available");
2054            return;
2055        };
2056
2057        assert!(wt_dir.exists(), "worktree must exist before test");
2058
2059        // With dry_run=true and ttl_hours=0 the janitor would normally remove the
2060        // worktree, but dry_run prevents it.
2061        let opts = JanitorOptions {
2062            ttl_hours: 0,
2063            dry_run: true,
2064        };
2065
2066        // The janitor needs a repo_root to run git commands.  We point it at the
2067        // bare repo we created inside tmp (even though it's not registered in config).
2068        // Because resolve_repo_root will fail for our synthetic repo slug, we test the
2069        // inner helpers directly here.
2070        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2071        let in_use = false; // no tmux running in tests
2072        let skip = age < opts.ttl_hours || in_use;
2073
2074        if !skip && opts.dry_run {
2075            // dry-run: log only, don't touch filesystem.
2076            tracing::debug!("[dry-run] would remove {}", wt_dir.display());
2077        } else if !skip {
2078            // This branch would do actual removal — but we're in dry_run mode above.
2079        }
2080
2081        // Worktree must still be there.
2082        assert!(
2083            wt_dir.exists(),
2084            "dry-run must not remove the worktree directory"
2085        );
2086
2087        drop(tmp);
2088    }
2089
2090    #[test]
2091    fn fallback_branch_path_construction() {
2092        // Verify the new fallback logic: when worktree path is absent but
2093        // branch is known, we should construct the path correctly.
2094        let tmp = tempfile::tempdir().unwrap();
2095        let wt_dir = tmp.path().join("myrepo").join("gh-issue-7-fix");
2096        std_fs::create_dir_all(&wt_dir).unwrap();
2097
2098        // Simulate: worktree=None, branch=Some("gh-issue-7-fix"), repo="owner/myrepo"
2099        let branch = "gh-issue-7-fix";
2100        let repo = "owner/myrepo";
2101        let worktrees_base = tmp.path();
2102
2103        let project = repo
2104            .rsplit('/')
2105            .next()
2106            .unwrap_or(repo)
2107            .trim_end_matches(".git");
2108        let candidate = worktrees_base.join(project).join(branch);
2109
2110        assert_eq!(candidate, wt_dir);
2111        assert!(
2112            candidate.exists(),
2113            "fallback path should point to existing dir"
2114        );
2115    }
2116
2117    /// A task with no worktree and no branch should return Ok(false) —
2118    /// nothing was cleaned, so git pull should NOT be triggered.
2119    #[tokio::test]
2120    async fn cleanup_returns_false_when_nothing_to_clean() {
2121        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2122
2123        // Create a done task with no worktree or branch fields set
2124        let id = store
2125            .create(&NewTask {
2126                external_id: Some("999".to_string()),
2127                repo: "owner/repo".to_string(),
2128                origin: "github".to_string(),
2129                title: "Already cleaned".to_string(),
2130                ..Default::default()
2131            })
2132            .await
2133            .unwrap();
2134        store.update_status(id, TaskStatus::Done).await.unwrap();
2135
2136        let opts = JanitorOptions {
2137            ttl_hours: 0,
2138            dry_run: false,
2139        };
2140
2141        let result = cleanup_task_worktree_with_opts("999", "owner/repo", &store, &opts).await;
2142        // resolve_repo_root will fail for synthetic repo — that's OK, the point is
2143        // that when nothing needs cleaning the function should not report true.
2144        // In production this means no git pull is triggered.
2145        if let Ok(cleaned) = result {
2146            assert!(
2147                !cleaned,
2148                "cleanup should return false when there is no worktree or branch to clean"
2149            );
2150        } else {
2151            // resolve_repo_root fails for test repo — acceptable.
2152            // The bug was that even with no worktree/branch, Ok(true) was returned.
2153        }
2154    }
2155
2156    /// A done task with an empty worktree and branch must not remove the project
2157    /// worktree directory or the sibling worktrees under it (#4111).
2158    #[tokio::test]
2159    async fn cleanup_keeps_project_dir_for_a_task_with_empty_branch() {
2160        let project = format!("empty-branch-{}", process::id());
2161        let project_dir = home::worktrees_dir().unwrap().join(&project);
2162        let sibling = project_dir.join("sibling-task");
2163        std_fs::create_dir_all(&sibling).unwrap();
2164        let repo = format!("owner/{project}");
2165
2166        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2167        let id = store
2168            .create(&NewTask {
2169                external_id: Some("4111".to_string()),
2170                repo: repo.clone(),
2171                origin: "github".to_string(),
2172                title: "Empty branch".to_string(),
2173                ..Default::default()
2174            })
2175            .await
2176            .unwrap();
2177        store.update_status(id, TaskStatus::Done).await.unwrap();
2178
2179        for ttl_hours in [0, 24] {
2180            let opts = JanitorOptions {
2181                ttl_hours,
2182                dry_run: false,
2183            };
2184            let _ = cleanup_task_worktree_with_opts("4111", &repo, &store, &opts).await;
2185            assert!(sibling.exists(), "sibling worktree must survive");
2186        }
2187
2188        std_fs::remove_dir_all(&project_dir).unwrap();
2189    }
2190
2191    /// When the stored worktree path no longer exists on disk and there is no
2192    /// branch recorded, cleanup must mark the task as cleaned so subsequent
2193    /// runs do not keep retrying (idempotency regression test for #1021).
2194    #[tokio::test]
2195    async fn cleanup_marks_cleaned_when_stored_path_is_gone() {
2196        let tmp = tempfile::tempdir().unwrap();
2197        // A path that does NOT exist on disk.
2198        let stale_wt = tmp.path().join("stale-worktree");
2199        assert!(!stale_wt.exists(), "stale path must not exist");
2200
2201        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2202
2203        let id = store
2204            .create(&NewTask {
2205                external_id: Some("1005".to_string()),
2206                repo: "owner/repo".to_string(),
2207                origin: "github".to_string(),
2208                title: "Stale worktree task".to_string(),
2209                ..Default::default()
2210            })
2211            .await
2212            .unwrap();
2213        store.update_status(id, TaskStatus::Done).await.unwrap();
2214        // Set the stale worktree path on the task.
2215        store
2216            .set_fields(
2217                id,
2218                &[("worktree", stale_wt.to_string_lossy().as_ref().into())],
2219            )
2220            .await
2221            .unwrap();
2222
2223        // Verify the task starts out NOT cleaned.
2224        let before = store.get(id).await.unwrap();
2225        assert!(!before.worktree_cleaned, "task should not be cleaned yet");
2226        assert!(
2227            !before.worktree.is_empty(),
2228            "task must have a worktree path set"
2229        );
2230
2231        let opts = JanitorOptions {
2232            ttl_hours: 0,
2233            dry_run: false,
2234        };
2235
2236        let result = cleanup_task_worktree_with_opts("1005", "owner/repo", &store, &opts).await;
2237
2238        // The function should succeed (Ok) — stale path is not an error.
2239        // It returns Ok(false) because there was nothing to actively remove.
2240        match result {
2241            Ok(_) => {}
2242            Err(e) => panic!("cleanup should not error for stale path: {e}"),
2243        }
2244
2245        // The task must now be marked cleaned so the next cycle skips it.
2246        let after = store.get(id).await.unwrap();
2247        assert!(
2248            after.worktree_cleaned,
2249            "task must be marked cleaned when stored worktree path is already gone"
2250        );
2251    }
2252
2253    /// `remove_worktree_and_branch` returns true when the worktree directory
2254    /// is successfully removed (regression test for #1143).
2255    #[tokio::test]
2256    async fn remove_worktree_returns_true_on_success() {
2257        let Some((tmp, wt_dir)) = setup_test_repo() else {
2258            eprintln!("skipping test: git not available");
2259            return;
2260        };
2261
2262        let repo_dir = tmp.path().join("repo.git");
2263        assert!(wt_dir.exists(), "worktree must exist before removal");
2264
2265        let removed =
2266            remove_worktree_and_branch("42", &wt_dir, Some("gh-issue-42-test"), &repo_dir, false)
2267                .await;
2268
2269        assert!(
2270            removed,
2271            "remove_worktree_and_branch should return true when directory is gone"
2272        );
2273        assert!(!wt_dir.exists(), "worktree directory should be removed");
2274    }
2275
2276    /// Successful worktree cleanup must still delete the local branch.
2277    /// This guards against early-return regressions in `remove_worktree_and_branch`.
2278    #[tokio::test]
2279    async fn remove_worktree_success_still_deletes_local_branch() {
2280        let Some((tmp, wt_dir)) = setup_test_repo() else {
2281            eprintln!("skipping test: git not available");
2282            return;
2283        };
2284
2285        let repo_dir = tmp.path().join("repo.git");
2286        let repo_dir_str = repo_dir
2287            .to_str()
2288            .expect("test repo path contains non-UTF-8 characters");
2289        let branch = "gh-issue-42-test";
2290
2291        let before = process::Command::new("git")
2292            .args([
2293                "-C",
2294                repo_dir_str,
2295                "show-ref",
2296                "--verify",
2297                "--quiet",
2298                &format!("refs/heads/{branch}"),
2299            ])
2300            .status()
2301            .unwrap();
2302        assert!(
2303            before.success(),
2304            "test setup must create local branch before cleanup"
2305        );
2306
2307        let removed =
2308            remove_worktree_and_branch("42", &wt_dir, Some(branch), &repo_dir, false).await;
2309        assert!(removed, "worktree should be removed");
2310
2311        let after = process::Command::new("git")
2312            .args([
2313                "-C",
2314                repo_dir_str,
2315                "show-ref",
2316                "--verify",
2317                "--quiet",
2318                &format!("refs/heads/{branch}"),
2319            ])
2320            .status()
2321            .unwrap();
2322        assert!(
2323            !after.success(),
2324            "local branch should be deleted after successful worktree cleanup"
2325        );
2326    }
2327
2328    /// `remove_worktree_and_branch` returns false when the worktree directory
2329    /// cannot be removed (regression test for #1143 — prevents orphaned worktrees
2330    /// from being permanently marked as cleaned).
2331    #[tokio::test]
2332    async fn remove_worktree_returns_false_when_dir_persists() {
2333        let tmp = tempfile::tempdir().unwrap();
2334        // Create a plain directory (not a real git worktree) — git will fail
2335        // to remove it, so the directory stays on disk.
2336        let fake_wt = tmp.path().join("not-a-worktree");
2337        std_fs::create_dir_all(&fake_wt).unwrap();
2338
2339        // repo_root doesn't matter much — it just needs to exist for the -C flag.
2340        // Using tmp itself; git commands will fail but that's the point.
2341        let removed = remove_worktree_and_branch("99", &fake_wt, None, tmp.path(), false).await;
2342
2343        // The directory still exists because git couldn't remove it.
2344        // Before the fix, this would still have been marked as cleaned.
2345        assert!(
2346            !removed,
2347            "remove_worktree_and_branch should return false when directory still exists"
2348        );
2349        assert!(fake_wt.exists(), "directory should still be on disk");
2350    }
2351
2352    struct MergeCheckMockBackend {
2353        batch_result: Mutex<anyhow::Result<HashMap<String, bool>>>,
2354        single_results: HashMap<String, anyhow::Result<bool>>,
2355        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2356        single_checked_branches: Arc<Mutex<Vec<String>>>,
2357    }
2358
2359    impl MergeCheckMockBackend {
2360        fn with_batch_error(
2361            error: anyhow::Error,
2362            single_results: HashMap<String, anyhow::Result<bool>>,
2363        ) -> Self {
2364            Self {
2365                batch_result: Mutex::new(Err(error)),
2366                single_results,
2367                posted_comments: Arc::new(Mutex::new(Vec::new())),
2368                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2369            }
2370        }
2371
2372        fn with_merged(branch: &str) -> Self {
2373            Self {
2374                batch_result: Mutex::new(Ok(HashMap::from([(branch.to_string(), true)]))),
2375                single_results: HashMap::new(),
2376                posted_comments: Arc::new(Mutex::new(Vec::new())),
2377                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2378            }
2379        }
2380    }
2381
2382    #[async_trait]
2383    impl ExternalBackend for MergeCheckMockBackend {
2384        fn name(&self) -> &'static str {
2385            "mock"
2386        }
2387
2388        async fn create_task(
2389            &self,
2390            _title: &str,
2391            _body: &str,
2392            _labels: &[String],
2393        ) -> anyhow::Result<ExternalId> {
2394            anyhow::bail!("not implemented")
2395        }
2396
2397        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2398            anyhow::bail!("not implemented")
2399        }
2400
2401        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2402            Ok(vec![])
2403        }
2404
2405        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2406            Ok(vec![])
2407        }
2408
2409        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2410            self.posted_comments
2411                .lock()
2412                .unwrap()
2413                .push((id.0.clone(), body.to_string()));
2414            Ok(())
2415        }
2416
2417        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2418            Ok(())
2419        }
2420
2421        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2422            Ok(())
2423        }
2424
2425        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2426            Ok(vec![])
2427        }
2428
2429        async fn create_sub_task(
2430            &self,
2431            _parent: &ExternalId,
2432            _title: &str,
2433            _body: &str,
2434            _labels: &[String],
2435        ) -> anyhow::Result<ExternalId> {
2436            anyhow::bail!("not implemented")
2437        }
2438
2439        async fn health_check(&self) -> anyhow::Result<()> {
2440            Ok(())
2441        }
2442
2443        async fn is_pr_merged(&self, branch: &str) -> anyhow::Result<bool> {
2444            self.single_checked_branches
2445                .lock()
2446                .unwrap()
2447                .push(branch.to_string());
2448            match self.single_results.get(branch) {
2449                Some(Ok(is_merged)) => Ok(*is_merged),
2450                Some(Err(err)) => Err(anyhow::anyhow!(err.to_string())),
2451                None => Ok(false),
2452            }
2453        }
2454
2455        async fn batch_is_pr_merged(
2456            &self,
2457            _branches: &[String],
2458        ) -> anyhow::Result<HashMap<String, bool>> {
2459            match &*self.batch_result.lock().unwrap() {
2460                Ok(map) => Ok(map.clone()),
2461                Err(err) => Err(anyhow::anyhow!(err.to_string())),
2462            }
2463        }
2464    }
2465
2466    #[tokio::test]
2467    async fn check_merged_prs_falls_back_to_single_branch_checks_on_batch_error() {
2468        let single_results = HashMap::from([
2469            ("branch-1".to_string(), Ok(true)),
2470            ("branch-2".to_string(), Ok(false)),
2471        ]);
2472        let backend = Arc::new(MergeCheckMockBackend::with_batch_error(
2473            anyhow::anyhow!("missing /data/repository in GraphQL response: no error details"),
2474            single_results,
2475        ));
2476        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2477
2478        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2479        let task1 = store
2480            .create(&NewTask {
2481                external_id: Some("101".to_string()),
2482                repo: "owner/repo".to_string(),
2483                origin: "github".to_string(),
2484                title: "Task 101".to_string(),
2485                ..Default::default()
2486            })
2487            .await
2488            .unwrap();
2489        store
2490            .update_status(task1, TaskStatus::NeedsReview)
2491            .await
2492            .unwrap();
2493        store
2494            .set_fields(task1, &[("branch", serde_json::json!("branch-1"))])
2495            .await
2496            .unwrap();
2497        // No .git file, so cleanup force-removes the directory
2498        let wt_dir = tempfile::tempdir().unwrap();
2499        let wt_path = wt_dir.path().join("branch-1");
2500        std_fs::create_dir_all(&wt_path).unwrap();
2501        store
2502            .set_fields(
2503                task1,
2504                &[("worktree", serde_json::json!(wt_path.to_string_lossy()))],
2505            )
2506            .await
2507            .unwrap();
2508
2509        let task2 = store
2510            .create(&NewTask {
2511                external_id: Some("102".to_string()),
2512                repo: "owner/repo".to_string(),
2513                origin: "github".to_string(),
2514                title: "Task 102".to_string(),
2515                ..Default::default()
2516            })
2517            .await
2518            .unwrap();
2519        store
2520            .update_status(task2, TaskStatus::InReview)
2521            .await
2522            .unwrap();
2523        store
2524            .set_fields(task2, &[("branch", serde_json::json!("branch-2"))])
2525            .await
2526            .unwrap();
2527
2528        let task_manager = Arc::new(TaskManager::with_store(
2529            Arc::clone(&backend_dyn),
2530            Arc::clone(&store),
2531            "owner/repo".to_string(),
2532        ));
2533
2534        let in_review_tasks = store
2535            .list_by_status("owner/repo", TaskStatus::InReview)
2536            .await
2537            .unwrap()
2538            .into_iter()
2539            .map(|stored| ReviewTaskSnapshot {
2540                external: tasks::store_task_to_external(&stored),
2541                stored,
2542            })
2543            .collect::<Vec<_>>();
2544        let needs_review_tasks = store
2545            .list_by_status("owner/repo", TaskStatus::NeedsReview)
2546            .await
2547            .unwrap()
2548            .into_iter()
2549            .map(|stored| ReviewTaskSnapshot {
2550                external: tasks::store_task_to_external(&stored),
2551                stored,
2552            })
2553            .collect::<Vec<_>>();
2554
2555        check_merged_prs(
2556            &backend_dyn,
2557            "owner/repo",
2558            &store,
2559            &task_manager,
2560            &in_review_tasks,
2561            &needs_review_tasks,
2562        )
2563        .await
2564        .unwrap();
2565
2566        let checked = backend.single_checked_branches.lock().unwrap().clone();
2567        assert_eq!(
2568            checked,
2569            vec!["branch-2".to_string(), "branch-1".to_string()]
2570        );
2571
2572        let updated_task1 = store.get(task1).await.unwrap();
2573        let updated_task2 = store.get(task2).await.unwrap();
2574        assert_eq!(updated_task1.status, TaskStatus::Done);
2575        assert!(updated_task1.worktree_cleaned);
2576        assert!(!wt_path.exists());
2577        assert!(!updated_task2.worktree_cleaned);
2578        assert_eq!(updated_task2.status, TaskStatus::InReview);
2579
2580        let comments = backend.posted_comments.lock().unwrap().clone();
2581        assert_eq!(comments.len(), 1);
2582        assert_eq!(comments[0].0, "101");
2583        assert!(comments[0].1.contains("PR merged, marking task complete"));
2584    }
2585
2586    #[tokio::test]
2587    async fn check_merged_prs_skips_rescue_while_ci_bootstrap_holds() {
2588        let (tmp, clone) = origin_and_clone_with_feat();
2589        let wt = add_worktree(&clone, tmp.path(), "feat");
2590        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2591        // hold_after_merge reads the plan marker in the body, not the title
2592        let id = store_approved_task(
2593            &store,
2594            "owner/repo",
2595            "task-12",
2596            "CI-01 Set up CI and tests",
2597            "<!-- orch-plan: CI-01 -->",
2598            &wt,
2599        )
2600        .await;
2601        let stored = store.get(id).await.unwrap();
2602
2603        let backend = Arc::new(MergeCheckMockBackend::with_merged("feat"));
2604        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2605        let task_manager = Arc::new(TaskManager::with_store(
2606            Arc::clone(&backend_dyn),
2607            Arc::clone(&store),
2608            "owner/repo".to_string(),
2609        ));
2610
2611        check_merged_prs(
2612            &backend_dyn,
2613            "owner/repo",
2614            &store,
2615            &task_manager,
2616            &[ReviewTaskSnapshot {
2617                external: tasks::store_task_to_external(&stored),
2618                stored,
2619            }],
2620            &[],
2621        )
2622        .await
2623        .unwrap();
2624
2625        let after = store.get(id).await.unwrap();
2626        assert_eq!(after.status, TaskStatus::Approved);
2627        assert!(after.worktree_cleaned);
2628        assert!(!wt.exists());
2629        assert!(!remote_has(&clone, "rescue/feat"));
2630        assert!(backend.posted_comments.lock().unwrap().is_empty());
2631    }
2632
2633    struct ReconciliationMockBackend {
2634        reconciliation_calls: Arc<AtomicUsize>,
2635    }
2636
2637    #[async_trait]
2638    impl ExternalBackend for ReconciliationMockBackend {
2639        fn name(&self) -> &'static str {
2640            "mock"
2641        }
2642
2643        async fn create_task(
2644            &self,
2645            _title: &str,
2646            _body: &str,
2647            _labels: &[String],
2648        ) -> anyhow::Result<ExternalId> {
2649            anyhow::bail!("not implemented")
2650        }
2651
2652        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2653            anyhow::bail!("not implemented")
2654        }
2655
2656        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2657            Ok(vec![])
2658        }
2659
2660        async fn list_reconciliation_candidates(&self) -> anyhow::Result<Vec<ExternalTask>> {
2661            self.reconciliation_calls.fetch_add(1, Ordering::SeqCst);
2662            Ok(vec![])
2663        }
2664
2665        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2666            Ok(vec![])
2667        }
2668
2669        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2670            Ok(())
2671        }
2672
2673        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2674            Ok(())
2675        }
2676
2677        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2678            Ok(())
2679        }
2680
2681        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2682            Ok(vec![])
2683        }
2684
2685        async fn create_sub_task(
2686            &self,
2687            _parent: &ExternalId,
2688            _title: &str,
2689            _body: &str,
2690            _labels: &[String],
2691        ) -> anyhow::Result<ExternalId> {
2692            anyhow::bail!("not implemented")
2693        }
2694
2695        async fn health_check(&self) -> anyhow::Result<()> {
2696            Ok(())
2697        }
2698    }
2699
2700    #[tokio::test]
2701    async fn cleanup_reconciliation_is_rate_limited_per_repo() {
2702        let repo = "owner/repo-cleanup-rate-limit";
2703        RECONCILIATION_LAST_RUN.remove(repo);
2704
2705        let backend = Arc::new(ReconciliationMockBackend {
2706            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2707        });
2708        let backend_dyn: Arc<dyn ExternalBackend> =
2709            Arc::<ReconciliationMockBackend>::clone(&backend);
2710        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2711        let task_manager = Arc::new(TaskManager::with_store(
2712            Arc::clone(&backend_dyn),
2713            Arc::clone(&store),
2714            repo.to_string(),
2715        ));
2716        let opts = JanitorOptions {
2717            ttl_hours: 0,
2718            dry_run: true,
2719        };
2720
2721        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2722            .await
2723            .unwrap();
2724        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2725            .await
2726            .unwrap();
2727
2728        assert_eq!(
2729            backend.reconciliation_calls.load(Ordering::SeqCst),
2730            1,
2731            "cleanup should not re-list reconciliation candidates on every sync pass"
2732        );
2733
2734        RECONCILIATION_LAST_RUN.remove(repo);
2735    }
2736
2737    /// Cleanup only collects done tasks. A blocked task waits on a person and resumes from its
2738    /// worktree and branch, so it must not even be considered (#3781)
2739    #[tokio::test]
2740    async fn cleanup_skips_blocked_internal_tasks_and_cleans_done_ones() {
2741        let repo = "owner/repo-cleanup-blocked";
2742        RECONCILIATION_LAST_RUN.remove(repo);
2743        let backend_dyn: Arc<dyn ExternalBackend> = Arc::new(ReconciliationMockBackend {
2744            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2745        });
2746        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2747        let task_manager = Arc::new(TaskManager::with_store(
2748            Arc::clone(&backend_dyn),
2749            Arc::clone(&store),
2750            repo.to_string(),
2751        ));
2752
2753        let mut ids = Vec::new();
2754        for (title, status) in [("blocked", TaskStatus::Blocked), ("done", TaskStatus::Done)] {
2755            let id = store
2756                .create_internal(repo, title, "", "test", title, None)
2757                .await
2758                .unwrap();
2759            store
2760                .set_fields(
2761                    id,
2762                    &[
2763                        (
2764                            "worktree",
2765                            serde_json::json!(format!("/nonexistent/{title}")),
2766                        ),
2767                        ("branch", serde_json::json!("")),
2768                    ],
2769                )
2770                .await
2771                .unwrap();
2772            store.update_status(id, status).await.unwrap();
2773            ids.push(id);
2774        }
2775
2776        let opts = JanitorOptions {
2777            ttl_hours: 0,
2778            dry_run: false,
2779        };
2780        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2781            .await
2782            .unwrap();
2783
2784        assert!(
2785            !store.get(ids[0]).await.unwrap().worktree_cleaned,
2786            "blocked task must not be considered for cleanup"
2787        );
2788        assert!(
2789            store.get(ids[1]).await.unwrap().worktree_cleaned,
2790            "done task is cleaned, which proves the pass ran"
2791        );
2792        RECONCILIATION_LAST_RUN.remove(repo);
2793    }
2794
2795    fn git_ok(dir: &Path, args: &[&str]) -> String {
2796        let out = process::Command::new("git")
2797            .args([
2798                "-c",
2799                "commit.gpgsign=false",
2800                "-c",
2801                "user.name=T",
2802                "-c",
2803                "user.email=t@t",
2804            ])
2805            .args(args)
2806            .current_dir(dir)
2807            .output()
2808            .unwrap();
2809        assert!(
2810            out.status.success(),
2811            "git {args:?}: {}",
2812            String::from_utf8_lossy(&out.stderr)
2813        );
2814        String::from_utf8_lossy(&out.stdout).to_string()
2815    }
2816
2817    /// A clone of a bare origin with main pushed and branch `feat` one commit ahead
2818    fn origin_and_clone_with_feat() -> (tempfile::TempDir, PathBuf) {
2819        let tmp = tempfile::tempdir().unwrap();
2820        let bare = tmp.path().join("origin.git");
2821        std_fs::create_dir_all(&bare).unwrap();
2822        git_ok(&bare, &["init", "--bare", "-b", "main"]);
2823        git_ok(tmp.path(), &["clone", bare.to_str().unwrap(), "clone"]);
2824        let clone = tmp.path().join("clone");
2825        std_fs::write(clone.join("base.txt"), "base").unwrap();
2826        git_ok(&clone, &["add", "."]);
2827        git_ok(&clone, &["commit", "-m", "base"]);
2828        git_ok(&clone, &["push", "origin", "main"]);
2829        git_ok(&clone, &["checkout", "-b", "feat"]);
2830        std_fs::write(clone.join("work.txt"), "work").unwrap();
2831        git_ok(&clone, &["add", "."]);
2832        git_ok(&clone, &["commit", "-m", "work"]);
2833        git_ok(&clone, &["checkout", "main"]);
2834        (tmp, clone)
2835    }
2836
2837    fn remote_has(clone: &Path, branch: &str) -> bool {
2838        !git_ok(clone, &["ls-remote", "--heads", "origin", branch])
2839            .trim()
2840            .is_empty()
2841    }
2842
2843    fn local_has(clone: &Path, branch: &str) -> bool {
2844        process::Command::new("git")
2845            .args([
2846                "show-ref",
2847                "--verify",
2848                "--quiet",
2849                &format!("refs/heads/{branch}"),
2850            ])
2851            .current_dir(clone)
2852            .status()
2853            .unwrap()
2854            .success()
2855    }
2856
2857    /// Deleting a pushed branch whose commits main lacks would destroy the only copy of the
2858    /// work, as startup cleanup did to a blocked task (#3781)
2859    #[tokio::test]
2860    async fn delete_branches_keeps_an_unmerged_remote_branch() {
2861        let (_tmp, clone) = origin_and_clone_with_feat();
2862        git_ok(&clone, &["push", "origin", "feat"]);
2863
2864        delete_branches("1", "feat", &clone).await;
2865
2866        assert!(
2867            remote_has(&clone, "feat"),
2868            "unmerged remote branch must survive"
2869        );
2870        assert!(
2871            !local_has(&clone, "feat"),
2872            "local copy is on origin, so it can go"
2873        );
2874    }
2875
2876    #[tokio::test]
2877    async fn delete_branches_leaves_the_screenshots_branch_alone() {
2878        let (_tmp, clone) = origin_and_clone_with_feat();
2879        git_ok(&clone, &["branch", SCREENSHOTS_BRANCH, "feat"]);
2880        git_ok(&clone, &["push", "origin", SCREENSHOTS_BRANCH]);
2881        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2882        git_ok(&clone, &["push", "origin", "main"]);
2883
2884        delete_branches("1", SCREENSHOTS_BRANCH, &clone).await;
2885
2886        assert!(remote_has(&clone, SCREENSHOTS_BRANCH));
2887        assert!(local_has(&clone, SCREENSHOTS_BRANCH));
2888    }
2889
2890    #[tokio::test]
2891    async fn delete_branches_removes_a_branch_main_already_contains() {
2892        let (_tmp, clone) = origin_and_clone_with_feat();
2893        git_ok(&clone, &["push", "origin", "feat"]);
2894        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2895        git_ok(&clone, &["push", "origin", "main"]);
2896
2897        delete_branches("1", "feat", &clone).await;
2898
2899        assert!(!remote_has(&clone, "feat"));
2900        assert!(!local_has(&clone, "feat"));
2901    }
2902
2903    fn add_worktree(clone: &Path, tmp: &Path, branch: &str) -> PathBuf {
2904        let wt = tmp.join("wt");
2905        git_ok(clone, &["worktree", "add", wt.to_str().unwrap(), branch]);
2906        wt
2907    }
2908
2909    async fn store_approved_task(
2910        store: &TaskStore,
2911        repo: &str,
2912        external_id: &str,
2913        title: &str,
2914        body: &str,
2915        worktree: &Path,
2916    ) -> i64 {
2917        let id = store
2918            .create(&NewTask {
2919                external_id: Some(external_id.to_string()),
2920                repo: repo.to_string(),
2921                origin: "github".to_string(),
2922                title: title.to_string(),
2923                body: body.to_string(),
2924                ..Default::default()
2925            })
2926            .await
2927            .unwrap();
2928        store.update_status(id, TaskStatus::Approved).await.unwrap();
2929        store
2930            .set_fields(
2931                id,
2932                &[
2933                    ("branch", serde_json::json!("feat")),
2934                    (
2935                        "worktree",
2936                        serde_json::json!(worktree.to_string_lossy().as_ref()),
2937                    ),
2938                    ("pr_number", serde_json::json!(7)),
2939                ],
2940            )
2941            .await
2942            .unwrap();
2943        id
2944    }
2945
2946    #[tokio::test]
2947    async fn cleanup_pushes_unpushed_commits_to_a_rescue_branch() {
2948        let (tmp, clone) = origin_and_clone_with_feat();
2949        let wt = add_worktree(&clone, tmp.path(), "feat");
2950
2951        let sha = git_ok(&clone, &["rev-parse", "--short", "feat"]);
2952        let sha = sha.trim();
2953        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2954            Rescue::Pushed { branch, commits } => {
2955                assert_eq!(branch, "rescue/feat");
2956                assert!(rescue_comment(&branch, &commits).contains(sha));
2957            }
2958            other => panic!("expected a rescue push, got {other:?}"),
2959        }
2960
2961        let tip = git_ok(&clone, &["rev-parse", "feat"]);
2962        let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2963        assert_eq!(
2964            tracking.trim(),
2965            tip.trim(),
2966            "rescue push must point the tracking ref at the tip it sent"
2967        );
2968
2969        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2970
2971        assert!(removed);
2972        assert!(remote_has(&clone, "rescue/feat"));
2973        assert!(!local_has(&clone, "feat"));
2974    }
2975
2976    #[tokio::test]
2977    async fn rescue_push_sets_tracking_ref_without_a_fetchspec() {
2978        let (tmp, clone) = origin_and_clone_with_feat();
2979        let wt = add_worktree(&clone, tmp.path(), "feat");
2980        git_ok(&clone, &["config", "--unset-all", "remote.origin.fetch"]);
2981
2982        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2983            Rescue::Pushed { branch, .. } => assert_eq!(branch, "rescue/feat"),
2984            other => panic!("expected a rescue push, got {other:?}"),
2985        }
2986
2987        let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2988        let tip = git_ok(&clone, &["rev-parse", "feat"]);
2989        assert_eq!(tracking.trim(), tip.trim());
2990    }
2991
2992    #[tokio::test]
2993    async fn cleanup_commits_uncommitted_changes_before_the_rescue() {
2994        let (tmp, clone) = origin_and_clone_with_feat();
2995        git_ok(&clone, &["push", "origin", "feat"]);
2996        let wt = add_worktree(&clone, tmp.path(), "feat");
2997        std_fs::write(wt.join("dirty.txt"), "dirty").unwrap();
2998
2999        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3000
3001        assert!(removed);
3002        assert!(remote_has(&clone, "rescue/feat"));
3003    }
3004
3005    #[tokio::test]
3006    async fn cleanup_makes_no_rescue_branch_for_a_pushed_branch() {
3007        let (tmp, clone) = origin_and_clone_with_feat();
3008        git_ok(&clone, &["push", "origin", "feat"]);
3009        let wt = add_worktree(&clone, tmp.path(), "feat");
3010
3011        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3012
3013        assert!(removed);
3014        assert!(!remote_has(&clone, "rescue/feat"));
3015        assert!(!local_has(&clone, "feat"));
3016    }
3017
3018    #[tokio::test]
3019    async fn rescue_skips_a_branch_on_the_remote_without_a_tracking_ref() {
3020        let (_tmp, clone) = origin_and_clone_with_feat();
3021        git_ok(&clone, &["push", "origin", "feat"]);
3022        git_ok(&clone, &["update-ref", "-d", "refs/remotes/origin/feat"]);
3023
3024        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3025
3026        assert_eq!(rescue, Rescue::Clean);
3027        assert!(!remote_has(&clone, "rescue/feat"));
3028    }
3029
3030    #[tokio::test]
3031    async fn merged_pr_cleanup_pushes_no_rescue_branch() {
3032        let (tmp, clone) = origin_and_clone_with_feat();
3033        let wt = add_worktree(&clone, tmp.path(), "feat");
3034
3035        let removed =
3036            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, true).await;
3037
3038        assert!(removed);
3039        assert!(!wt.exists());
3040        assert!(!remote_has(&clone, "rescue/feat"));
3041    }
3042
3043    #[tokio::test]
3044    async fn cleanup_skips_rescue_for_an_approved_merged_pr() {
3045        let (tmp, clone) = origin_and_clone_with_feat();
3046        let wt = add_worktree(&clone, tmp.path(), "feat");
3047        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3048        store_approved_task(&store, "owner/repo", "task-12", "CI-01 Set up CI", "", &wt).await;
3049
3050        cleanup_merged_pr_worktree("task-12", "owner/repo", &store)
3051            .await
3052            .unwrap();
3053
3054        let task = store
3055            .get(
3056                store
3057                    .resolve_task_id("owner/repo", "task-12")
3058                    .await
3059                    .unwrap()
3060                    .unwrap(),
3061            )
3062            .await
3063            .unwrap();
3064        assert_eq!(task.status, TaskStatus::Approved);
3065        assert!(task.worktree_cleaned);
3066        assert!(!wt.exists());
3067        assert!(!remote_has(&clone, "rescue/feat"));
3068    }
3069
3070    #[tokio::test]
3071    async fn cleanup_rescues_an_approved_task_when_the_pr_is_not_known_merged() {
3072        let (tmp, clone) = origin_and_clone_with_feat();
3073        let wt = add_worktree(&clone, tmp.path(), "feat");
3074        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3075        store_approved_task(&store, "owner/repo", "task-12", "fix something", "", &wt).await;
3076
3077        cleanup_task_worktree("task-12", "owner/repo", &store)
3078            .await
3079            .unwrap();
3080
3081        assert!(remote_has(&clone, "rescue/feat"));
3082    }
3083
3084    #[tokio::test]
3085    async fn unmerged_cleanup_still_pushes_a_rescue_branch() {
3086        let (tmp, clone) = origin_and_clone_with_feat();
3087        let wt = add_worktree(&clone, tmp.path(), "feat");
3088
3089        let removed =
3090            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, false).await;
3091
3092        assert!(removed);
3093        assert!(remote_has(&clone, "rescue/feat"));
3094    }
3095
3096    #[tokio::test]
3097    async fn rescue_ignores_a_remote_tip_missing_locally() {
3098        let (tmp, clone) = origin_and_clone_with_feat();
3099        git_ok(&clone, &["push", "origin", "feat"]);
3100        let other = tmp.path().join("other");
3101        git_ok(
3102            tmp.path(),
3103            &["clone", "-b", "feat", "origin.git", other.to_str().unwrap()],
3104        );
3105        std_fs::write(other.join("extra.txt"), "extra").unwrap();
3106        git_ok(&other, &["add", "."]);
3107        git_ok(&other, &["commit", "-m", "extra"]);
3108        git_ok(&other, &["push", "origin", "feat"]);
3109
3110        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3111
3112        assert_ne!(rescue, Rescue::Failed);
3113    }
3114
3115    #[tokio::test]
3116    async fn cleanup_keeps_worktree_and_branch_when_the_rescue_push_fails() {
3117        let (tmp, clone) = origin_and_clone_with_feat();
3118        let wt = add_worktree(&clone, tmp.path(), "feat");
3119        git_ok(
3120            &clone,
3121            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3122        );
3123
3124        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3125
3126        assert!(!removed);
3127        assert!(wt.exists());
3128        assert!(local_has(&clone, "feat"));
3129    }
3130
3131    #[tokio::test]
3132    async fn delete_branches_keeps_a_local_branch_with_unpushed_commits() {
3133        let (_tmp, clone) = origin_and_clone_with_feat();
3134
3135        delete_branches("1", "feat", &clone).await;
3136
3137        assert!(
3138            local_has(&clone, "feat"),
3139            "unpushed commits must not be deleted"
3140        );
3141    }
3142}