Skip to main content

orch/engine/
cleanup.rs

1//! Worktree cleanup and merged-PR detection.
2//!
3//! Contains the post-merge cleanup pipeline: removing git worktrees,
4//! deleting local/remote branches, pulling main, and detecting
5//! already-merged PRs so their tasks can be marked done.
6
7use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
8use crate::cmd::CommandErrorContext;
9use crate::engine::ci_bootstrap;
10use crate::engine::tasks::{TaskManager, store_task_to_external};
11use crate::store;
12use crate::store::TaskStatus;
13use crate::store::TaskStore;
14use crate::store::store_log_activity;
15use dashmap::{DashMap, DashSet};
16use futures::future;
17use std::collections::HashMap;
18use std::collections::HashSet;
19use std::env;
20use std::fs as std_fs;
21use std::io::ErrorKind;
22use std::path::Path;
23use std::path::PathBuf;
24use std::process::Output;
25use std::sync::Arc;
26use std::sync::LazyLock;
27use std::time::SystemTime;
28use std::time::{Duration, Instant};
29use tokio::fs;
30use tokio::process::Command;
31use tokio::time::sleep;
32use tokio::time::timeout;
33
34use super::sync::ReviewTaskSnapshot;
35use crate::config;
36use crate::engine;
37use crate::engine::auto_merge;
38use crate::engine::runner::cow;
39use crate::engine::runner::git_ops;
40use crate::engine::runner::screenshots::SCREENSHOTS_BRANCH;
41use crate::engine::runner::worktree;
42use crate::github::http::GhHttp;
43use crate::home;
44
45/// Reconciliation calls can be slow on repos with many issues or during GitHub instability.
46/// Cleanup runs in a background task (tokio::spawn) so it does not block the main tick loop.
47/// 30 s gives enough headroom for paginated list_all_tasks calls while still bounding the
48/// overall background-task duration.
49const RECONCILIATION_LIST_TIMEOUT: Duration = Duration::from_secs(30);
50/// Closed-issue reconciliation is a best-effort cleanup fallback, not a per-sync requirement.
51/// Run it on a slower cadence so repeated GitHub slowness does not consume 30 s every sync.
52const RECONCILIATION_INTERVAL: Duration = Duration::from_mins(15);
53
54const RUN_OUTPUT_RETENTION_DAYS: i32 = 30;
55const RUN_PRUNE_INTERVAL_SECS: i64 = 24 * 60 * 60;
56const RUN_PRUNE_KV_KEY: &str = "task_runs_pruned_at";
57
58/// Per-repo in-flight guard for closed-issue reconciliation/cleanup.
59/// Prevents overlapping cleanup runs from piling up across sync cycles.
60static CLEANUP_IN_FLIGHT: LazyLock<DashSet<String>> = LazyLock::new(DashSet::new);
61/// Per-repo reconciliation cadence tracker.
62static RECONCILIATION_LAST_RUN: LazyLock<DashMap<String, Instant>> = LazyLock::new(DashMap::new);
63
64struct CleanupInFlightGuard {
65    repo: String,
66}
67
68impl CleanupInFlightGuard {
69    fn try_acquire(repo: &str) -> Option<Self> {
70        if CLEANUP_IN_FLIGHT.insert(repo.to_string()) {
71            Some(Self {
72                repo: repo.to_string(),
73            })
74        } else {
75            None
76        }
77    }
78}
79
80impl Drop for CleanupInFlightGuard {
81    fn drop(&mut self) {
82        CLEANUP_IN_FLIGHT.remove(&self.repo);
83    }
84}
85
86fn should_run_reconciliation(repo: &str) -> bool {
87    let now = Instant::now();
88    if let Some(mut last_run) = RECONCILIATION_LAST_RUN.get_mut(repo) {
89        if last_run.elapsed() < RECONCILIATION_INTERVAL {
90            return false;
91        }
92        *last_run = now;
93        true
94    } else {
95        RECONCILIATION_LAST_RUN.insert(repo.to_string(), now);
96        true
97    }
98}
99
100/// Options controlling the worktree janitor.
101#[derive(Debug, Clone)]
102pub struct JanitorOptions {
103    /// Minimum age (in hours) of the worktree directory before it is eligible for removal.
104    ///
105    /// A worktree whose filesystem mtime is more recent than this threshold is skipped
106    /// even if the task is already in a terminal state. This provides a safety buffer
107    /// against races (e.g. review agent still writing when the janitor runs).
108    ///
109    /// Default: 24.  Configurable via `workflow.worktree_janitor_ttl_hours` in config.
110    pub ttl_hours: u64,
111
112    /// When `true`, log what would be done but skip all destructive operations.
113    ///
114    /// Configurable via `workflow.worktree_janitor_dry_run: true` in config.
115    pub dry_run: bool,
116}
117
118impl Default for JanitorOptions {
119    fn default() -> Self {
120        Self {
121            ttl_hours: 24,
122            dry_run: false,
123        }
124    }
125}
126
127impl JanitorOptions {
128    /// Read options from the global config, falling back to defaults.
129    pub fn from_config() -> Self {
130        let ttl_hours = config::get("workflow.worktree_janitor_ttl_hours")
131            .ok()
132            .and_then(|v| v.parse::<u64>().ok())
133            .unwrap_or(24);
134
135        let dry_run = config::get("workflow.worktree_janitor_dry_run").is_ok_and(|v| v == "true");
136
137        Self { ttl_hours, dry_run }
138    }
139}
140
141/// Cleanup worktrees for completed tasks.
142///
143/// Targets tasks that are finished: status:done OR closed on GitHub
144/// (e.g. when a PR merge auto-closes the issue via "Fixes #N").
145///
146/// Note: This is a fallback. Primary cleanup happens inline in
147/// `auto_merge_pr` via `cleanup_task_worktree`. This catches edge cases
148/// where the inline cleanup missed (e.g., manual merges, auto-closed issues).
149pub(crate) async fn cleanup_done_worktrees(
150    backend: &Arc<dyn ExternalBackend>,
151    repo: &str,
152    task_manager: &Arc<TaskManager>,
153    store: &Arc<TaskStore>,
154) -> anyhow::Result<()> {
155    let opts = JanitorOptions::from_config();
156    cleanup_done_worktrees_with_opts(backend, repo, task_manager, store, &opts).await
157}
158
159/// Clear the stored output of runs for tasks done longer than `RUN_OUTPUT_RETENTION_DAYS`, once a day
160pub(crate) async fn prune_old_run_output(store: &TaskStore) -> anyhow::Result<()> {
161    let now = chrono::Utc::now().timestamp();
162    let last = store
163        .kv_get(RUN_PRUNE_KV_KEY)
164        .await?
165        .and_then(|v| v.parse::<i64>().ok())
166        .unwrap_or(0);
167    if now - last < RUN_PRUNE_INTERVAL_SECS {
168        return Ok(());
169    }
170    store.kv_set(RUN_PRUNE_KV_KEY, &now.to_string()).await?;
171
172    let pruned = store.prune_old_runs(RUN_OUTPUT_RETENTION_DAYS).await?;
173    if pruned > 0 {
174        tracing::info!(pruned, "cleared the output of old task runs");
175    }
176    Ok(())
177}
178
179async fn reconcile_closed_tasks(
180    task_ids: &mut Vec<String>,
181    all_tasks: &[ExternalTask],
182    task_manager: &TaskManager,
183    store: &Arc<TaskStore>,
184    repo: &str,
185) {
186    const MAX_RECONCILE_PER_CYCLE: usize = 10;
187
188    let done_set: HashSet<String> = task_ids.iter().cloned().collect();
189    let mut closed_not_in_store = Vec::new();
190    for task in all_tasks {
191        if task.state != "closed" || done_set.contains(&task.id.0) {
192            continue;
193        }
194        if ci_bootstrap::hold_after_merge(task, store, repo).await {
195            continue;
196        }
197        closed_not_in_store.push(task);
198    }
199
200    // Split: issues that already have status:done label just need worktree
201    // cleanup (no API call needed), vs those needing label reconciliation.
202    let mut already_labeled: Vec<String> = Vec::new();
203    let mut needs_reconcile: Vec<&ExternalTask> = Vec::new();
204    for task in &closed_not_in_store {
205        if task.labels.iter().any(|l| l == "status:done") {
206            already_labeled.push(task.id.0.clone());
207        } else {
208            needs_reconcile.push(task);
209        }
210    }
211
212    // Add already-labeled issues directly (no API call).
213    if !already_labeled.is_empty() {
214        tracing::debug!(
215            count = already_labeled.len(),
216            "closed issues already labeled status:done — adding to cleanup set"
217        );
218        task_ids.extend(already_labeled);
219    }
220
221    // Reconcile issues missing the done label — cap to avoid rate-limit
222    // exhaustion. Remaining issues will be picked up in subsequent cycles.
223    if needs_reconcile.len() > MAX_RECONCILE_PER_CYCLE {
224        tracing::info!(
225            total = needs_reconcile.len(),
226            batch = MAX_RECONCILE_PER_CYCLE,
227            "capping closed-issue reconciliation to avoid rate-limit exhaustion"
228        );
229    }
230    let mut futs = Vec::new();
231    for task in needs_reconcile.iter().take(MAX_RECONCILE_PER_CYCLE) {
232        let id = task.id.clone();
233        let task_id_str = task.id.0.clone();
234        let labels = task.labels.clone();
235        futs.push(async move {
236            tracing::info!(
237                task_id = task_id_str,
238                labels = ?labels,
239                "closed issue with stale status label — reconciling to done and cleaning up"
240            );
241            match task_manager.update_task_status(&id, Status::Done).await {
242                Ok(()) => Some(task_id_str),
243                Err(e) => {
244                    tracing::warn!(
245                        task_id = task_id_str,
246                        err = %e,
247                        "failed to reconcile closed issue status to done"
248                    );
249                    None
250                }
251            }
252        });
253    }
254    let results = future::join_all(futs).await;
255    task_ids.extend(results.into_iter().flatten());
256}
257
258/// Cleanup worktrees for completed tasks with explicit options.
259///
260/// Separated from `cleanup_done_worktrees` so that integration tests can
261/// inject specific options (e.g. `ttl_hours: 0`, `dry_run: true`) without
262/// touching the global config.
263pub(crate) async fn cleanup_done_worktrees_with_opts(
264    backend: &Arc<dyn ExternalBackend>,
265    repo: &str,
266    task_manager: &Arc<TaskManager>,
267    store: &Arc<TaskStore>,
268    opts: &JanitorOptions,
269) -> anyhow::Result<()> {
270    let Some(_guard) = CleanupInFlightGuard::try_acquire(repo) else {
271        tracing::debug!(
272            repo = %repo,
273            "cleanup already in progress for repo — skipping overlapping run"
274        );
275        return Ok(());
276    };
277
278    // Read done tasks from the store first; fall back to backend before first sync.
279    let done_tasks = task_manager.list_all_by_status(Status::Done).await?;
280    tracing::debug!(count = done_tasks.len(), "checking done tasks for cleanup");
281
282    // Collect task IDs from external done tasks.
283    let mut task_ids: Vec<String> = done_tasks.iter().map(|t| t.id.0.clone()).collect();
284
285    // Also include closed-but-not-done external tasks.
286    // When a PR merge auto-closes a GitHub issue (via "Fixes #N"), the issue
287    // state becomes "closed" but orch never updates the status label to "done".
288    // These orphaned worktrees accumulate unless we catch them here.
289    // Note: This still queries the backend because GitHub `state` (open/closed)
290    // is backend-specific and not tracked in the store.
291    //
292    // Use list_reconciliation_candidates() (closed issues only, bounded recent window) directly.
293    // list_all_tasks() fetches every issue ever created (state=all, all pages) and
294    // consistently times out on repos with many issues — it is not appropriate for
295    // a cleanup fallback path.
296    if should_run_reconciliation(repo) {
297        match timeout(
298            RECONCILIATION_LIST_TIMEOUT,
299            backend.list_reconciliation_candidates(),
300        )
301        .await
302        {
303            Ok(Ok(candidates)) if !candidates.is_empty() => {
304                tracing::debug!(
305                    count = candidates.len(),
306                    "reconciling closed issues from candidates"
307                );
308                reconcile_closed_tasks(&mut task_ids, &candidates, task_manager, store, repo).await;
309            }
310            Ok(Ok(_)) => {
311                tracing::debug!("no reconciliation candidates found");
312            }
313            Ok(Err(e)) => {
314                tracing::warn!(err = %e, "failed to list reconciliation candidates");
315            }
316            Err(_) => {
317                tracing::warn!(
318                    timeout_secs = RECONCILIATION_LIST_TIMEOUT.as_secs(),
319                    retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
320                    "timed out listing reconciliation candidates"
321                );
322            }
323        }
324    } else {
325        tracing::debug!(
326            retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
327            "skipping closed-issue reconciliation until interval elapses"
328        );
329    }
330
331    // Also include internal done tasks.
332    if let Ok(internal_done) = task_manager.list_internal_by_status(TaskStatus::Done).await {
333        task_ids.extend(internal_done.iter().map(|t| t.id.0.clone()));
334    }
335
336    tracing::debug!(
337        count = task_ids.len(),
338        "checking all terminal tasks for cleanup"
339    );
340
341    if let Err(e) = auto_merge::sweep_ci_timer_keys(store, repo, &task_ids).await {
342        tracing::warn!(err = %e, "failed to sweep CI timer keys");
343    }
344
345    // Prefetch all cleaned external IDs in one query instead of N+1 per-task lookups.
346    // NOTE: This is now primarily a hint to skip processing, NOT a correctness
347    // guarantee — tasks with external_id NOT in the store at all (or mismatched
348    // repo) will still be cleaned by iterating all worktree directories.
349    let already_cleaned = store.cleaned_external_ids(repo).await.unwrap_or_default();
350
351    let mut cleaned_any = false;
352    for task_id in &task_ids {
353        // Skip if already cleaned (checked via prefetched set).
354        if already_cleaned.contains(task_id) {
355            continue;
356        }
357
358        match cleanup_task_worktree_with_opts(task_id, repo, store, opts).await {
359            Ok(true) => cleaned_any = true,
360            Ok(false) => {
361                // Skipped (TTL guard, tmux guard, or nothing to remove) — do not
362                // pull main; we did not change the repo state.
363            }
364            Err(e) => {
365                tracing::warn!(task_id, err = %e, "worktree cleanup failed for task");
366            }
367        }
368    }
369
370    // ── Orphaned worktree scanner ─────────────────────────────────────────────
371    //
372    // Even if every task in our list was already cleaned (or had no worktree),
373    // there may be worktree directories on disk that are not owned by any task.
374    // This happens when a task was ingested into the store but its worktree_cleaned
375    // flag was never set — e.g. because the task was never in `list_all_by_status`
376    // (unknown repo / missing from store at cleanup time), or the cleanup skipped
377    // due to TTL guard and never retried.
378    //
379    // We scan all worktree directories in this repo's worktrees subdirectory and
380    // remove any that are not referenced by an existing task. This catches the
381    // "done but worktree not cleaned" case that `orch service doctor` reports.
382    let worktrees_base = match home::worktrees_dir() {
383        Ok(base) => Some(base),
384        Err(e) => {
385            tracing::debug!(err = %e, "worktrees_dir() unavailable — skipping orphan scan");
386            None
387        }
388    };
389
390    if let Some(base) = worktrees_base {
391        let project = repo
392            .rsplit('/')
393            .next()
394            .unwrap_or(repo)
395            .trim_end_matches(".git");
396        let repo_worktrees = base.join(project);
397
398        // Build set of all known worktree paths for this repo (all statuses).
399        let mut combined: HashSet<String> = HashSet::new();
400        if let Ok(tasks) = store.list_all(repo).await {
401            for t in tasks {
402                if !t.worktree.is_empty() {
403                    combined.insert(t.worktree);
404                }
405            }
406        }
407
408        // Also include active worktrees (in case task is in-memory but has a dir).
409        if let Ok(project_dir) = resolve_project_dir_for_repo(repo)
410            && let Ok(active_wts) = worktree::list_project_worktrees(&project_dir).await
411        {
412            for wt in active_wts {
413                combined.insert(wt.to_string_lossy().to_string());
414            }
415        }
416
417        // Scan all branch directories and remove orphaned ones.
418        if let Ok(mut entries) = fs::read_dir(&repo_worktrees).await {
419            while let Ok(Some(entry)) = entries.next_entry().await {
420                let Ok(entry_meta) = entry.metadata().await else {
421                    continue;
422                };
423                if !entry_meta.is_dir() {
424                    continue;
425                }
426                let wt_path = entry.path().to_string_lossy().to_string();
427                if combined.contains(&wt_path) {
428                    continue;
429                }
430                tracing::info!(worktree = %wt_path, "found orphaned worktree — attempting cleanup");
431                match resolve_repo_root_for_orphaned_worktree(&entry.path()).await {
432                    Ok(repo_root) => {
433                        let removed = remove_worktree_and_branch(
434                            "orphan-scan",
435                            &entry.path(),
436                            None,
437                            Path::new(&repo_root),
438                            false,
439                        )
440                        .await;
441                        if removed {
442                            tracing::info!(worktree = %wt_path, "orphaned worktree removed");
443                        } else {
444                            tracing::warn!(worktree = %wt_path, "orphaned worktree could not be removed");
445                        }
446                    }
447                    Err(e) => {
448                        tracing::warn!(worktree = %wt_path, err = %e, "could not resolve repo root for orphaned worktree");
449                    }
450                }
451            }
452        }
453    }
454
455    // Prune stale worktree metadata and pull main after cleanup.
456    if cleaned_any
457        && !opts.dry_run
458        && let Ok(repo_root) = resolve_repo_root(repo).await
459    {
460        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
461
462        // Prune stale .git/worktrees/ entries whose directories no longer exist.
463        let prune_result = timeout(
464            GIT_TIMEOUT,
465            Command::new("git")
466                .args(["-C", &repo_root, "worktree", "prune"])
467                .output_with_context(),
468        )
469        .await;
470        match prune_result {
471            Ok(Ok(output)) if output.status.success() => {
472                tracing::debug!(%repo, "pruned stale worktree metadata");
473            }
474            Ok(Ok(output)) => {
475                let stderr = String::from_utf8_lossy(&output.stderr);
476                tracing::warn!(%repo, err = %stderr, "git worktree prune failed");
477            }
478            Ok(Err(e)) => {
479                tracing::warn!(%repo, err = %e, "git worktree prune failed");
480            }
481            Err(_) => {
482                tracing::warn!(%repo, "git worktree prune timed out after 60s");
483            }
484        }
485
486        // Fetch instead of pull: refresh `origin/*` refs without touching
487        // the operator's checked-out branch or working tree. `pull` would
488        // operate on whatever HEAD happens to be (often a feature branch,
489        // not main) and would silently no-op on divergence.
490        let fetch_result = timeout(
491            GIT_TIMEOUT,
492            git_ops::git()
493                .await
494                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
495                .output_with_context(),
496        )
497        .await;
498        match fetch_result {
499            Ok(Ok(output)) if output.status.success() => {
500                tracing::info!(%repo, "fetched origin after worktree cleanup");
501            }
502            Ok(Ok(output)) => {
503                let stderr = String::from_utf8_lossy(&output.stderr);
504                tracing::debug!(%repo, err = %stderr, "git fetch skipped after cleanup");
505            }
506            Ok(Err(e)) => {
507                tracing::debug!(%repo, err = %e, "git fetch failed after cleanup");
508            }
509            Err(_) => {
510                tracing::debug!(%repo, "git fetch timed out after 60s");
511            }
512        }
513    }
514
515    Ok(())
516}
517
518/// Cleanup a single task's worktree and branches.
519///
520/// Removes the git worktree, deletes local + remote branches,
521/// fetches origin to refresh remote tracking refs, and marks the task as cleaned.
522///
523/// This function is used for inline post-merge cleanup (called from
524/// auto-merge flows) and must attempt immediate removal — it constructs
525/// janitor options with `ttl_hours = 0` so the janitor age guard does not
526/// postpone removing freshly-created worktrees.
527pub(crate) async fn cleanup_task_worktree(
528    task_id: &str,
529    repo: &str,
530    store: &Arc<TaskStore>,
531) -> anyhow::Result<()> {
532    cleanup_task_worktree_impl(task_id, repo, store, false).await
533}
534
535/// Cleanup after a merged pull request, including while `hold_after_merge` keeps the task approved
536pub(crate) async fn cleanup_merged_pr_worktree(
537    task_id: &str,
538    repo: &str,
539    store: &Arc<TaskStore>,
540) -> anyhow::Result<()> {
541    cleanup_task_worktree_impl(task_id, repo, store, true).await
542}
543
544async fn cleanup_task_worktree_impl(
545    task_id: &str,
546    repo: &str,
547    store: &Arc<TaskStore>,
548    known_pr_merged: bool,
549) -> anyhow::Result<()> {
550    let opts = JanitorOptions {
551        ttl_hours: 0,
552        ..Default::default()
553    };
554    let cleaned =
555        cleanup_task_worktree_with_opts_impl(task_id, repo, store, &opts, known_pr_merged).await?;
556
557    // Fetch origin once after inline cleanup (post-merge single-task path).
558    // Only fetch if cleanup actually happened (not skipped due to active session).
559    // Fetch (not pull) so the operator's checked-out branch and working tree
560    // are never touched — see the matching comment in cleanup_orphaned_worktrees.
561    if cleaned && let Ok(repo_root) = resolve_repo_root(repo).await {
562        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
563        let fetch_result = timeout(
564            GIT_TIMEOUT,
565            git_ops::git()
566                .await
567                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
568                .output_with_context(),
569        )
570        .await;
571        match fetch_result {
572            Ok(Ok(output)) if output.status.success() => {
573                tracing::debug!(%repo, "fetched origin after task cleanup");
574            }
575            Ok(Ok(output)) => {
576                let stderr = String::from_utf8_lossy(&output.stderr);
577                tracing::debug!(%repo, err = %stderr, "git fetch skipped after task cleanup");
578            }
579            Ok(Err(e)) => {
580                tracing::debug!(%repo, err = %e, "git fetch failed after task cleanup");
581            }
582            Err(_) => {
583                tracing::debug!(%repo, "git fetch timed out after 60s");
584            }
585        }
586    }
587
588    Ok(())
589}
590
591/// Cleanup a single task's worktree and branches with explicit janitor options.
592///
593/// Returns `Ok(true)` when cleanup was actually performed and the task was
594/// marked as cleaned.  Returns `Ok(false)` when cleanup was intentionally
595/// skipped (TTL guard, active tmux session guard, or nothing on disk to remove).
596/// Returns `Err` on unexpected failures (e.g. cannot resolve the repo root).
597pub(crate) async fn cleanup_task_worktree_with_opts(
598    task_id: &str,
599    repo: &str,
600    store: &Arc<TaskStore>,
601    opts: &JanitorOptions,
602) -> anyhow::Result<bool> {
603    cleanup_task_worktree_with_opts_impl(task_id, repo, store, opts, false).await
604}
605
606async fn cleanup_task_worktree_with_opts_impl(
607    task_id: &str,
608    repo: &str,
609    store: &Arc<TaskStore>,
610    opts: &JanitorOptions,
611    known_pr_merged: bool,
612) -> anyhow::Result<bool> {
613    let (worktree, branch, keep_remote_branch, merged, external_id, pr_merged) =
614        store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
615            .await
616            .map_or((None, None, false, false, None, false), |t| {
617                // Blocked tasks with a PR should keep their remote branch so the
618                // PR stays open for human review.
619                let keep = t.status == TaskStatus::Blocked && t.pr_number.is_some();
620                let merged = t.status == TaskStatus::Done || known_pr_merged;
621                (
622                    Some(t.worktree),
623                    Some(t.branch),
624                    keep,
625                    merged,
626                    t.external_id,
627                    // A squash leaves the branch commits on no ref, so skip rescue when the PR merged
628                    known_pr_merged || (t.status == TaskStatus::Done && t.pr_number.is_some()),
629                )
630            });
631
632    let worktree_path = worktree.as_ref().map(PathBuf::from);
633
634    // Get worktrees base path
635    let worktrees_base = home::worktrees_dir().unwrap_or_else(|_| PathBuf::from(".orch/worktrees"));
636
637    // Determine which directory to remove.
638    //
639    // Priority:
640    //   1. stored "worktree" path, if it exists on disk.
641    //   2. Construct from worktrees_base + project name + branch, if it exists.
642    let worktree_to_remove = if let Some(ref wt) = worktree_path {
643        if match fs::try_exists(wt).await {
644            Ok(true) => true,
645            Ok(false) => false,
646            Err(e) => {
647                tracing::warn!(task_id, worktree = %wt.display(), err = %e, "try_exists failed on stored worktree path — trying branch-based fallback");
648                false
649            }
650        } {
651            Some(wt.clone())
652        } else {
653            // The recorded worktree path doesn't exist — try reconstructing from branch.
654            if let Some(ref b) = branch {
655                let project = repo
656                    .rsplit('/')
657                    .next()
658                    .unwrap_or(repo)
659                    .trim_end_matches(".git");
660                let candidate = worktrees_base.join(project).join(b);
661                if match fs::try_exists(&candidate).await {
662                    Ok(true) => true,
663                    Ok(false) => false,
664                    Err(e) => {
665                        tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
666                        false
667                    }
668                } {
669                    Some(candidate)
670                } else {
671                    None
672                }
673            } else {
674                None
675            }
676        }
677    } else if let Some(ref b) = branch {
678        // No worktree path in store — try branch-based path.
679        let project = repo
680            .rsplit('/')
681            .next()
682            .unwrap_or(repo)
683            .trim_end_matches(".git");
684        let candidate = worktrees_base.join(project).join(b);
685        if match fs::try_exists(&candidate).await {
686            Ok(true) => true,
687            Ok(false) => false,
688            Err(e) => {
689                tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
690                false
691            }
692        } {
693            Some(candidate)
694        } else {
695            None
696        }
697    } else {
698        None
699    };
700
701    let mut did_clean = false;
702
703    // If there's nothing to remove (no worktree path and no branch),
704    // treat as a no-op and return Ok(false) without attempting to
705    // resolve the repo root. Resolving the repo root can fail for
706    // projects not registered in config and that should not make
707    // cleanup a hard error when there is nothing to do.
708    let branch_nonempty = branch.as_ref().filter(|b| !b.is_empty());
709    if worktree_to_remove.is_none() && branch_nonempty.is_none() {
710        // If the stored worktree path is non-empty but doesn't exist on disk,
711        // the worktree is already gone — mark it cleaned so we stop retrying.
712        if worktree_path
713            .as_ref()
714            .is_some_and(|p| !p.as_os_str().is_empty())
715        {
716            tracing::debug!(
717                task_id,
718                worktree = ?worktree_path,
719                "stored worktree path no longer exists on disk — marking cleaned"
720            );
721            if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
722                && let Err(e) = store.mark_cleaned(store_id).await
723            {
724                tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned in store — will retry");
725            }
726        }
727        return Ok(false);
728    }
729
730    if let Some(wt) = worktree_to_remove {
731        // TTL guard: skip if the worktree directory is too young.
732        if let Some(age_hours) = worktree_age_hours(&wt).await
733            && age_hours < opts.ttl_hours
734        {
735            tracing::debug!(
736                task_id,
737                worktree = %wt.display(),
738                age_hours,
739                ttl_hours = opts.ttl_hours,
740                "worktree too young for cleanup, skipping"
741            );
742            return Ok(false);
743        }
744
745        // Tmux guard: skip if any active pane still has its cwd inside this worktree.
746        if is_worktree_in_active_session(&wt).await {
747            tracing::warn!(
748                task_id,
749                worktree = %wt.display(),
750                "worktree is referenced by an active tmux session — skipping cleanup"
751            );
752            return Ok(false);
753        }
754
755        // Validate git metadata before attempting any git operations.
756        // If the .git file points to a deleted gitdir the worktree is
757        // unrecoverable — force-remove the directory and skip git commands.
758        let gitdir_valid = worktree::validate_worktree_gitdir(&wt).await;
759
760        if opts.dry_run {
761            tracing::info!(
762                task_id,
763                worktree = %wt.display(),
764                branch = ?branch,
765                gitdir_valid,
766                "[dry-run] would remove worktree and branch"
767            );
768        } else if !gitdir_valid {
769            tracing::warn!(
770                task_id,
771                worktree = %wt.display(),
772                "worktree has broken git metadata — force-removing directory without git operations"
773            );
774            match fs::remove_dir_all(&wt).await {
775                Ok(()) => {
776                    did_clean = true;
777                }
778                Err(e) => {
779                    tracing::warn!(
780                        task_id,
781                        worktree = %wt.display(),
782                        err = %e,
783                        "failed to force-remove broken worktree directory"
784                    );
785                }
786            }
787        } else {
788            tracing::info!(task_id, worktree = %wt.display(), "removing worktree");
789            // Derive repo root from the worktree itself (for cross-project support).
790            // If the worktree belongs to a different project than the current repo
791            // context (e.g., internal task with worktree in another project), we can
792            // still find the repo root by examining the worktree's .git file.
793            let repo_root = match resolve_repo_root_from_worktree(&wt).await {
794                Ok(root) => root,
795                Err(_) => resolve_repo_root(repo).await?,
796            };
797            if let Some(br) = branch.as_deref().filter(|b| !b.is_empty())
798                && !pr_merged
799                && !rescue_and_notify(
800                    task_id,
801                    repo,
802                    external_id.as_deref(),
803                    Some(&wt),
804                    br,
805                    Path::new(&repo_root),
806                )
807                .await
808            {
809                return Ok(false);
810            }
811            if merged {
812                cow::warm_from_merged(
813                    Path::new(&repo_root),
814                    &worktree::project_name(Path::new(&repo_root)),
815                    &wt,
816                )
817                .await;
818            }
819            let removed = remove_worktree_and_branch_unchecked(
820                task_id,
821                &wt,
822                branch.as_deref(),
823                Path::new(&repo_root),
824                keep_remote_branch,
825            )
826            .await;
827            if removed {
828                did_clean = true;
829            } else {
830                tracing::warn!(
831                    task_id,
832                    worktree = %wt.display(),
833                    "worktree directory still exists after removal attempt — not marking as cleaned"
834                );
835            }
836        }
837    } else if let Some(ref br) = branch {
838        // Worktree directory is already gone, but the branch may still exist
839        // on the remote. Delete it to avoid orphaned branches.
840        if !opts.dry_run && !keep_remote_branch {
841            tracing::debug!(task_id, branch = %br, "no worktree on disk, cleaning up branch only");
842            // Resolve repo root lazily — branch cleanup needs it.
843            let repo_root = resolve_repo_root(repo).await?;
844            if !pr_merged
845                && !rescue_and_notify(
846                    task_id,
847                    repo,
848                    external_id.as_deref(),
849                    None,
850                    br,
851                    Path::new(&repo_root),
852                )
853                .await
854            {
855                return Ok(false);
856            }
857            delete_branches(task_id, br, Path::new(&repo_root)).await;
858            did_clean = true;
859        }
860    }
861
862    if did_clean {
863        // Mark as cleaned in store so we don't retry next tick.
864        // If resolve_task_id returns None the task is not (or no longer) in
865        // the store; that is fine — we already did the on-disk work.
866        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
867            && let Err(e) = store.mark_cleaned(store_id).await
868        {
869            tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned — will retry");
870            // Don't log activity; retry next tick will do the actual cleanup
871            return Ok(did_clean);
872        }
873        store_log_activity(
874            &Some(Arc::clone(store)),
875            repo,
876            task_id,
877            "branch_delete",
878            None,
879            None,
880            None,
881            None,
882            Some(&serde_json::json!({
883                "worktree": worktree_path.map(|p| p.display().to_string()),
884                "branch": branch,
885                "keep_remote_branch": keep_remote_branch,
886            })),
887        )
888        .await;
889    }
890
891    Ok(did_clean)
892}
893
894/// Result of saving a task branch's unpushed work before cleanup deletes it
895#[derive(Debug, PartialEq, Eq)]
896pub(crate) enum Rescue {
897    /// Every commit is on a remote ref already
898    Clean,
899    /// The commits were pushed to `rescue/<branch>`
900    Pushed {
901        branch: String,
902        commits: Vec<String>,
903    },
904    /// Git failed, so the worktree and branch must stay
905    Failed,
906}
907
908async fn git_in(dir: &Path, args: &[&str]) -> Option<Output> {
909    git_ops::git()
910        .await
911        .arg("-C")
912        .arg(dir)
913        .args(args)
914        .output_with_context()
915        .await
916        .ok()
917}
918
919/**
920 * Cleanup deletes the worktree and the local branch, which can be the only refs to commits the
921 * agent made before its run ended without a push (#3999). Uncommitted changes are committed
922 * first, then everything not on a remote ref goes to `rescue/<branch>`
923 **/
924pub(crate) async fn rescue_unpushed_work(
925    task_id: &str,
926    wt: Option<&Path>,
927    br: &str,
928    repo_root: &Path,
929) -> Rescue {
930    if let Some(wt) = wt
931        && fs::try_exists(wt.join(".git")).await.unwrap_or(false)
932    {
933        let Some(status) = git_in(wt, &["status", "--porcelain"]).await else {
934            tracing::warn!(task_id, branch = %br, "cannot read worktree status, keeping worktree");
935            return Rescue::Failed;
936        };
937        if !status.status.success() {
938            tracing::warn!(task_id, branch = %br, "git status failed, keeping worktree");
939            return Rescue::Failed;
940        }
941        if !status.stdout.is_empty() {
942            let committed = git_in(wt, &["add", "-A"])
943                .await
944                .is_some_and(|o| o.status.success())
945                && git_in(
946                    wt,
947                    &[
948                        "-c",
949                        "user.name=orch",
950                        "-c",
951                        "user.email=orch@localhost",
952                        "commit",
953                        "-m",
954                        "rescue: uncommitted changes saved by orch cleanup",
955                    ],
956                )
957                .await
958                .is_some_and(|o| o.status.success());
959            if !committed {
960                tracing::warn!(task_id, branch = %br, "cannot commit uncommitted changes, keeping worktree");
961                return Rescue::Failed;
962            }
963        }
964    }
965
966    let local_ref = format!("refs/heads/{br}");
967    let exists = git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
968        .await
969        .is_some_and(|o| o.status.success());
970    if !exists {
971        return Rescue::Clean;
972    }
973    // Commits on the local default branch survive the branch delete, so they do not count
974    let default_ref = format!(
975        "refs/heads/{}",
976        worktree::detect_default_branch(repo_root).await
977    );
978    // A bare clone has no remote-tracking refs, so ask the remote for the branch tip
979    let remote_tip = git_in(repo_root, &["ls-remote", "origin", &local_ref])
980        .await
981        .filter(|o| o.status.success())
982        .and_then(|o| {
983            String::from_utf8_lossy(&o.stdout)
984                .split_whitespace()
985                .next()
986                .map(str::to_string)
987        });
988    // A tip someone else pushed may be missing locally, and rev-list fails on an unknown object
989    let remote_tip = match remote_tip {
990        Some(tip) => {
991            let known = git_in(repo_root, &["cat-file", "-e", &format!("{tip}^{{commit}}")])
992                .await
993                .is_some_and(|o| o.status.success());
994            known.then_some(tip)
995        }
996        None => None,
997    };
998    let mut args = vec!["rev-list", "--oneline", &local_ref, "--not", "--remotes"];
999    if let Some(tip) = remote_tip.as_deref() {
1000        args.push(tip);
1001    }
1002    let default_exists = default_ref != local_ref
1003        && git_in(
1004            repo_root,
1005            &["rev-parse", "--verify", "--quiet", &default_ref],
1006        )
1007        .await
1008        .is_some_and(|o| o.status.success());
1009    if default_exists {
1010        args.push(&default_ref);
1011    }
1012    let Some(list) = git_in(repo_root, &args)
1013        .await
1014        .filter(|o| o.status.success())
1015    else {
1016        tracing::warn!(task_id, branch = %br, "cannot list unpushed commits, keeping worktree and branch");
1017        return Rescue::Failed;
1018    };
1019    let commits: Vec<String> = String::from_utf8_lossy(&list.stdout)
1020        .lines()
1021        .map(str::to_string)
1022        .collect();
1023    if commits.is_empty() {
1024        return Rescue::Clean;
1025    }
1026
1027    let rescue = format!("rescue/{br}");
1028    let refspec = format!("{local_ref}:refs/heads/{rescue}");
1029    let pushed = git_in(
1030        repo_root,
1031        &["-c", "lfs.locksverify=false", "push", "origin", &refspec],
1032    )
1033    .await;
1034    if pushed.is_some_and(|o| o.status.success()) {
1035        tracing::warn!(task_id, branch = %br, rescue = %rescue, commits = commits.len(), "pushed unpushed commits to rescue branch");
1036        Rescue::Pushed {
1037            branch: rescue,
1038            commits,
1039        }
1040    } else {
1041        tracing::warn!(task_id, branch = %br, rescue = %rescue, "rescue push failed, keeping worktree and branch");
1042        Rescue::Failed
1043    }
1044}
1045
1046fn rescue_comment(rescue_branch: &str, commits: &[String]) -> String {
1047    let list = commits
1048        .iter()
1049        .map(|c| format!("- `{c}`"))
1050        .collect::<Vec<_>>()
1051        .join("\n");
1052    format!(
1053        "Cleanup found commits on this task's branch that were never pushed. They are saved on `{rescue_branch}`:\n\n{list}"
1054    )
1055}
1056
1057/// Save unpushed work and tell the task about it. Returns false when cleanup must not go on
1058async fn rescue_and_notify(
1059    task_id: &str,
1060    repo: &str,
1061    external_id: Option<&str>,
1062    wt: Option<&Path>,
1063    br: &str,
1064    repo_root: &Path,
1065) -> bool {
1066    match rescue_unpushed_work(task_id, wt, br, repo_root).await {
1067        Rescue::Clean => true,
1068        Rescue::Failed => false,
1069        Rescue::Pushed { branch, commits } => {
1070            if let Some(number) = external_id.filter(|n| n.parse::<u64>().is_ok()) {
1071                match GhHttp::new() {
1072                    Ok(gh) => {
1073                        if let Err(e) = gh
1074                            .add_comment(repo, number, &rescue_comment(&branch, &commits))
1075                            .await
1076                        {
1077                            tracing::warn!(task_id, err = %e, "failed to comment on rescue branch");
1078                        }
1079                    }
1080                    Err(e) => {
1081                        tracing::warn!(task_id, err = %e, "cannot comment on rescue branch");
1082                    }
1083                }
1084            }
1085            true
1086        }
1087    }
1088}
1089
1090/// Remove a git worktree directory and its local + remote branches.
1091/// When `keep_remote_branch` is true, only the worktree and local branch are
1092/// removed — the remote branch stays so that any linked PR remains open
1093/// (used for blocked tasks awaiting human review).
1094/// Returns `true` if the worktree directory is gone after the call
1095/// (either we removed it or it was already absent).
1096pub(crate) async fn remove_worktree_and_branch(
1097    task_id: &str,
1098    wt: &Path,
1099    branch: Option<&str>,
1100    repo_root: &Path,
1101    keep_remote_branch: bool,
1102) -> bool {
1103    remove_worktree_and_branch_for(task_id, wt, branch, repo_root, keep_remote_branch, false).await
1104}
1105
1106/// Same as `remove_worktree_and_branch`, but a merged PR skips the rescue. A squash leaves the
1107/// branch commits on no ref, so the rescue would push a branch for work that already landed
1108pub(crate) async fn remove_worktree_and_branch_for(
1109    task_id: &str,
1110    wt: &Path,
1111    branch: Option<&str>,
1112    repo_root: &Path,
1113    keep_remote_branch: bool,
1114    pr_merged: bool,
1115) -> bool {
1116    if !pr_merged
1117        && let Some(br) = branch.filter(|b| !b.is_empty())
1118        && rescue_unpushed_work(task_id, Some(wt), br, repo_root).await == Rescue::Failed
1119    {
1120        return false;
1121    }
1122
1123    remove_worktree_and_branch_unchecked(task_id, wt, branch, repo_root, keep_remote_branch).await
1124}
1125
1126/// The caller has already rescued the branch or knows its PR merged
1127async fn remove_worktree_and_branch_unchecked(
1128    task_id: &str,
1129    wt: &Path,
1130    branch: Option<&str>,
1131    repo_root: &Path,
1132    keep_remote_branch: bool,
1133) -> bool {
1134    let wt_str = wt.to_string_lossy().to_string();
1135    let repo_root_str = repo_root.to_string_lossy();
1136
1137    // Remove worktree FIRST, then delete the branch.
1138    // Git refuses to remove a worktree if its branch is already deleted.
1139    let remove_result = Command::new("git")
1140        .args([
1141            "-C",
1142            repo_root_str.as_ref(),
1143            "worktree",
1144            "remove",
1145            &wt_str,
1146            "--force",
1147        ])
1148        .output_with_context()
1149        .await;
1150
1151    match remove_result {
1152        Ok(output) if output.status.success() => {
1153            const MAX_RETRIES: usize = 3;
1154            const RETRY_DELAY_MS: u64 = 2000;
1155            tracing::info!(task_id, "worktree removed (git metadata)");
1156            // `git worktree remove` only removes git metadata — the physical directory
1157            // with working files may remain. Clean it up with retry logic, since macOS
1158            // sometimes needs a moment to release file handles from a recently-exited
1159            // process. Retrying with a delay addresses "Directory not empty" errors.
1160            let mut removed = false;
1161            for attempt in 0..MAX_RETRIES {
1162                match fs::remove_dir_all(wt).await {
1163                    Ok(()) => {
1164                        removed = true;
1165                        break;
1166                    }
1167                    Err(e) if e.kind() == ErrorKind::NotFound => {
1168                        // Directory was already removed by git or another cleanup pass.
1169                        removed = true;
1170                        break;
1171                    }
1172                    Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => {
1173                        if attempt < MAX_RETRIES - 1 {
1174                            tracing::debug!(
1175                                task_id,
1176                                path = %wt.display(),
1177                                attempt = attempt + 1,
1178                                "worktree directory not empty — retrying after delay"
1179                            );
1180                            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
1181                        } else {
1182                            tracing::warn!(
1183                                task_id,
1184                                path = %wt.display(),
1185                                err = %e,
1186                                "failed to remove worktree directory after {} attempts (Directory not empty)",
1187                                MAX_RETRIES
1188                            );
1189                        }
1190                    }
1191                    Err(e) => {
1192                        tracing::warn!(task_id, path = %wt.display(), err = %e, "failed to remove worktree directory");
1193                        break;
1194                    }
1195                }
1196            }
1197            if removed {
1198                tracing::info!(task_id, path = %wt.display(), "worktree physical directory removed");
1199            } else {
1200                tracing::warn!(
1201                    task_id,
1202                    path = %wt.display(),
1203                    "worktree directory still present after git worktree remove"
1204                );
1205            }
1206        }
1207        Ok(output) => {
1208            let stderr = String::from_utf8_lossy(&output.stderr);
1209            // "is not a working tree" means git's metadata is stale — the path
1210            // exists on disk but is no longer registered as a worktree. Treat
1211            // this as an already-gone worktree: physically remove the directory
1212            // and prune stale git metadata. Downgrade to debug — this is an
1213            // expected idempotency case, not a real error.
1214            if stderr.contains("is not a working tree") {
1215                tracing::debug!(
1216                    task_id,
1217                    path = %wt.display(),
1218                    "path is not a registered worktree (stale metadata) — removing directory and pruning"
1219                );
1220                if let Err(e) = fs::remove_dir_all(wt).await {
1221                    // Only warn if the directory actually still exists after removal attempt.
1222                    if match fs::try_exists(wt).await {
1223                        Ok(true) => true,
1224                        Ok(false) => false,
1225                        Err(e) => {
1226                            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree does not exist");
1227                            false
1228                        }
1229                    } {
1230                        tracing::warn!(
1231                            task_id,
1232                            path = %wt.display(),
1233                            err = %e,
1234                            "failed to remove stale worktree directory"
1235                        );
1236                    }
1237                } else {
1238                    tracing::debug!(task_id, path = %wt.display(), "stale worktree directory removed");
1239                }
1240                // Prune stale worktree entries from git metadata so the warning
1241                // stops recurring on future `git worktree list` / cleanup cycles.
1242                if let Err(e) = Command::new("git")
1243                    .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1244                    .output_with_context()
1245                    .await
1246                {
1247                    tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1248                }
1249            } else {
1250                tracing::warn!(task_id, err = %stderr, "failed to remove worktree");
1251            }
1252        }
1253        Err(e) => {
1254            tracing::warn!(task_id, err = %e, "failed to remove worktree");
1255        }
1256    }
1257
1258    // Prune stale .git/worktrees/ entries left behind after removal.
1259    if let Err(e) = Command::new("git")
1260        .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1261        .output_with_context()
1262        .await
1263    {
1264        tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1265    }
1266
1267    // Delete local and remote branch from the main repo root (worktree is already gone).
1268    // When keep_remote_branch is set, only delete the local branch — the remote stays
1269    // so any linked PR remains open for human review.
1270    if let Some(br) = branch {
1271        if keep_remote_branch {
1272            delete_local_branch(task_id, br, repo_root).await;
1273        } else {
1274            delete_branches(task_id, br, repo_root).await;
1275        }
1276    }
1277
1278    // Definitively check: is the worktree directory actually gone?
1279    match fs::try_exists(wt).await {
1280        Ok(false) => true, // confirmed gone
1281        Ok(true) => false, // still there
1282        Err(e) => {
1283            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree still present");
1284            false // conservative: don't claim success
1285        }
1286    }
1287}
1288
1289/// Commits reachable from `rev` and from none of `others`. `None` when git cannot answer
1290async fn commits_outside(repo_root: &Path, rev: &str, others: &[&str]) -> Option<u64> {
1291    let mut args = vec![
1292        "rev-list".to_string(),
1293        "--count".to_string(),
1294        rev.to_string(),
1295    ];
1296    for other in others {
1297        let exists = Command::new("git")
1298            .args([
1299                "-C",
1300                &repo_root.to_string_lossy(),
1301                "rev-parse",
1302                "--verify",
1303                "--quiet",
1304                other,
1305            ])
1306            .output_with_context()
1307            .await
1308            .is_ok_and(|o| o.status.success());
1309        if exists {
1310            args.push("--not".to_string());
1311            args.push(other.to_string());
1312        }
1313    }
1314    let output = Command::new("git")
1315        .arg("-C")
1316        .arg(repo_root)
1317        .args(&args)
1318        .output_with_context()
1319        .await
1320        .ok()?;
1321    if !output.status.success() {
1322        return None;
1323    }
1324    String::from_utf8_lossy(&output.stdout).trim().parse().ok()
1325}
1326
1327/// Delete the local branch only when origin or the default branch holds every commit on it
1328async fn delete_local_branch(task_id: &str, br: &str, repo_root: &Path) {
1329    let default_branch = worktree::detect_default_branch(repo_root).await;
1330    let remote_ref = format!("refs/remotes/origin/{br}");
1331    let default_ref = format!("refs/remotes/origin/{default_branch}");
1332    let local_ref = format!("refs/heads/{br}");
1333    let local_default_ref = format!("refs/heads/{default_branch}");
1334    let rescue_ref = format!("refs/remotes/origin/rescue/{br}");
1335    match commits_outside(
1336        repo_root,
1337        &local_ref,
1338        &[&remote_ref, &default_ref, &local_default_ref, &rescue_ref],
1339    )
1340    .await
1341    {
1342        Some(0) => {}
1343        Some(n) => {
1344            tracing::warn!(task_id, branch = %br, unpushed = n, "local branch holds commits origin does not have, keeping it");
1345            return;
1346        }
1347        None => {
1348            tracing::debug!(task_id, branch = %br, "local branch missing or unreadable, nothing to delete");
1349            return;
1350        }
1351    }
1352    let output = Command::new("git")
1353        .args(["-C", &repo_root.to_string_lossy(), "branch", "-D", br])
1354        .output_with_context()
1355        .await;
1356    match output {
1357        Ok(o) if o.status.success() => {
1358            tracing::info!(task_id, branch = %br, "local branch deleted");
1359        }
1360        Ok(o) => {
1361            let stderr = String::from_utf8_lossy(&o.stderr);
1362            tracing::debug!(task_id, err = %stderr, "local branch delete skipped (may not exist)");
1363        }
1364        Err(e) => {
1365            tracing::warn!(task_id, err = %e, "failed to delete local branch");
1366        }
1367    }
1368}
1369
1370/**
1371 * Delete local and remote branches for a task, but never the only copy of a commit.
1372 * The remote branch goes only when the default branch already contains all of it, which
1373 * covers fast-forward and merge-commit merges. A squash-merged branch stays, and GitHub's
1374 * "automatically delete head branches" removes it. Every other caller (startup rebase
1375 * failure, closed-issue reconciliation, orphan sweeps) may hold unmerged work (#3781)
1376 **/
1377pub(crate) async fn delete_branches(task_id: &str, br: &str, repo_root: &Path) {
1378    if br == SCREENSHOTS_BRANCH {
1379        return;
1380    }
1381    let default_branch = worktree::detect_default_branch(repo_root).await;
1382    let _ = git_ops::git()
1383        .await
1384        .args([
1385            "-C",
1386            &repo_root.to_string_lossy(),
1387            "fetch",
1388            "origin",
1389            &default_branch,
1390            br,
1391        ])
1392        .output_with_context()
1393        .await;
1394
1395    delete_local_branch(task_id, br, repo_root).await;
1396
1397    let remote_ref = format!("refs/remotes/origin/{br}");
1398    let default_ref = format!("refs/remotes/origin/{default_branch}");
1399    match commits_outside(repo_root, &remote_ref, &[&default_ref]).await {
1400        Some(0) => {}
1401        Some(n) => {
1402            tracing::info!(task_id, branch = %br, unmerged = n, "remote branch has commits the default branch lacks, keeping it");
1403            return;
1404        }
1405        None => {
1406            tracing::debug!(task_id, branch = %br, "remote branch missing or unreadable, nothing to delete");
1407            return;
1408        }
1409    }
1410
1411    let remote_delete = git_ops::git()
1412        .await
1413        .args([
1414            "-C",
1415            &repo_root.to_string_lossy(),
1416            "-c",
1417            "lfs.locksverify=false",
1418            "push",
1419            "origin",
1420            "--delete",
1421            br,
1422        ])
1423        .output_with_context()
1424        .await;
1425
1426    match remote_delete {
1427        Ok(output) if output.status.success() => {
1428            tracing::info!(task_id, branch = %br, "remote branch deleted");
1429            git_ops::clear_push_lease(repo_root, br).await;
1430        }
1431        Ok(output) => {
1432            let stderr = String::from_utf8_lossy(&output.stderr);
1433            tracing::debug!(task_id, err = %stderr, "remote branch delete skipped");
1434        }
1435        Err(e) => {
1436            tracing::warn!(task_id, err = %e, "failed to delete remote branch");
1437        }
1438    }
1439}
1440
1441/// Returns `true` if any active tmux pane has its cwd inside `worktree`.
1442///
1443/// Uses `tmux list-panes -a -F '#{pane_current_path}'` to enumerate all
1444/// pane working directories across every session. If tmux is not running
1445/// (command fails), we conservatively return `false` so cleanup can proceed.
1446async fn is_worktree_in_active_session(worktree: &Path) -> bool {
1447    let output = Command::new("tmux")
1448        .args(["list-panes", "-a", "-F", "#{pane_current_path}"])
1449        .output()
1450        .await;
1451
1452    match output {
1453        Ok(o) if o.status.success() => {
1454            let worktree_str = worktree.to_string_lossy();
1455            String::from_utf8_lossy(&o.stdout).lines().any(|line| {
1456                let pane_path = line.trim();
1457                // Match exact path or any path that starts with worktree + separator.
1458                pane_path == worktree_str.as_ref()
1459                    || pane_path.starts_with(&format!("{worktree_str}/"))
1460            })
1461        }
1462        // tmux not running or no sessions — treat as safe to clean.
1463        _ => false,
1464    }
1465}
1466
1467/// Returns how many hours old the worktree directory is, based on its mtime.
1468///
1469/// Returns `None` if the directory does not exist or the mtime cannot be read.
1470async fn worktree_age_hours(worktree: &Path) -> Option<u64> {
1471    // Use tokio::fs to avoid blocking the async runtime when reading metadata.
1472    let metadata = fs::metadata(worktree).await.ok()?;
1473    let modified = metadata.modified().ok()?;
1474    let age = SystemTime::now().duration_since(modified).ok()?;
1475    Some(age.as_secs() / 3600)
1476}
1477
1478/// Resolve the repo root for an orphaned worktree and remove it.
1479///
1480/// Used by `orch service doctor --fix` and `orch service prune` to clean up worktrees that are
1481/// no longer owned by any task. Returns `true` if the worktree directory is gone
1482/// after the call.
1483pub(crate) async fn cleanup_orphaned_worktree(task_id: &str, wt: &Path) -> bool {
1484    let repo_root = match resolve_repo_root_for_orphaned_worktree(wt).await {
1485        Ok(root) => root,
1486        Err(e) => {
1487            tracing::warn!(
1488                worktree = %wt.display(),
1489                error = %e,
1490                "cannot resolve repo root for orphaned worktree"
1491            );
1492            return false;
1493        }
1494    };
1495    remove_worktree_and_branch(task_id, wt, None, Path::new(&repo_root), false).await
1496}
1497
1498/// Resolve the main git repository root for an orphaned worktree.
1499///
1500/// First tries to parse the `.git` file's `gitdir:` pointer (two levels up from
1501/// the pointer gives the bare repo root, which is the orch convention). Falls back
1502/// to looking up registered projects by directory name when the `.git` file is
1503/// absent or unparseable.
1504pub(crate) async fn resolve_repo_root_for_orphaned_worktree(wt: &Path) -> anyhow::Result<String> {
1505    let git_file = wt.join(".git");
1506    if fs::try_exists(&git_file).await.unwrap_or(false)
1507        && let Ok(content) = fs::read_to_string(&git_file).await
1508    {
1509        for line in content.lines() {
1510            if let Some(gitdir) = line.strip_prefix("gitdir: ") {
1511                // gitdir points to: <repo>/.git/worktrees/<name> (non-bare)
1512                //                or <repo>.git/worktrees/<name>   (bare)
1513                // Two levels up reaches the .git dir / bare repo root.
1514                let gitdir_path = PathBuf::from(gitdir);
1515                if let Some(repo_root) = gitdir_path.parent().and_then(|p| p.parent()) {
1516                    return Ok(repo_root.display().to_string());
1517                }
1518            }
1519        }
1520    }
1521
1522    // Fallback: look up registered projects by the project-name component of the
1523    // worktree path (format: ~/.orch/worktrees/<project>/<branch>).
1524    let worktrees_dir = home::worktrees_dir()?;
1525    if let Ok(relative) = wt.strip_prefix(&worktrees_dir)
1526        && let Some(project_name) = relative.components().next()
1527    {
1528        let project_name = project_name.as_os_str().to_string_lossy();
1529
1530        if let Ok(paths) = config::get_project_paths() {
1531            for path_str in &paths {
1532                let path = Path::new(path_str);
1533                if let Some(name) = path.file_name()
1534                    && name.to_string_lossy() == project_name
1535                {
1536                    return Ok(path_str.clone());
1537                }
1538            }
1539        }
1540
1541        // Try bare clone at ~/.orch/projects/<owner>/<project>.git
1542        let parts: Vec<&str> = project_name.split('/').collect();
1543        let bare = if parts.len() == 2 {
1544            home::projects_dir()
1545                .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1546                .unwrap_or_default()
1547        } else {
1548            return Err(anyhow::anyhow!(
1549                "cannot determine owner for single-segment project name '{project_name}' — \
1550                     bare repo location is ambiguous; register the project with 'owner/repo' format"
1551            ));
1552        };
1553        if fs::try_exists(&bare).await.unwrap_or(false) {
1554            return Ok(bare.display().to_string());
1555        }
1556    }
1557
1558    anyhow::bail!(
1559        "cannot resolve repo root for orphaned worktree at {} — tried .git parsing and project lookup",
1560        wt.display()
1561    )
1562}
1563
1564/// Resolve the main git repository root from a worktree's .git file.
1565///
1566/// For cross-project worktree cleanup, we cannot rely on `resolve_repo_root` because
1567/// the worktree may belong to a different project than the current repo context.
1568/// Instead, we extract the repo root from the worktree's `.git` file which contains
1569/// a `gitdir:` pointer to the main repo's git directory.
1570pub async fn resolve_repo_root_from_worktree(wt: &Path) -> anyhow::Result<String> {
1571    // For worktrees, .git is a file (not a directory) containing gitdir path
1572    let git_file = wt.join(".git");
1573    if !fs::try_exists(&git_file).await.unwrap_or(false) {
1574        anyhow::bail!(".git file not found in worktree at {}", wt.display());
1575    }
1576
1577    let content = fs::read_to_string(&git_file)
1578        .await
1579        .map_err(|e| anyhow::anyhow!("failed to read .git file: {e}"))?;
1580
1581    // Parse gitdir line: "gitdir: /path/to/repo/.git/worktrees/<name>"
1582    for line in content.lines() {
1583        if let Some(gitdir) = line.strip_prefix("gitdir:") {
1584            let gitdir = gitdir.trim();
1585            // The main repo is the parent of the .git directory
1586            // gitdir points to: <repo>/.git/worktrees/<name>
1587            // We need to go up 3 levels: worktrees/<name> -> .git -> repo
1588            let gitdir_path = Path::new(gitdir);
1589            if let Some(repo_root) = gitdir_path
1590                .parent() // worktrees/<name>
1591                .and_then(|p| p.parent()) // .git
1592                .and_then(|p| p.parent())
1593            // repo root
1594            {
1595                return Ok(repo_root.display().to_string());
1596            }
1597        }
1598    }
1599
1600    anyhow::bail!(
1601        "cannot resolve repo root from worktree .git file at {}",
1602        wt.display()
1603    )
1604}
1605
1606/// Resolve the main git repository root path for a project.
1607///
1608/// Looks up the local project path from config, then verifies it's a git repo.
1609/// This avoids relying on cwd (which is undefined under launchd services).
1610pub(crate) async fn resolve_repo_root(repo: &str) -> anyhow::Result<String> {
1611    // Look up the local path from registered projects
1612    let paths = config::get_project_paths().unwrap_or_default();
1613    for path_str in &paths {
1614        let path = Path::new(path_str);
1615        let orch_yml = path.join(".orch.yml");
1616        let legacy = path.join(".orchestrator.yml");
1617        let config_file = if fs::try_exists(&orch_yml).await.unwrap_or(false) {
1618            orch_yml
1619        } else if fs::try_exists(&legacy).await.unwrap_or(false) {
1620            legacy
1621        } else {
1622            continue;
1623        };
1624        if let Ok(content) = fs::read_to_string(&config_file).await
1625            && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1626            && let Some(r) = doc
1627                .get("gh")
1628                .and_then(|gh| gh.get("repo"))
1629                .and_then(|r| r.as_str())
1630            && r == repo
1631        {
1632            return Ok(path_str.clone());
1633        }
1634    }
1635
1636    // Fallback: try bare clone in ~/.orch/projects/<owner>/<repo>.git
1637    let parts: Vec<&str> = repo.split('/').collect();
1638    let bare = if parts.len() == 2 {
1639        home::projects_dir()
1640            .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1641            .unwrap_or_default()
1642    } else {
1643        PathBuf::new()
1644    };
1645    if fs::try_exists(&bare).await.unwrap_or(false) {
1646        return Ok(bare.display().to_string());
1647    }
1648
1649    anyhow::bail!(
1650        "cannot find local path for project {repo} — \
1651         checked {} registered project(s) and bare clone at {}",
1652        paths.len(),
1653        bare.display()
1654    )
1655}
1656
1657/// Resolve the project directory for a given repo string.
1658///
1659/// Replicates the logic from `TaskRunner::resolve_project_dir()` without requiring
1660/// a Runner instance. Used by the orphan scanner to enumerate active worktrees.
1661fn resolve_project_dir_for_repo(repo: &str) -> anyhow::Result<PathBuf> {
1662    // Check PROJECT_DIR env var first
1663    if let Ok(dir) = env::var("PROJECT_DIR")
1664        && !dir.is_empty()
1665    {
1666        return Ok(PathBuf::from(dir));
1667    }
1668
1669    // Look up registered projects in config
1670    if let Ok(paths) = config::get_project_paths() {
1671        for path_str in &paths {
1672            let path = Path::new(path_str);
1673            let orch_yml = path.join(".orch.yml");
1674            let legacy = path.join(".orchestrator.yml");
1675            let config_file = if Path::new(&orch_yml).exists() {
1676                orch_yml
1677            } else if Path::new(&legacy).exists() {
1678                legacy
1679            } else {
1680                continue;
1681            };
1682            if let Ok(content) = std_fs::read_to_string(&config_file)
1683                && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1684                && let Some(r) = doc
1685                    .get("gh")
1686                    .and_then(|gh| gh.get("repo"))
1687                    .and_then(|r| r.as_str())
1688                && r == repo
1689            {
1690                return Ok(path_str.clone().into());
1691            }
1692        }
1693    }
1694
1695    anyhow::bail!("project dir not found for repo: {repo}")
1696}
1697
1698/// Check for merged PRs and update task status accordingly.
1699///
1700/// Queries status:in_review and status:needs_review tasks, checks if their PR
1701/// is merged, and updates status to done if merged.
1702pub(crate) async fn check_merged_prs(
1703    backend: &Arc<dyn ExternalBackend>,
1704    repo: &str,
1705    store: &Arc<TaskStore>,
1706    task_manager: &Arc<TaskManager>,
1707    in_review_tasks: &[ReviewTaskSnapshot],
1708    needs_review_tasks: &[ReviewTaskSnapshot],
1709) -> anyhow::Result<()> {
1710    let mut branch_map: HashMap<String, String> = HashMap::new();
1711    let all_review_tasks: Vec<&ReviewTaskSnapshot> = in_review_tasks
1712        .iter()
1713        .chain(needs_review_tasks.iter())
1714        .collect();
1715
1716    for task in &all_review_tasks {
1717        if !task.stored.branch.is_empty() {
1718            branch_map.insert(task.external.id.0.clone(), task.stored.branch.clone());
1719        }
1720    }
1721
1722    tracing::debug!(
1723        count = all_review_tasks.len(),
1724        "checking review tasks for merged PRs"
1725    );
1726
1727    // Collect (task_id, branch) pairs — skip tasks with no branch recorded.
1728    // Use the pre-built branch map; fall back to store lookup only for tasks
1729    // not in the map (e.g. internal tasks, backend fallback path).
1730    let mut task_branches: Vec<(String, String)> = Vec::new();
1731    for task in all_review_tasks {
1732        let task_id = &task.external.id.0;
1733        let branch = if let Some(b) = branch_map.get(task_id) {
1734            b.clone()
1735        } else {
1736            match store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
1737                .await
1738                .map(|t| t.branch)
1739            {
1740                Some(b) if !b.is_empty() => b,
1741                _ => {
1742                    tracing::debug!(task_id, "no branch info, skipping PR check");
1743                    continue;
1744                }
1745            }
1746        };
1747        task_branches.push((task_id.clone(), branch));
1748    }
1749
1750    if task_branches.is_empty() {
1751        return Ok(());
1752    }
1753
1754    // Single GraphQL call for all branches — N REST calls → 1 GraphQL call.
1755    let branches: Vec<String> = task_branches.iter().map(|(_, b)| b.clone()).collect();
1756    let merged_map = match backend.batch_is_pr_merged(&branches).await {
1757        Ok(map) => map,
1758        Err(e) => {
1759            tracing::warn!(
1760                err = %e,
1761                branch_count = branches.len(),
1762                "batch PR merge check failed, retrying affected branches individually"
1763            );
1764            fallback_is_pr_merged_by_branch(backend, &branches).await
1765        }
1766    };
1767
1768    for (task_id, branch) in task_branches {
1769        let is_merged = merged_map.get(&branch).copied().unwrap_or(false);
1770        if !is_merged {
1771            continue;
1772        }
1773
1774        tracing::info!(task_id, branch = %branch, "PR merged, marking task complete");
1775
1776        let stored = store::opt_store_get_task(&Some(Arc::clone(store)), repo, &task_id).await;
1777        if let Some(ref stored) = stored {
1778            let external = store_task_to_external(stored);
1779            if ci_bootstrap::hold_after_merge(&external, store, repo).await {
1780                tracing::info!(
1781                    task_id,
1782                    branch = %branch,
1783                    "CI bootstrap PR merged, holding done until the ruleset exists"
1784                );
1785                if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1786                    tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1787                }
1788                continue;
1789            }
1790        }
1791
1792        let id = ExternalId(task_id.clone());
1793        if let Err(e) = task_manager.update_task_status(&id, Status::Done).await {
1794            tracing::warn!(task_id, err = %e, "failed to update status to done");
1795            continue;
1796        }
1797
1798        if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1799            tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1800        }
1801
1802        let comment = format!("PR merged, marking task complete{}", engine::orch_footer());
1803        if let Err(e) = backend.post_comment(&id, &comment).await {
1804            tracing::warn!(task_id, err = %e, "failed to post comment");
1805        }
1806    }
1807
1808    Ok(())
1809}
1810
1811async fn fallback_is_pr_merged_by_branch(
1812    backend: &Arc<dyn ExternalBackend>,
1813    branches: &[String],
1814) -> HashMap<String, bool> {
1815    let futs: Vec<_> = branches
1816        .iter()
1817        .map(|branch| {
1818            let backend = Arc::clone(backend);
1819            let branch = branch.clone();
1820            async move {
1821                match backend.is_pr_merged(&branch).await {
1822                    Ok(is_merged) => Some((branch, is_merged)),
1823                    Err(err) => {
1824                        tracing::warn!(
1825                            branch,
1826                            err = %err,
1827                            "fallback PR merge check failed for branch"
1828                        );
1829                        None
1830                    }
1831                }
1832            }
1833        })
1834        .collect();
1835
1836    future::join_all(futs).await.into_iter().flatten().collect()
1837}
1838
1839#[cfg(test)]
1840mod tests {
1841    use super::*;
1842    use crate::engine::tasks;
1843    use crate::store::{NewTask, TaskStore};
1844    use async_trait::async_trait;
1845    use std::collections::HashMap;
1846    use std::process;
1847    use std::sync::Mutex;
1848    use std::sync::atomic::{AtomicUsize, Ordering};
1849
1850    // worktree_age_hours
1851
1852    #[tokio::test]
1853    async fn worktree_age_hours_returns_none_for_missing_dir() {
1854        let tmp = tempfile::tempdir().unwrap();
1855        let missing = tmp.path().join("does-not-exist");
1856        assert!(worktree_age_hours(&missing).await.is_none());
1857    }
1858
1859    #[tokio::test]
1860    async fn worktree_age_hours_returns_some_for_existing_dir() {
1861        let tmp = tempfile::tempdir().unwrap();
1862        // A freshly created directory should have age ~0 hours.
1863        let age = worktree_age_hours(tmp.path()).await;
1864        assert!(age.is_some());
1865        assert_eq!(age.unwrap(), 0, "freshly created dir should have age 0h");
1866    }
1867
1868    // JanitorOptions::from_config defaults
1869
1870    #[test]
1871    fn janitor_options_default_values() {
1872        let opts = JanitorOptions::default();
1873        assert_eq!(opts.ttl_hours, 24);
1874        assert!(!opts.dry_run);
1875    }
1876
1877    // cleanup_task_worktree_with_opts TTL guard (skips if git missing)
1878
1879    /// Create a minimal git repo + worktree; return (tmp, worktree_dir).
1880    fn setup_test_repo() -> Option<(tempfile::TempDir, PathBuf)> {
1881        let tmp = tempfile::tempdir().ok()?;
1882
1883        // Create a real git repo (not bare) so we can add worktrees.
1884        let repo_dir = tmp.path().join("repo.git");
1885        std_fs::create_dir_all(&repo_dir).ok()?;
1886
1887        let init = process::Command::new("git")
1888            .args([
1889                "init",
1890                "--initial-branch=main",
1891                repo_dir
1892                    .to_str()
1893                    .expect("test repo path contains non-UTF-8 characters"),
1894            ])
1895            .output()
1896            .ok()?;
1897        if !init.status.success() {
1898            // Older git may not have --initial-branch; fall back.
1899            let _ = process::Command::new("git")
1900                .args([
1901                    "init",
1902                    repo_dir
1903                        .to_str()
1904                        .expect("test repo path contains non-UTF-8 characters"),
1905                ])
1906                .output()
1907                .ok()?;
1908        }
1909
1910        // Create an initial commit so the repo has a HEAD.
1911        let readme = repo_dir.join("README.md");
1912        std_fs::write(&readme, "# test").ok()?;
1913        process::Command::new("git")
1914            .args(["-C", repo_dir.to_str()?, "add", "."])
1915            .output()
1916            .ok()?;
1917        process::Command::new("git")
1918            .args([
1919                "-C",
1920                repo_dir.to_str()?,
1921                "-c",
1922                "user.email=test@test.com",
1923                "-c",
1924                "user.name=Test",
1925                "-c",
1926                "commit.gpgsign=false",
1927                "commit",
1928                "--allow-empty",
1929                "-m",
1930                "init",
1931            ])
1932            .output()
1933            .ok()?;
1934
1935        // Create a worktree branch.
1936        let wt_dir = tmp
1937            .path()
1938            .join("worktrees")
1939            .join("repo.git")
1940            .join("gh-issue-42-test");
1941        std_fs::create_dir_all(wt_dir.parent()?).ok()?;
1942        let wt_out = process::Command::new("git")
1943            .args([
1944                "-C",
1945                repo_dir.to_str()?,
1946                "worktree",
1947                "add",
1948                "-b",
1949                "gh-issue-42-test",
1950                wt_dir.to_str()?,
1951            ])
1952            .output()
1953            .ok()?;
1954
1955        if !wt_out.status.success() {
1956            return None;
1957        }
1958
1959        Some((tmp, wt_dir))
1960    }
1961
1962    #[tokio::test]
1963    async fn janitor_skips_young_worktree() {
1964        let Some((tmp, wt_dir)) = setup_test_repo() else {
1965            eprintln!("skipping test: git not available");
1966            return;
1967        };
1968
1969        // The worktree was just created → age is 0h < ttl_hours=24 → must be skipped.
1970        assert!(wt_dir.exists(), "worktree should exist before janitor runs");
1971
1972        // Run the TTL guard check directly (without a full engine).
1973        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
1974        assert!(
1975            age < 24,
1976            "freshly created worktree should be younger than 24h (age={age}h)"
1977        );
1978
1979        // Explicitly verify: with ttl_hours=24 the worktree stays.
1980        let opts = JanitorOptions {
1981            ttl_hours: 24,
1982            dry_run: true,
1983        };
1984        // Age < ttl → should skip (worktree_age_hours < opts.ttl_hours).
1985        assert!(age < opts.ttl_hours, "janitor should skip young worktree");
1986
1987        drop(tmp); // cleanup tempdir
1988    }
1989
1990    #[tokio::test]
1991    async fn janitor_eligible_when_ttl_zero() {
1992        let Some((tmp, wt_dir)) = setup_test_repo() else {
1993            eprintln!("skipping test: git not available");
1994            return;
1995        };
1996
1997        // With ttl_hours=0 even a brand-new worktree is eligible.
1998        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
1999        let opts = JanitorOptions {
2000            ttl_hours: 0,
2001            dry_run: true,
2002        };
2003        assert!(
2004            age >= opts.ttl_hours,
2005            "worktree should be eligible when ttl=0 (age={age}h)"
2006        );
2007
2008        drop(tmp);
2009    }
2010
2011    #[tokio::test]
2012    async fn dry_run_does_not_remove_worktree() {
2013        // Verify that with dry_run=true the worktree directory is NOT removed.
2014        let Some((tmp, wt_dir)) = setup_test_repo() else {
2015            eprintln!("skipping test: git not available");
2016            return;
2017        };
2018
2019        assert!(wt_dir.exists(), "worktree must exist before test");
2020
2021        // With dry_run=true and ttl_hours=0 the janitor would normally remove the
2022        // worktree, but dry_run prevents it.
2023        let opts = JanitorOptions {
2024            ttl_hours: 0,
2025            dry_run: true,
2026        };
2027
2028        // The janitor needs a repo_root to run git commands.  We point it at the
2029        // bare repo we created inside tmp (even though it's not registered in config).
2030        // Because resolve_repo_root will fail for our synthetic repo slug, we test the
2031        // inner helpers directly here.
2032        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2033        let in_use = false; // no tmux running in tests
2034        let skip = age < opts.ttl_hours || in_use;
2035
2036        if !skip && opts.dry_run {
2037            // dry-run: log only, don't touch filesystem.
2038            tracing::debug!("[dry-run] would remove {}", wt_dir.display());
2039        } else if !skip {
2040            // This branch would do actual removal — but we're in dry_run mode above.
2041        }
2042
2043        // Worktree must still be there.
2044        assert!(
2045            wt_dir.exists(),
2046            "dry-run must not remove the worktree directory"
2047        );
2048
2049        drop(tmp);
2050    }
2051
2052    #[test]
2053    fn fallback_branch_path_construction() {
2054        // Verify the new fallback logic: when worktree path is absent but
2055        // branch is known, we should construct the path correctly.
2056        let tmp = tempfile::tempdir().unwrap();
2057        let wt_dir = tmp.path().join("myrepo").join("gh-issue-7-fix");
2058        std_fs::create_dir_all(&wt_dir).unwrap();
2059
2060        // Simulate: worktree=None, branch=Some("gh-issue-7-fix"), repo="owner/myrepo"
2061        let branch = "gh-issue-7-fix";
2062        let repo = "owner/myrepo";
2063        let worktrees_base = tmp.path();
2064
2065        let project = repo
2066            .rsplit('/')
2067            .next()
2068            .unwrap_or(repo)
2069            .trim_end_matches(".git");
2070        let candidate = worktrees_base.join(project).join(branch);
2071
2072        assert_eq!(candidate, wt_dir);
2073        assert!(
2074            candidate.exists(),
2075            "fallback path should point to existing dir"
2076        );
2077    }
2078
2079    /// A task with no worktree and no branch should return Ok(false) —
2080    /// nothing was cleaned, so git pull should NOT be triggered.
2081    #[tokio::test]
2082    async fn cleanup_returns_false_when_nothing_to_clean() {
2083        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2084
2085        // Create a done task with no worktree or branch fields set
2086        let id = store
2087            .create(&NewTask {
2088                external_id: Some("999".to_string()),
2089                repo: "owner/repo".to_string(),
2090                origin: "github".to_string(),
2091                title: "Already cleaned".to_string(),
2092                ..Default::default()
2093            })
2094            .await
2095            .unwrap();
2096        store.update_status(id, TaskStatus::Done).await.unwrap();
2097
2098        let opts = JanitorOptions {
2099            ttl_hours: 0,
2100            dry_run: false,
2101        };
2102
2103        let result = cleanup_task_worktree_with_opts("999", "owner/repo", &store, &opts).await;
2104        // resolve_repo_root will fail for synthetic repo — that's OK, the point is
2105        // that when nothing needs cleaning the function should not report true.
2106        // In production this means no git pull is triggered.
2107        if let Ok(cleaned) = result {
2108            assert!(
2109                !cleaned,
2110                "cleanup should return false when there is no worktree or branch to clean"
2111            );
2112        } else {
2113            // resolve_repo_root fails for test repo — acceptable.
2114            // The bug was that even with no worktree/branch, Ok(true) was returned.
2115        }
2116    }
2117
2118    /// When the stored worktree path no longer exists on disk and there is no
2119    /// branch recorded, cleanup must mark the task as cleaned so subsequent
2120    /// runs do not keep retrying (idempotency regression test for #1021).
2121    #[tokio::test]
2122    async fn cleanup_marks_cleaned_when_stored_path_is_gone() {
2123        let tmp = tempfile::tempdir().unwrap();
2124        // A path that does NOT exist on disk.
2125        let stale_wt = tmp.path().join("stale-worktree");
2126        assert!(!stale_wt.exists(), "stale path must not exist");
2127
2128        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2129
2130        let id = store
2131            .create(&NewTask {
2132                external_id: Some("1005".to_string()),
2133                repo: "owner/repo".to_string(),
2134                origin: "github".to_string(),
2135                title: "Stale worktree task".to_string(),
2136                ..Default::default()
2137            })
2138            .await
2139            .unwrap();
2140        store.update_status(id, TaskStatus::Done).await.unwrap();
2141        // Set the stale worktree path on the task.
2142        store
2143            .set_fields(
2144                id,
2145                &[("worktree", stale_wt.to_string_lossy().as_ref().into())],
2146            )
2147            .await
2148            .unwrap();
2149
2150        // Verify the task starts out NOT cleaned.
2151        let before = store.get(id).await.unwrap();
2152        assert!(!before.worktree_cleaned, "task should not be cleaned yet");
2153        assert!(
2154            !before.worktree.is_empty(),
2155            "task must have a worktree path set"
2156        );
2157
2158        let opts = JanitorOptions {
2159            ttl_hours: 0,
2160            dry_run: false,
2161        };
2162
2163        let result = cleanup_task_worktree_with_opts("1005", "owner/repo", &store, &opts).await;
2164
2165        // The function should succeed (Ok) — stale path is not an error.
2166        // It returns Ok(false) because there was nothing to actively remove.
2167        match result {
2168            Ok(_) => {}
2169            Err(e) => panic!("cleanup should not error for stale path: {e}"),
2170        }
2171
2172        // The task must now be marked cleaned so the next cycle skips it.
2173        let after = store.get(id).await.unwrap();
2174        assert!(
2175            after.worktree_cleaned,
2176            "task must be marked cleaned when stored worktree path is already gone"
2177        );
2178    }
2179
2180    /// `remove_worktree_and_branch` returns true when the worktree directory
2181    /// is successfully removed (regression test for #1143).
2182    #[tokio::test]
2183    async fn remove_worktree_returns_true_on_success() {
2184        let Some((tmp, wt_dir)) = setup_test_repo() else {
2185            eprintln!("skipping test: git not available");
2186            return;
2187        };
2188
2189        let repo_dir = tmp.path().join("repo.git");
2190        assert!(wt_dir.exists(), "worktree must exist before removal");
2191
2192        let removed =
2193            remove_worktree_and_branch("42", &wt_dir, Some("gh-issue-42-test"), &repo_dir, false)
2194                .await;
2195
2196        assert!(
2197            removed,
2198            "remove_worktree_and_branch should return true when directory is gone"
2199        );
2200        assert!(!wt_dir.exists(), "worktree directory should be removed");
2201    }
2202
2203    /// Successful worktree cleanup must still delete the local branch.
2204    /// This guards against early-return regressions in `remove_worktree_and_branch`.
2205    #[tokio::test]
2206    async fn remove_worktree_success_still_deletes_local_branch() {
2207        let Some((tmp, wt_dir)) = setup_test_repo() else {
2208            eprintln!("skipping test: git not available");
2209            return;
2210        };
2211
2212        let repo_dir = tmp.path().join("repo.git");
2213        let repo_dir_str = repo_dir
2214            .to_str()
2215            .expect("test repo path contains non-UTF-8 characters");
2216        let branch = "gh-issue-42-test";
2217
2218        let before = process::Command::new("git")
2219            .args([
2220                "-C",
2221                repo_dir_str,
2222                "show-ref",
2223                "--verify",
2224                "--quiet",
2225                &format!("refs/heads/{branch}"),
2226            ])
2227            .status()
2228            .unwrap();
2229        assert!(
2230            before.success(),
2231            "test setup must create local branch before cleanup"
2232        );
2233
2234        let removed =
2235            remove_worktree_and_branch("42", &wt_dir, Some(branch), &repo_dir, false).await;
2236        assert!(removed, "worktree should be removed");
2237
2238        let after = process::Command::new("git")
2239            .args([
2240                "-C",
2241                repo_dir_str,
2242                "show-ref",
2243                "--verify",
2244                "--quiet",
2245                &format!("refs/heads/{branch}"),
2246            ])
2247            .status()
2248            .unwrap();
2249        assert!(
2250            !after.success(),
2251            "local branch should be deleted after successful worktree cleanup"
2252        );
2253    }
2254
2255    /// `remove_worktree_and_branch` returns false when the worktree directory
2256    /// cannot be removed (regression test for #1143 — prevents orphaned worktrees
2257    /// from being permanently marked as cleaned).
2258    #[tokio::test]
2259    async fn remove_worktree_returns_false_when_dir_persists() {
2260        let tmp = tempfile::tempdir().unwrap();
2261        // Create a plain directory (not a real git worktree) — git will fail
2262        // to remove it, so the directory stays on disk.
2263        let fake_wt = tmp.path().join("not-a-worktree");
2264        std_fs::create_dir_all(&fake_wt).unwrap();
2265
2266        // repo_root doesn't matter much — it just needs to exist for the -C flag.
2267        // Using tmp itself; git commands will fail but that's the point.
2268        let removed = remove_worktree_and_branch("99", &fake_wt, None, tmp.path(), false).await;
2269
2270        // The directory still exists because git couldn't remove it.
2271        // Before the fix, this would still have been marked as cleaned.
2272        assert!(
2273            !removed,
2274            "remove_worktree_and_branch should return false when directory still exists"
2275        );
2276        assert!(fake_wt.exists(), "directory should still be on disk");
2277    }
2278
2279    struct MergeCheckMockBackend {
2280        batch_result: Mutex<anyhow::Result<HashMap<String, bool>>>,
2281        single_results: HashMap<String, anyhow::Result<bool>>,
2282        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2283        single_checked_branches: Arc<Mutex<Vec<String>>>,
2284    }
2285
2286    impl MergeCheckMockBackend {
2287        fn with_batch_error(
2288            error: anyhow::Error,
2289            single_results: HashMap<String, anyhow::Result<bool>>,
2290        ) -> Self {
2291            Self {
2292                batch_result: Mutex::new(Err(error)),
2293                single_results,
2294                posted_comments: Arc::new(Mutex::new(Vec::new())),
2295                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2296            }
2297        }
2298
2299        fn with_merged(branch: &str) -> Self {
2300            Self {
2301                batch_result: Mutex::new(Ok(HashMap::from([(branch.to_string(), true)]))),
2302                single_results: HashMap::new(),
2303                posted_comments: Arc::new(Mutex::new(Vec::new())),
2304                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2305            }
2306        }
2307    }
2308
2309    #[async_trait]
2310    impl ExternalBackend for MergeCheckMockBackend {
2311        fn name(&self) -> &'static str {
2312            "mock"
2313        }
2314
2315        async fn create_task(
2316            &self,
2317            _title: &str,
2318            _body: &str,
2319            _labels: &[String],
2320        ) -> anyhow::Result<ExternalId> {
2321            anyhow::bail!("not implemented")
2322        }
2323
2324        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2325            anyhow::bail!("not implemented")
2326        }
2327
2328        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2329            Ok(vec![])
2330        }
2331
2332        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2333            Ok(vec![])
2334        }
2335
2336        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2337            self.posted_comments
2338                .lock()
2339                .unwrap()
2340                .push((id.0.clone(), body.to_string()));
2341            Ok(())
2342        }
2343
2344        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2345            Ok(())
2346        }
2347
2348        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2349            Ok(())
2350        }
2351
2352        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2353            Ok(vec![])
2354        }
2355
2356        async fn create_sub_task(
2357            &self,
2358            _parent: &ExternalId,
2359            _title: &str,
2360            _body: &str,
2361            _labels: &[String],
2362        ) -> anyhow::Result<ExternalId> {
2363            anyhow::bail!("not implemented")
2364        }
2365
2366        async fn health_check(&self) -> anyhow::Result<()> {
2367            Ok(())
2368        }
2369
2370        async fn is_pr_merged(&self, branch: &str) -> anyhow::Result<bool> {
2371            self.single_checked_branches
2372                .lock()
2373                .unwrap()
2374                .push(branch.to_string());
2375            match self.single_results.get(branch) {
2376                Some(Ok(is_merged)) => Ok(*is_merged),
2377                Some(Err(err)) => Err(anyhow::anyhow!(err.to_string())),
2378                None => Ok(false),
2379            }
2380        }
2381
2382        async fn batch_is_pr_merged(
2383            &self,
2384            _branches: &[String],
2385        ) -> anyhow::Result<HashMap<String, bool>> {
2386            match &*self.batch_result.lock().unwrap() {
2387                Ok(map) => Ok(map.clone()),
2388                Err(err) => Err(anyhow::anyhow!(err.to_string())),
2389            }
2390        }
2391    }
2392
2393    #[tokio::test]
2394    async fn check_merged_prs_falls_back_to_single_branch_checks_on_batch_error() {
2395        let single_results = HashMap::from([
2396            ("branch-1".to_string(), Ok(true)),
2397            ("branch-2".to_string(), Ok(false)),
2398        ]);
2399        let backend = Arc::new(MergeCheckMockBackend::with_batch_error(
2400            anyhow::anyhow!("missing /data/repository in GraphQL response: no error details"),
2401            single_results,
2402        ));
2403        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2404
2405        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2406        let task1 = store
2407            .create(&NewTask {
2408                external_id: Some("101".to_string()),
2409                repo: "owner/repo".to_string(),
2410                origin: "github".to_string(),
2411                title: "Task 101".to_string(),
2412                ..Default::default()
2413            })
2414            .await
2415            .unwrap();
2416        store
2417            .update_status(task1, TaskStatus::NeedsReview)
2418            .await
2419            .unwrap();
2420        store
2421            .set_fields(task1, &[("branch", serde_json::json!("branch-1"))])
2422            .await
2423            .unwrap();
2424        // No .git file, so cleanup force-removes the directory
2425        let wt_dir = tempfile::tempdir().unwrap();
2426        let wt_path = wt_dir.path().join("branch-1");
2427        std_fs::create_dir_all(&wt_path).unwrap();
2428        store
2429            .set_fields(
2430                task1,
2431                &[("worktree", serde_json::json!(wt_path.to_string_lossy()))],
2432            )
2433            .await
2434            .unwrap();
2435
2436        let task2 = store
2437            .create(&NewTask {
2438                external_id: Some("102".to_string()),
2439                repo: "owner/repo".to_string(),
2440                origin: "github".to_string(),
2441                title: "Task 102".to_string(),
2442                ..Default::default()
2443            })
2444            .await
2445            .unwrap();
2446        store
2447            .update_status(task2, TaskStatus::InReview)
2448            .await
2449            .unwrap();
2450        store
2451            .set_fields(task2, &[("branch", serde_json::json!("branch-2"))])
2452            .await
2453            .unwrap();
2454
2455        let task_manager = Arc::new(TaskManager::with_store(
2456            Arc::clone(&backend_dyn),
2457            Arc::clone(&store),
2458            "owner/repo".to_string(),
2459        ));
2460
2461        let in_review_tasks = store
2462            .list_by_status("owner/repo", TaskStatus::InReview)
2463            .await
2464            .unwrap()
2465            .into_iter()
2466            .map(|stored| ReviewTaskSnapshot {
2467                external: tasks::store_task_to_external(&stored),
2468                stored,
2469            })
2470            .collect::<Vec<_>>();
2471        let needs_review_tasks = store
2472            .list_by_status("owner/repo", TaskStatus::NeedsReview)
2473            .await
2474            .unwrap()
2475            .into_iter()
2476            .map(|stored| ReviewTaskSnapshot {
2477                external: tasks::store_task_to_external(&stored),
2478                stored,
2479            })
2480            .collect::<Vec<_>>();
2481
2482        check_merged_prs(
2483            &backend_dyn,
2484            "owner/repo",
2485            &store,
2486            &task_manager,
2487            &in_review_tasks,
2488            &needs_review_tasks,
2489        )
2490        .await
2491        .unwrap();
2492
2493        let checked = backend.single_checked_branches.lock().unwrap().clone();
2494        assert_eq!(
2495            checked,
2496            vec!["branch-2".to_string(), "branch-1".to_string()]
2497        );
2498
2499        let updated_task1 = store.get(task1).await.unwrap();
2500        let updated_task2 = store.get(task2).await.unwrap();
2501        assert_eq!(updated_task1.status, TaskStatus::Done);
2502        assert!(updated_task1.worktree_cleaned);
2503        assert!(!wt_path.exists());
2504        assert!(!updated_task2.worktree_cleaned);
2505        assert_eq!(updated_task2.status, TaskStatus::InReview);
2506
2507        let comments = backend.posted_comments.lock().unwrap().clone();
2508        assert_eq!(comments.len(), 1);
2509        assert_eq!(comments[0].0, "101");
2510        assert!(comments[0].1.contains("PR merged, marking task complete"));
2511    }
2512
2513    #[tokio::test]
2514    async fn check_merged_prs_skips_rescue_while_ci_bootstrap_holds() {
2515        let (tmp, clone) = origin_and_clone_with_feat();
2516        let wt = add_worktree(&clone, tmp.path(), "feat");
2517        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2518        // hold_after_merge reads the plan marker in the body, not the title
2519        let id = store_approved_task(
2520            &store,
2521            "owner/repo",
2522            "task-12",
2523            "CI-01 Set up CI and tests",
2524            "<!-- orch-plan: CI-01 -->",
2525            &wt,
2526        )
2527        .await;
2528        let stored = store.get(id).await.unwrap();
2529
2530        let backend = Arc::new(MergeCheckMockBackend::with_merged("feat"));
2531        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2532        let task_manager = Arc::new(TaskManager::with_store(
2533            Arc::clone(&backend_dyn),
2534            Arc::clone(&store),
2535            "owner/repo".to_string(),
2536        ));
2537
2538        check_merged_prs(
2539            &backend_dyn,
2540            "owner/repo",
2541            &store,
2542            &task_manager,
2543            &[ReviewTaskSnapshot {
2544                external: tasks::store_task_to_external(&stored),
2545                stored,
2546            }],
2547            &[],
2548        )
2549        .await
2550        .unwrap();
2551
2552        let after = store.get(id).await.unwrap();
2553        assert_eq!(after.status, TaskStatus::Approved);
2554        assert!(after.worktree_cleaned);
2555        assert!(!wt.exists());
2556        assert!(!remote_has(&clone, "rescue/feat"));
2557        assert!(backend.posted_comments.lock().unwrap().is_empty());
2558    }
2559
2560    struct ReconciliationMockBackend {
2561        reconciliation_calls: Arc<AtomicUsize>,
2562    }
2563
2564    #[async_trait]
2565    impl ExternalBackend for ReconciliationMockBackend {
2566        fn name(&self) -> &'static str {
2567            "mock"
2568        }
2569
2570        async fn create_task(
2571            &self,
2572            _title: &str,
2573            _body: &str,
2574            _labels: &[String],
2575        ) -> anyhow::Result<ExternalId> {
2576            anyhow::bail!("not implemented")
2577        }
2578
2579        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2580            anyhow::bail!("not implemented")
2581        }
2582
2583        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2584            Ok(vec![])
2585        }
2586
2587        async fn list_reconciliation_candidates(&self) -> anyhow::Result<Vec<ExternalTask>> {
2588            self.reconciliation_calls.fetch_add(1, Ordering::SeqCst);
2589            Ok(vec![])
2590        }
2591
2592        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2593            Ok(vec![])
2594        }
2595
2596        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2597            Ok(())
2598        }
2599
2600        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2601            Ok(())
2602        }
2603
2604        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2605            Ok(())
2606        }
2607
2608        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2609            Ok(vec![])
2610        }
2611
2612        async fn create_sub_task(
2613            &self,
2614            _parent: &ExternalId,
2615            _title: &str,
2616            _body: &str,
2617            _labels: &[String],
2618        ) -> anyhow::Result<ExternalId> {
2619            anyhow::bail!("not implemented")
2620        }
2621
2622        async fn health_check(&self) -> anyhow::Result<()> {
2623            Ok(())
2624        }
2625    }
2626
2627    #[tokio::test]
2628    async fn cleanup_reconciliation_is_rate_limited_per_repo() {
2629        let repo = "owner/repo-cleanup-rate-limit";
2630        RECONCILIATION_LAST_RUN.remove(repo);
2631
2632        let backend = Arc::new(ReconciliationMockBackend {
2633            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2634        });
2635        let backend_dyn: Arc<dyn ExternalBackend> =
2636            Arc::<ReconciliationMockBackend>::clone(&backend);
2637        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2638        let task_manager = Arc::new(TaskManager::with_store(
2639            Arc::clone(&backend_dyn),
2640            Arc::clone(&store),
2641            repo.to_string(),
2642        ));
2643        let opts = JanitorOptions {
2644            ttl_hours: 0,
2645            dry_run: true,
2646        };
2647
2648        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2649            .await
2650            .unwrap();
2651        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2652            .await
2653            .unwrap();
2654
2655        assert_eq!(
2656            backend.reconciliation_calls.load(Ordering::SeqCst),
2657            1,
2658            "cleanup should not re-list reconciliation candidates on every sync pass"
2659        );
2660
2661        RECONCILIATION_LAST_RUN.remove(repo);
2662    }
2663
2664    /// Cleanup only collects done tasks. A blocked task waits on a person and resumes from its
2665    /// worktree and branch, so it must not even be considered (#3781)
2666    #[tokio::test]
2667    async fn cleanup_skips_blocked_internal_tasks_and_cleans_done_ones() {
2668        let repo = "owner/repo-cleanup-blocked";
2669        RECONCILIATION_LAST_RUN.remove(repo);
2670        let backend_dyn: Arc<dyn ExternalBackend> = Arc::new(ReconciliationMockBackend {
2671            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2672        });
2673        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2674        let task_manager = Arc::new(TaskManager::with_store(
2675            Arc::clone(&backend_dyn),
2676            Arc::clone(&store),
2677            repo.to_string(),
2678        ));
2679
2680        let mut ids = Vec::new();
2681        for (title, status) in [("blocked", TaskStatus::Blocked), ("done", TaskStatus::Done)] {
2682            let id = store
2683                .create_internal(repo, title, "", "test", title, None)
2684                .await
2685                .unwrap();
2686            store
2687                .set_fields(
2688                    id,
2689                    &[
2690                        (
2691                            "worktree",
2692                            serde_json::json!(format!("/nonexistent/{title}")),
2693                        ),
2694                        ("branch", serde_json::json!("")),
2695                    ],
2696                )
2697                .await
2698                .unwrap();
2699            store.update_status(id, status).await.unwrap();
2700            ids.push(id);
2701        }
2702
2703        let opts = JanitorOptions {
2704            ttl_hours: 0,
2705            dry_run: false,
2706        };
2707        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2708            .await
2709            .unwrap();
2710
2711        assert!(
2712            !store.get(ids[0]).await.unwrap().worktree_cleaned,
2713            "blocked task must not be considered for cleanup"
2714        );
2715        assert!(
2716            store.get(ids[1]).await.unwrap().worktree_cleaned,
2717            "done task is cleaned, which proves the pass ran"
2718        );
2719        RECONCILIATION_LAST_RUN.remove(repo);
2720    }
2721
2722    fn git_ok(dir: &Path, args: &[&str]) -> String {
2723        let out = process::Command::new("git")
2724            .args([
2725                "-c",
2726                "commit.gpgsign=false",
2727                "-c",
2728                "user.name=T",
2729                "-c",
2730                "user.email=t@t",
2731            ])
2732            .args(args)
2733            .current_dir(dir)
2734            .output()
2735            .unwrap();
2736        assert!(
2737            out.status.success(),
2738            "git {args:?}: {}",
2739            String::from_utf8_lossy(&out.stderr)
2740        );
2741        String::from_utf8_lossy(&out.stdout).to_string()
2742    }
2743
2744    /// A clone of a bare origin with main pushed and branch `feat` one commit ahead
2745    fn origin_and_clone_with_feat() -> (tempfile::TempDir, PathBuf) {
2746        let tmp = tempfile::tempdir().unwrap();
2747        let bare = tmp.path().join("origin.git");
2748        std_fs::create_dir_all(&bare).unwrap();
2749        git_ok(&bare, &["init", "--bare", "-b", "main"]);
2750        git_ok(tmp.path(), &["clone", bare.to_str().unwrap(), "clone"]);
2751        let clone = tmp.path().join("clone");
2752        std_fs::write(clone.join("base.txt"), "base").unwrap();
2753        git_ok(&clone, &["add", "."]);
2754        git_ok(&clone, &["commit", "-m", "base"]);
2755        git_ok(&clone, &["push", "origin", "main"]);
2756        git_ok(&clone, &["checkout", "-b", "feat"]);
2757        std_fs::write(clone.join("work.txt"), "work").unwrap();
2758        git_ok(&clone, &["add", "."]);
2759        git_ok(&clone, &["commit", "-m", "work"]);
2760        git_ok(&clone, &["checkout", "main"]);
2761        (tmp, clone)
2762    }
2763
2764    fn remote_has(clone: &Path, branch: &str) -> bool {
2765        !git_ok(clone, &["ls-remote", "--heads", "origin", branch])
2766            .trim()
2767            .is_empty()
2768    }
2769
2770    fn local_has(clone: &Path, branch: &str) -> bool {
2771        process::Command::new("git")
2772            .args([
2773                "show-ref",
2774                "--verify",
2775                "--quiet",
2776                &format!("refs/heads/{branch}"),
2777            ])
2778            .current_dir(clone)
2779            .status()
2780            .unwrap()
2781            .success()
2782    }
2783
2784    /// Deleting a pushed branch whose commits main lacks would destroy the only copy of the
2785    /// work, as startup cleanup did to a blocked task (#3781)
2786    #[tokio::test]
2787    async fn delete_branches_keeps_an_unmerged_remote_branch() {
2788        let (_tmp, clone) = origin_and_clone_with_feat();
2789        git_ok(&clone, &["push", "origin", "feat"]);
2790
2791        delete_branches("1", "feat", &clone).await;
2792
2793        assert!(
2794            remote_has(&clone, "feat"),
2795            "unmerged remote branch must survive"
2796        );
2797        assert!(
2798            !local_has(&clone, "feat"),
2799            "local copy is on origin, so it can go"
2800        );
2801    }
2802
2803    #[tokio::test]
2804    async fn delete_branches_leaves_the_screenshots_branch_alone() {
2805        let (_tmp, clone) = origin_and_clone_with_feat();
2806        git_ok(&clone, &["branch", SCREENSHOTS_BRANCH, "feat"]);
2807        git_ok(&clone, &["push", "origin", SCREENSHOTS_BRANCH]);
2808        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2809        git_ok(&clone, &["push", "origin", "main"]);
2810
2811        delete_branches("1", SCREENSHOTS_BRANCH, &clone).await;
2812
2813        assert!(remote_has(&clone, SCREENSHOTS_BRANCH));
2814        assert!(local_has(&clone, SCREENSHOTS_BRANCH));
2815    }
2816
2817    #[tokio::test]
2818    async fn delete_branches_removes_a_branch_main_already_contains() {
2819        let (_tmp, clone) = origin_and_clone_with_feat();
2820        git_ok(&clone, &["push", "origin", "feat"]);
2821        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2822        git_ok(&clone, &["push", "origin", "main"]);
2823
2824        delete_branches("1", "feat", &clone).await;
2825
2826        assert!(!remote_has(&clone, "feat"));
2827        assert!(!local_has(&clone, "feat"));
2828    }
2829
2830    fn add_worktree(clone: &Path, tmp: &Path, branch: &str) -> PathBuf {
2831        let wt = tmp.join("wt");
2832        git_ok(clone, &["worktree", "add", wt.to_str().unwrap(), branch]);
2833        wt
2834    }
2835
2836    async fn store_approved_task(
2837        store: &TaskStore,
2838        repo: &str,
2839        external_id: &str,
2840        title: &str,
2841        body: &str,
2842        worktree: &Path,
2843    ) -> i64 {
2844        let id = store
2845            .create(&NewTask {
2846                external_id: Some(external_id.to_string()),
2847                repo: repo.to_string(),
2848                origin: "github".to_string(),
2849                title: title.to_string(),
2850                body: body.to_string(),
2851                ..Default::default()
2852            })
2853            .await
2854            .unwrap();
2855        store.update_status(id, TaskStatus::Approved).await.unwrap();
2856        store
2857            .set_fields(
2858                id,
2859                &[
2860                    ("branch", serde_json::json!("feat")),
2861                    (
2862                        "worktree",
2863                        serde_json::json!(worktree.to_string_lossy().as_ref()),
2864                    ),
2865                    ("pr_number", serde_json::json!(7)),
2866                ],
2867            )
2868            .await
2869            .unwrap();
2870        id
2871    }
2872
2873    #[tokio::test]
2874    async fn cleanup_pushes_unpushed_commits_to_a_rescue_branch() {
2875        let (tmp, clone) = origin_and_clone_with_feat();
2876        let wt = add_worktree(&clone, tmp.path(), "feat");
2877
2878        let sha = git_ok(&clone, &["rev-parse", "--short", "feat"]);
2879        let sha = sha.trim();
2880        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2881            Rescue::Pushed { branch, commits } => {
2882                assert_eq!(branch, "rescue/feat");
2883                assert!(rescue_comment(&branch, &commits).contains(sha));
2884            }
2885            other => panic!("expected a rescue push, got {other:?}"),
2886        }
2887
2888        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2889
2890        assert!(removed);
2891        assert!(remote_has(&clone, "rescue/feat"));
2892        assert!(!local_has(&clone, "feat"));
2893    }
2894
2895    #[tokio::test]
2896    async fn cleanup_commits_uncommitted_changes_before_the_rescue() {
2897        let (tmp, clone) = origin_and_clone_with_feat();
2898        git_ok(&clone, &["push", "origin", "feat"]);
2899        let wt = add_worktree(&clone, tmp.path(), "feat");
2900        std_fs::write(wt.join("dirty.txt"), "dirty").unwrap();
2901
2902        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2903
2904        assert!(removed);
2905        assert!(remote_has(&clone, "rescue/feat"));
2906    }
2907
2908    #[tokio::test]
2909    async fn cleanup_makes_no_rescue_branch_for_a_pushed_branch() {
2910        let (tmp, clone) = origin_and_clone_with_feat();
2911        git_ok(&clone, &["push", "origin", "feat"]);
2912        let wt = add_worktree(&clone, tmp.path(), "feat");
2913
2914        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2915
2916        assert!(removed);
2917        assert!(!remote_has(&clone, "rescue/feat"));
2918        assert!(!local_has(&clone, "feat"));
2919    }
2920
2921    #[tokio::test]
2922    async fn rescue_skips_a_branch_on_the_remote_without_a_tracking_ref() {
2923        let (_tmp, clone) = origin_and_clone_with_feat();
2924        git_ok(&clone, &["push", "origin", "feat"]);
2925        git_ok(&clone, &["update-ref", "-d", "refs/remotes/origin/feat"]);
2926
2927        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
2928
2929        assert_eq!(rescue, Rescue::Clean);
2930        assert!(!remote_has(&clone, "rescue/feat"));
2931    }
2932
2933    #[tokio::test]
2934    async fn merged_pr_cleanup_pushes_no_rescue_branch() {
2935        let (tmp, clone) = origin_and_clone_with_feat();
2936        let wt = add_worktree(&clone, tmp.path(), "feat");
2937
2938        let removed =
2939            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, true).await;
2940
2941        assert!(removed);
2942        assert!(!wt.exists());
2943        assert!(!remote_has(&clone, "rescue/feat"));
2944    }
2945
2946    #[tokio::test]
2947    async fn cleanup_skips_rescue_for_an_approved_merged_pr() {
2948        let (tmp, clone) = origin_and_clone_with_feat();
2949        let wt = add_worktree(&clone, tmp.path(), "feat");
2950        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2951        store_approved_task(&store, "owner/repo", "task-12", "CI-01 Set up CI", "", &wt).await;
2952
2953        cleanup_merged_pr_worktree("task-12", "owner/repo", &store)
2954            .await
2955            .unwrap();
2956
2957        let task = store
2958            .get(
2959                store
2960                    .resolve_task_id("owner/repo", "task-12")
2961                    .await
2962                    .unwrap()
2963                    .unwrap(),
2964            )
2965            .await
2966            .unwrap();
2967        assert_eq!(task.status, TaskStatus::Approved);
2968        assert!(task.worktree_cleaned);
2969        assert!(!wt.exists());
2970        assert!(!remote_has(&clone, "rescue/feat"));
2971    }
2972
2973    #[tokio::test]
2974    async fn cleanup_rescues_an_approved_task_when_the_pr_is_not_known_merged() {
2975        let (tmp, clone) = origin_and_clone_with_feat();
2976        let wt = add_worktree(&clone, tmp.path(), "feat");
2977        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2978        store_approved_task(&store, "owner/repo", "task-12", "fix something", "", &wt).await;
2979
2980        cleanup_task_worktree("task-12", "owner/repo", &store)
2981            .await
2982            .unwrap();
2983
2984        assert!(remote_has(&clone, "rescue/feat"));
2985    }
2986
2987    #[tokio::test]
2988    async fn unmerged_cleanup_still_pushes_a_rescue_branch() {
2989        let (tmp, clone) = origin_and_clone_with_feat();
2990        let wt = add_worktree(&clone, tmp.path(), "feat");
2991
2992        let removed =
2993            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, false).await;
2994
2995        assert!(removed);
2996        assert!(remote_has(&clone, "rescue/feat"));
2997    }
2998
2999    #[tokio::test]
3000    async fn rescue_ignores_a_remote_tip_missing_locally() {
3001        let (tmp, clone) = origin_and_clone_with_feat();
3002        git_ok(&clone, &["push", "origin", "feat"]);
3003        let other = tmp.path().join("other");
3004        git_ok(
3005            tmp.path(),
3006            &["clone", "-b", "feat", "origin.git", other.to_str().unwrap()],
3007        );
3008        std_fs::write(other.join("extra.txt"), "extra").unwrap();
3009        git_ok(&other, &["add", "."]);
3010        git_ok(&other, &["commit", "-m", "extra"]);
3011        git_ok(&other, &["push", "origin", "feat"]);
3012
3013        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3014
3015        assert_ne!(rescue, Rescue::Failed);
3016    }
3017
3018    #[tokio::test]
3019    async fn cleanup_keeps_worktree_and_branch_when_the_rescue_push_fails() {
3020        let (tmp, clone) = origin_and_clone_with_feat();
3021        let wt = add_worktree(&clone, tmp.path(), "feat");
3022        git_ok(
3023            &clone,
3024            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3025        );
3026
3027        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3028
3029        assert!(!removed);
3030        assert!(wt.exists());
3031        assert!(local_has(&clone, "feat"));
3032    }
3033
3034    #[tokio::test]
3035    async fn delete_branches_keeps_a_local_branch_with_unpushed_commits() {
3036        let (_tmp, clone) = origin_and_clone_with_feat();
3037
3038        delete_branches("1", "feat", &clone).await;
3039
3040        assert!(
3041            local_has(&clone, "feat"),
3042            "unpushed commits must not be deleted"
3043        );
3044    }
3045}