Skip to main content

orch/engine/
cleanup.rs

1//! Worktree cleanup and merged-PR detection.
2//!
3//! Contains the post-merge cleanup pipeline: removing git worktrees,
4//! deleting local/remote branches, pulling main, and detecting
5//! already-merged PRs so their tasks can be marked done.
6
7use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
8use crate::cmd::CommandErrorContext;
9use crate::engine::ci_bootstrap;
10use crate::engine::tasks::{TaskManager, store_task_to_external};
11use crate::store;
12use crate::store::TaskStatus;
13use crate::store::TaskStore;
14use crate::store::store_log_activity;
15use dashmap::{DashMap, DashSet};
16use futures::future;
17use std::collections::HashMap;
18use std::collections::HashSet;
19use std::env;
20use std::fs as std_fs;
21use std::io::ErrorKind;
22use std::path::Path;
23use std::path::PathBuf;
24use std::process::Output;
25use std::sync::Arc;
26use std::sync::LazyLock;
27use std::time::SystemTime;
28use std::time::{Duration, Instant};
29use tokio::fs;
30use tokio::process::Command;
31use tokio::time::sleep;
32use tokio::time::timeout;
33
34use super::sync::ReviewTaskSnapshot;
35use crate::config;
36use crate::engine;
37use crate::engine::auto_merge;
38use crate::engine::runner::cow;
39use crate::engine::runner::git_ops;
40use crate::engine::runner::screenshots::SCREENSHOTS_BRANCH;
41use crate::engine::runner::worktree;
42use crate::github::http::GhHttp;
43use crate::home;
44
45/// Reconciliation calls can be slow on repos with many issues or during GitHub instability.
46/// Cleanup runs in a background task (tokio::spawn) so it does not block the main tick loop.
47/// 30 s gives enough headroom for paginated list_all_tasks calls while still bounding the
48/// overall background-task duration.
49const RECONCILIATION_LIST_TIMEOUT: Duration = Duration::from_secs(30);
50/// Closed-issue reconciliation is a best-effort cleanup fallback, not a per-sync requirement.
51/// Run it on a slower cadence so repeated GitHub slowness does not consume 30 s every sync.
52const RECONCILIATION_INTERVAL: Duration = Duration::from_mins(15);
53
54const RUN_OUTPUT_RETENTION_DAYS: i32 = 30;
55const RUN_PRUNE_INTERVAL_SECS: i64 = 24 * 60 * 60;
56const RUN_PRUNE_KV_KEY: &str = "task_runs_pruned_at";
57
58/// Per-repo in-flight guard for closed-issue reconciliation/cleanup.
59/// Prevents overlapping cleanup runs from piling up across sync cycles.
60static CLEANUP_IN_FLIGHT: LazyLock<DashSet<String>> = LazyLock::new(DashSet::new);
61/// Per-repo reconciliation cadence tracker.
62static RECONCILIATION_LAST_RUN: LazyLock<DashMap<String, Instant>> = LazyLock::new(DashMap::new);
63
64struct CleanupInFlightGuard {
65    repo: String,
66}
67
68impl CleanupInFlightGuard {
69    fn try_acquire(repo: &str) -> Option<Self> {
70        if CLEANUP_IN_FLIGHT.insert(repo.to_string()) {
71            Some(Self {
72                repo: repo.to_string(),
73            })
74        } else {
75            None
76        }
77    }
78}
79
80impl Drop for CleanupInFlightGuard {
81    fn drop(&mut self) {
82        CLEANUP_IN_FLIGHT.remove(&self.repo);
83    }
84}
85
86fn should_run_reconciliation(repo: &str) -> bool {
87    let now = Instant::now();
88    if let Some(mut last_run) = RECONCILIATION_LAST_RUN.get_mut(repo) {
89        if last_run.elapsed() < RECONCILIATION_INTERVAL {
90            return false;
91        }
92        *last_run = now;
93        true
94    } else {
95        RECONCILIATION_LAST_RUN.insert(repo.to_string(), now);
96        true
97    }
98}
99
100/// Options controlling the worktree janitor.
101#[derive(Debug, Clone)]
102pub struct JanitorOptions {
103    /// Minimum age (in hours) of the worktree directory before it is eligible for removal.
104    ///
105    /// A worktree whose filesystem mtime is more recent than this threshold is skipped
106    /// even if the task is already in a terminal state. This provides a safety buffer
107    /// against races (e.g. review agent still writing when the janitor runs).
108    ///
109    /// Default: 24.  Configurable via `workflow.worktree_janitor_ttl_hours` in config.
110    pub ttl_hours: u64,
111
112    /// When `true`, log what would be done but skip all destructive operations.
113    ///
114    /// Configurable via `workflow.worktree_janitor_dry_run: true` in config.
115    pub dry_run: bool,
116}
117
118impl Default for JanitorOptions {
119    fn default() -> Self {
120        Self {
121            ttl_hours: 24,
122            dry_run: false,
123        }
124    }
125}
126
127impl JanitorOptions {
128    /// Read options from the global config, falling back to defaults.
129    pub fn from_config() -> Self {
130        let ttl_hours = config::get("workflow.worktree_janitor_ttl_hours")
131            .ok()
132            .and_then(|v| v.parse::<u64>().ok())
133            .unwrap_or(24);
134
135        let dry_run = config::get("workflow.worktree_janitor_dry_run").is_ok_and(|v| v == "true");
136
137        Self { ttl_hours, dry_run }
138    }
139}
140
141/// Cleanup worktrees for completed tasks.
142///
143/// Targets tasks that are finished: status:done OR closed on GitHub
144/// (e.g. when a PR merge auto-closes the issue via "Fixes #N").
145///
146/// Note: This is a fallback. Primary cleanup happens inline in
147/// `auto_merge_pr` via `cleanup_task_worktree`. This catches edge cases
148/// where the inline cleanup missed (e.g., manual merges, auto-closed issues).
149pub(crate) async fn cleanup_done_worktrees(
150    backend: &Arc<dyn ExternalBackend>,
151    repo: &str,
152    task_manager: &Arc<TaskManager>,
153    store: &Arc<TaskStore>,
154) -> anyhow::Result<()> {
155    let opts = JanitorOptions::from_config();
156    cleanup_done_worktrees_with_opts(backend, repo, task_manager, store, &opts).await
157}
158
159/// Clear the stored output of runs for tasks done longer than `RUN_OUTPUT_RETENTION_DAYS`, once a day
160pub(crate) async fn prune_old_run_output(store: &TaskStore) -> anyhow::Result<()> {
161    let now = chrono::Utc::now().timestamp();
162    let last = store
163        .kv_get(RUN_PRUNE_KV_KEY)
164        .await?
165        .and_then(|v| v.parse::<i64>().ok())
166        .unwrap_or(0);
167    if now - last < RUN_PRUNE_INTERVAL_SECS {
168        return Ok(());
169    }
170    store.kv_set(RUN_PRUNE_KV_KEY, &now.to_string()).await?;
171
172    let pruned = store.prune_old_runs(RUN_OUTPUT_RETENTION_DAYS).await?;
173    if pruned > 0 {
174        tracing::info!(pruned, "cleared the output of old task runs");
175    }
176    Ok(())
177}
178
179async fn reconcile_closed_tasks(
180    task_ids: &mut Vec<String>,
181    all_tasks: &[ExternalTask],
182    task_manager: &TaskManager,
183    store: &Arc<TaskStore>,
184    repo: &str,
185) {
186    const MAX_RECONCILE_PER_CYCLE: usize = 10;
187
188    let done_set: HashSet<String> = task_ids.iter().cloned().collect();
189    let mut closed_not_in_store = Vec::new();
190    for task in all_tasks {
191        if task.state != "closed" || done_set.contains(&task.id.0) {
192            continue;
193        }
194        if ci_bootstrap::hold_after_merge(task, store, repo).await {
195            continue;
196        }
197        closed_not_in_store.push(task);
198    }
199
200    // Split: issues that already have status:done label just need worktree
201    // cleanup (no API call needed), vs those needing label reconciliation.
202    let mut already_labeled: Vec<String> = Vec::new();
203    let mut needs_reconcile: Vec<&ExternalTask> = Vec::new();
204    for task in &closed_not_in_store {
205        if task.labels.iter().any(|l| l == "status:done") {
206            already_labeled.push(task.id.0.clone());
207        } else {
208            needs_reconcile.push(task);
209        }
210    }
211
212    // Add already-labeled issues directly (no API call).
213    if !already_labeled.is_empty() {
214        tracing::debug!(
215            count = already_labeled.len(),
216            "closed issues already labeled status:done — adding to cleanup set"
217        );
218        task_ids.extend(already_labeled);
219    }
220
221    // Reconcile issues missing the done label — cap to avoid rate-limit
222    // exhaustion. Remaining issues will be picked up in subsequent cycles.
223    if needs_reconcile.len() > MAX_RECONCILE_PER_CYCLE {
224        tracing::info!(
225            total = needs_reconcile.len(),
226            batch = MAX_RECONCILE_PER_CYCLE,
227            "capping closed-issue reconciliation to avoid rate-limit exhaustion"
228        );
229    }
230    let mut futs = Vec::new();
231    for task in needs_reconcile.iter().take(MAX_RECONCILE_PER_CYCLE) {
232        let id = task.id.clone();
233        let task_id_str = task.id.0.clone();
234        let labels = task.labels.clone();
235        futs.push(async move {
236            tracing::info!(
237                task_id = task_id_str,
238                labels = ?labels,
239                "closed issue with stale status label — reconciling to done and cleaning up"
240            );
241            match task_manager.update_task_status(&id, Status::Done).await {
242                Ok(()) => Some(task_id_str),
243                Err(e) => {
244                    tracing::warn!(
245                        task_id = task_id_str,
246                        err = %e,
247                        "failed to reconcile closed issue status to done"
248                    );
249                    None
250                }
251            }
252        });
253    }
254    let results = future::join_all(futs).await;
255    task_ids.extend(results.into_iter().flatten());
256}
257
258/// Cleanup worktrees for completed tasks with explicit options.
259///
260/// Separated from `cleanup_done_worktrees` so that integration tests can
261/// inject specific options (e.g. `ttl_hours: 0`, `dry_run: true`) without
262/// touching the global config.
263pub(crate) async fn cleanup_done_worktrees_with_opts(
264    backend: &Arc<dyn ExternalBackend>,
265    repo: &str,
266    task_manager: &Arc<TaskManager>,
267    store: &Arc<TaskStore>,
268    opts: &JanitorOptions,
269) -> anyhow::Result<()> {
270    let Some(_guard) = CleanupInFlightGuard::try_acquire(repo) else {
271        tracing::debug!(
272            repo = %repo,
273            "cleanup already in progress for repo — skipping overlapping run"
274        );
275        return Ok(());
276    };
277
278    // Read done tasks from the store first; fall back to backend before first sync.
279    let done_tasks = task_manager.list_all_by_status(Status::Done).await?;
280    tracing::debug!(count = done_tasks.len(), "checking done tasks for cleanup");
281
282    // Collect task IDs from external done tasks.
283    let mut task_ids: Vec<String> = done_tasks.iter().map(|t| t.id.0.clone()).collect();
284
285    // Also include closed-but-not-done external tasks.
286    // When a PR merge auto-closes a GitHub issue (via "Fixes #N"), the issue
287    // state becomes "closed" but orch never updates the status label to "done".
288    // These orphaned worktrees accumulate unless we catch them here.
289    // Note: This still queries the backend because GitHub `state` (open/closed)
290    // is backend-specific and not tracked in the store.
291    //
292    // Use list_reconciliation_candidates() (closed issues only, bounded recent window) directly.
293    // list_all_tasks() fetches every issue ever created (state=all, all pages) and
294    // consistently times out on repos with many issues — it is not appropriate for
295    // a cleanup fallback path.
296    if should_run_reconciliation(repo) {
297        match timeout(
298            RECONCILIATION_LIST_TIMEOUT,
299            backend.list_reconciliation_candidates(),
300        )
301        .await
302        {
303            Ok(Ok(candidates)) if !candidates.is_empty() => {
304                tracing::debug!(
305                    count = candidates.len(),
306                    "reconciling closed issues from candidates"
307                );
308                reconcile_closed_tasks(&mut task_ids, &candidates, task_manager, store, repo).await;
309            }
310            Ok(Ok(_)) => {
311                tracing::debug!("no reconciliation candidates found");
312            }
313            Ok(Err(e)) => {
314                tracing::warn!(err = %e, "failed to list reconciliation candidates");
315            }
316            Err(_) => {
317                tracing::warn!(
318                    timeout_secs = RECONCILIATION_LIST_TIMEOUT.as_secs(),
319                    retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
320                    "timed out listing reconciliation candidates"
321                );
322            }
323        }
324    } else {
325        tracing::debug!(
326            retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
327            "skipping closed-issue reconciliation until interval elapses"
328        );
329    }
330
331    // Also include internal done tasks.
332    if let Ok(internal_done) = task_manager.list_internal_by_status(TaskStatus::Done).await {
333        task_ids.extend(internal_done.iter().map(|t| t.id.0.clone()));
334    }
335
336    tracing::debug!(
337        count = task_ids.len(),
338        "checking all terminal tasks for cleanup"
339    );
340
341    if let Err(e) = auto_merge::sweep_ci_timer_keys(store, repo, &task_ids).await {
342        tracing::warn!(err = %e, "failed to sweep CI timer keys");
343    }
344
345    // Prefetch all cleaned external IDs in one query instead of N+1 per-task lookups.
346    // NOTE: This is now primarily a hint to skip processing, NOT a correctness
347    // guarantee — tasks with external_id NOT in the store at all (or mismatched
348    // repo) will still be cleaned by iterating all worktree directories.
349    let already_cleaned = store.cleaned_external_ids(repo).await.unwrap_or_default();
350
351    let mut cleaned_any = false;
352    for task_id in &task_ids {
353        // Skip if already cleaned (checked via prefetched set).
354        if already_cleaned.contains(task_id) {
355            continue;
356        }
357
358        match cleanup_task_worktree_with_opts(task_id, repo, store, opts).await {
359            Ok(true) => cleaned_any = true,
360            Ok(false) => {
361                // Skipped (TTL guard, tmux guard, or nothing to remove) — do not
362                // pull main; we did not change the repo state.
363            }
364            Err(e) => {
365                tracing::warn!(task_id, err = %e, "worktree cleanup failed for task");
366            }
367        }
368    }
369
370    // ── Orphaned worktree scanner ─────────────────────────────────────────────
371    //
372    // Even if every task in our list was already cleaned (or had no worktree),
373    // there may be worktree directories on disk that are not owned by any task.
374    // This happens when a task was ingested into the store but its worktree_cleaned
375    // flag was never set — e.g. because the task was never in `list_all_by_status`
376    // (unknown repo / missing from store at cleanup time), or the cleanup skipped
377    // due to TTL guard and never retried.
378    //
379    // We scan all worktree directories in this repo's worktrees subdirectory and
380    // remove any that are not referenced by an existing task. This catches the
381    // "done but worktree not cleaned" case that `orch service doctor` reports.
382    let worktrees_base = match home::worktrees_dir() {
383        Ok(base) => Some(base),
384        Err(e) => {
385            tracing::debug!(err = %e, "worktrees_dir() unavailable — skipping orphan scan");
386            None
387        }
388    };
389
390    if let Some(base) = worktrees_base {
391        let project = repo
392            .rsplit('/')
393            .next()
394            .unwrap_or(repo)
395            .trim_end_matches(".git");
396        let repo_worktrees = base.join(project);
397
398        // Build set of all known worktree paths for this repo (all statuses).
399        let mut combined: HashSet<String> = HashSet::new();
400        if let Ok(tasks) = store.list_all(repo).await {
401            for t in tasks {
402                if !t.worktree.is_empty() {
403                    combined.insert(t.worktree);
404                }
405            }
406        }
407
408        // Also include active worktrees (in case task is in-memory but has a dir).
409        if let Ok(project_dir) = resolve_project_dir_for_repo(repo)
410            && let Ok(active_wts) = worktree::list_project_worktrees(&project_dir).await
411        {
412            for wt in active_wts {
413                combined.insert(wt.to_string_lossy().to_string());
414            }
415        }
416
417        // Scan all branch directories and remove orphaned ones.
418        if let Ok(mut entries) = fs::read_dir(&repo_worktrees).await {
419            while let Ok(Some(entry)) = entries.next_entry().await {
420                let Ok(entry_meta) = entry.metadata().await else {
421                    continue;
422                };
423                if !entry_meta.is_dir() {
424                    continue;
425                }
426                let wt_path = entry.path().to_string_lossy().to_string();
427                if combined.contains(&wt_path) {
428                    continue;
429                }
430                tracing::info!(worktree = %wt_path, "found orphaned worktree — attempting cleanup");
431                match resolve_repo_root_for_orphaned_worktree(&entry.path()).await {
432                    Ok(repo_root) => {
433                        let removed = remove_worktree_and_branch(
434                            "orphan-scan",
435                            &entry.path(),
436                            None,
437                            Path::new(&repo_root),
438                            false,
439                        )
440                        .await;
441                        if removed {
442                            tracing::info!(worktree = %wt_path, "orphaned worktree removed");
443                        } else {
444                            tracing::warn!(worktree = %wt_path, "orphaned worktree could not be removed");
445                        }
446                    }
447                    Err(e) => {
448                        tracing::warn!(worktree = %wt_path, err = %e, "could not resolve repo root for orphaned worktree");
449                    }
450                }
451            }
452        }
453    }
454
455    // Prune stale worktree metadata and pull main after cleanup.
456    if cleaned_any
457        && !opts.dry_run
458        && let Ok(repo_root) = resolve_repo_root(repo).await
459    {
460        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
461
462        // Prune stale .git/worktrees/ entries whose directories no longer exist.
463        let prune_result = timeout(
464            GIT_TIMEOUT,
465            Command::new("git")
466                .args(["-C", &repo_root, "worktree", "prune"])
467                .output_with_context(),
468        )
469        .await;
470        match prune_result {
471            Ok(Ok(output)) if output.status.success() => {
472                tracing::debug!(%repo, "pruned stale worktree metadata");
473            }
474            Ok(Ok(output)) => {
475                let stderr = String::from_utf8_lossy(&output.stderr);
476                tracing::warn!(%repo, err = %stderr, "git worktree prune failed");
477            }
478            Ok(Err(e)) => {
479                tracing::warn!(%repo, err = %e, "git worktree prune failed");
480            }
481            Err(_) => {
482                tracing::warn!(%repo, "git worktree prune timed out after 60s");
483            }
484        }
485
486        // Fetch instead of pull: refresh `origin/*` refs without touching
487        // the operator's checked-out branch or working tree. `pull` would
488        // operate on whatever HEAD happens to be (often a feature branch,
489        // not main) and would silently no-op on divergence.
490        let fetch_result = timeout(
491            GIT_TIMEOUT,
492            git_ops::git()
493                .await
494                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
495                .output_with_context(),
496        )
497        .await;
498        match fetch_result {
499            Ok(Ok(output)) if output.status.success() => {
500                tracing::info!(%repo, "fetched origin after worktree cleanup");
501            }
502            Ok(Ok(output)) => {
503                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
504                tracing::debug!(%repo, err = %stderr, "git fetch skipped after cleanup");
505            }
506            Ok(Err(e)) => {
507                tracing::debug!(%repo, err = %e, "git fetch failed after cleanup");
508            }
509            Err(_) => {
510                tracing::debug!(%repo, "git fetch timed out after 60s");
511            }
512        }
513    }
514
515    Ok(())
516}
517
518/// Cleanup a single task's worktree and branches.
519///
520/// Removes the git worktree, deletes local + remote branches,
521/// fetches origin to refresh remote tracking refs, and marks the task as cleaned.
522///
523/// This function is used for inline post-merge cleanup (called from
524/// auto-merge flows) and must attempt immediate removal — it constructs
525/// janitor options with `ttl_hours = 0` so the janitor age guard does not
526/// postpone removing freshly-created worktrees.
527pub(crate) async fn cleanup_task_worktree(
528    task_id: &str,
529    repo: &str,
530    store: &Arc<TaskStore>,
531) -> anyhow::Result<()> {
532    cleanup_task_worktree_impl(task_id, repo, store, false).await
533}
534
535/// Cleanup after a merged pull request, including while `hold_after_merge` keeps the task approved
536pub(crate) async fn cleanup_merged_pr_worktree(
537    task_id: &str,
538    repo: &str,
539    store: &Arc<TaskStore>,
540) -> anyhow::Result<()> {
541    cleanup_task_worktree_impl(task_id, repo, store, true).await
542}
543
544async fn cleanup_task_worktree_impl(
545    task_id: &str,
546    repo: &str,
547    store: &Arc<TaskStore>,
548    known_pr_merged: bool,
549) -> anyhow::Result<()> {
550    let opts = JanitorOptions {
551        ttl_hours: 0,
552        ..Default::default()
553    };
554    let cleaned =
555        cleanup_task_worktree_with_opts_impl(task_id, repo, store, &opts, known_pr_merged).await?;
556
557    // Fetch origin once after inline cleanup (post-merge single-task path).
558    // Only fetch if cleanup actually happened (not skipped due to active session).
559    // Fetch (not pull) so the operator's checked-out branch and working tree
560    // are never touched — see the matching comment in cleanup_orphaned_worktrees.
561    if cleaned && let Ok(repo_root) = resolve_repo_root(repo).await {
562        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
563        let fetch_result = timeout(
564            GIT_TIMEOUT,
565            git_ops::git()
566                .await
567                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
568                .output_with_context(),
569        )
570        .await;
571        match fetch_result {
572            Ok(Ok(output)) if output.status.success() => {
573                tracing::debug!(%repo, "fetched origin after task cleanup");
574            }
575            Ok(Ok(output)) => {
576                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
577                tracing::debug!(%repo, err = %stderr, "git fetch skipped after task cleanup");
578            }
579            Ok(Err(e)) => {
580                tracing::debug!(%repo, err = %e, "git fetch failed after task cleanup");
581            }
582            Err(_) => {
583                tracing::debug!(%repo, "git fetch timed out after 60s");
584            }
585        }
586    }
587
588    Ok(())
589}
590
591/// Cleanup a single task's worktree and branches with explicit janitor options.
592///
593/// Returns `Ok(true)` when cleanup was actually performed and the task was
594/// marked as cleaned.  Returns `Ok(false)` when cleanup was intentionally
595/// skipped (TTL guard, active tmux session guard, or nothing on disk to remove).
596/// Returns `Err` on unexpected failures (e.g. cannot resolve the repo root).
597pub(crate) async fn cleanup_task_worktree_with_opts(
598    task_id: &str,
599    repo: &str,
600    store: &Arc<TaskStore>,
601    opts: &JanitorOptions,
602) -> anyhow::Result<bool> {
603    cleanup_task_worktree_with_opts_impl(task_id, repo, store, opts, false).await
604}
605
606async fn cleanup_task_worktree_with_opts_impl(
607    task_id: &str,
608    repo: &str,
609    store: &Arc<TaskStore>,
610    opts: &JanitorOptions,
611    known_pr_merged: bool,
612) -> anyhow::Result<bool> {
613    let (worktree, branch, keep_remote_branch, merged, external_id, pr_merged) =
614        store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
615            .await
616            .map_or((None, None, false, false, None, false), |t| {
617                // Blocked tasks with a PR should keep their remote branch so the
618                // PR stays open for human review.
619                let keep = t.status == TaskStatus::Blocked && t.pr_number.is_some();
620                let merged = t.status == TaskStatus::Done || known_pr_merged;
621                (
622                    Some(t.worktree),
623                    Some(t.branch),
624                    keep,
625                    merged,
626                    t.external_id,
627                    // A squash leaves the branch commits on no ref, so skip rescue when the PR merged
628                    known_pr_merged || (t.status == TaskStatus::Done && t.pr_number.is_some()),
629                )
630            });
631
632    let worktree_path = worktree.as_ref().map(PathBuf::from);
633
634    // Get worktrees base path
635    let worktrees_base = home::worktrees_dir().unwrap_or_else(|_| PathBuf::from(".orch/worktrees"));
636
637    // Determine which directory to remove.
638    //
639    // Priority:
640    //   1. stored "worktree" path, if it exists on disk.
641    //   2. Construct from worktrees_base + project name + branch, if it exists.
642    let worktree_to_remove = if let Some(ref wt) = worktree_path {
643        if match fs::try_exists(wt).await {
644            Ok(true) => true,
645            Ok(false) => false,
646            Err(e) => {
647                tracing::warn!(task_id, worktree = %wt.display(), err = %e, "try_exists failed on stored worktree path — trying branch-based fallback");
648                false
649            }
650        } {
651            Some(wt.clone())
652        } else {
653            // The recorded worktree path doesn't exist — try reconstructing from branch.
654            if let Some(ref b) = branch {
655                let project = repo
656                    .rsplit('/')
657                    .next()
658                    .unwrap_or(repo)
659                    .trim_end_matches(".git");
660                let candidate = worktrees_base.join(project).join(b);
661                if match fs::try_exists(&candidate).await {
662                    Ok(true) => true,
663                    Ok(false) => false,
664                    Err(e) => {
665                        tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
666                        false
667                    }
668                } {
669                    Some(candidate)
670                } else {
671                    None
672                }
673            } else {
674                None
675            }
676        }
677    } else if let Some(ref b) = branch {
678        // No worktree path in store — try branch-based path.
679        let project = repo
680            .rsplit('/')
681            .next()
682            .unwrap_or(repo)
683            .trim_end_matches(".git");
684        let candidate = worktrees_base.join(project).join(b);
685        if match fs::try_exists(&candidate).await {
686            Ok(true) => true,
687            Ok(false) => false,
688            Err(e) => {
689                tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
690                false
691            }
692        } {
693            Some(candidate)
694        } else {
695            None
696        }
697    } else {
698        None
699    };
700
701    let mut did_clean = false;
702
703    // If there's nothing to remove (no worktree path and no branch),
704    // treat as a no-op and return Ok(false) without attempting to
705    // resolve the repo root. Resolving the repo root can fail for
706    // projects not registered in config and that should not make
707    // cleanup a hard error when there is nothing to do.
708    let branch_nonempty = branch.as_ref().filter(|b| !b.is_empty());
709    if worktree_to_remove.is_none() && branch_nonempty.is_none() {
710        // If the stored worktree path is non-empty but doesn't exist on disk,
711        // the worktree is already gone — mark it cleaned so we stop retrying.
712        if worktree_path
713            .as_ref()
714            .is_some_and(|p| !p.as_os_str().is_empty())
715        {
716            tracing::debug!(
717                task_id,
718                worktree = ?worktree_path,
719                "stored worktree path no longer exists on disk — marking cleaned"
720            );
721            if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
722                && let Err(e) = store.mark_cleaned(store_id).await
723            {
724                tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned in store — will retry");
725            }
726        }
727        return Ok(false);
728    }
729
730    if let Some(wt) = worktree_to_remove {
731        // TTL guard: skip if the worktree directory is too young.
732        if let Some(age_hours) = worktree_age_hours(&wt).await
733            && age_hours < opts.ttl_hours
734        {
735            tracing::debug!(
736                task_id,
737                worktree = %wt.display(),
738                age_hours,
739                ttl_hours = opts.ttl_hours,
740                "worktree too young for cleanup, skipping"
741            );
742            return Ok(false);
743        }
744
745        // Tmux guard: skip if any active pane still has its cwd inside this worktree.
746        if is_worktree_in_active_session(&wt).await {
747            tracing::warn!(
748                task_id,
749                worktree = %wt.display(),
750                "worktree is referenced by an active tmux session — skipping cleanup"
751            );
752            return Ok(false);
753        }
754
755        // Validate git metadata before attempting any git operations.
756        // If the .git file points to a deleted gitdir the worktree is
757        // unrecoverable — force-remove the directory and skip git commands.
758        let gitdir_valid = worktree::validate_worktree_gitdir(&wt).await;
759
760        if opts.dry_run {
761            tracing::info!(
762                task_id,
763                worktree = %wt.display(),
764                branch = ?branch,
765                gitdir_valid,
766                "[dry-run] would remove worktree and branch"
767            );
768        } else if !gitdir_valid {
769            tracing::warn!(
770                task_id,
771                worktree = %wt.display(),
772                "worktree has broken git metadata — force-removing directory without git operations"
773            );
774            match fs::remove_dir_all(&wt).await {
775                Ok(()) => {
776                    did_clean = true;
777                }
778                Err(e) => {
779                    tracing::warn!(
780                        task_id,
781                        worktree = %wt.display(),
782                        err = %e,
783                        "failed to force-remove broken worktree directory"
784                    );
785                }
786            }
787        } else {
788            tracing::info!(task_id, worktree = %wt.display(), "removing worktree");
789            // Derive repo root from the worktree itself (for cross-project support).
790            // If the worktree belongs to a different project than the current repo
791            // context (e.g., internal task with worktree in another project), we can
792            // still find the repo root by examining the worktree's .git file.
793            let repo_root = match resolve_repo_root_from_worktree(&wt).await {
794                Ok(root) => root,
795                Err(_) => resolve_repo_root(repo).await?,
796            };
797            if let Some(br) = branch.as_deref().filter(|b| !b.is_empty())
798                && !pr_merged
799                && !rescue_and_notify(
800                    task_id,
801                    repo,
802                    external_id.as_deref(),
803                    Some(&wt),
804                    br,
805                    Path::new(&repo_root),
806                )
807                .await
808            {
809                return Ok(false);
810            }
811            if merged {
812                cow::warm_from_merged(
813                    Path::new(&repo_root),
814                    &worktree::project_name(Path::new(&repo_root)),
815                    &wt,
816                )
817                .await;
818            }
819            let removed = remove_worktree_and_branch_unchecked(
820                task_id,
821                &wt,
822                branch.as_deref(),
823                Path::new(&repo_root),
824                keep_remote_branch,
825            )
826            .await;
827            if removed {
828                did_clean = true;
829            } else {
830                tracing::warn!(
831                    task_id,
832                    worktree = %wt.display(),
833                    "worktree directory still exists after removal attempt — not marking as cleaned"
834                );
835            }
836        }
837    } else if let Some(ref br) = branch {
838        // Worktree directory is already gone, but the branch may still exist
839        // on the remote. Delete it to avoid orphaned branches.
840        if !opts.dry_run && !keep_remote_branch {
841            tracing::debug!(task_id, branch = %br, "no worktree on disk, cleaning up branch only");
842            // Resolve repo root lazily — branch cleanup needs it.
843            let repo_root = resolve_repo_root(repo).await?;
844            if !pr_merged
845                && !rescue_and_notify(
846                    task_id,
847                    repo,
848                    external_id.as_deref(),
849                    None,
850                    br,
851                    Path::new(&repo_root),
852                )
853                .await
854            {
855                return Ok(false);
856            }
857            delete_branches(task_id, br, Path::new(&repo_root)).await;
858            did_clean = true;
859        }
860    }
861
862    if did_clean {
863        // Mark as cleaned in store so we don't retry next tick.
864        // If resolve_task_id returns None the task is not (or no longer) in
865        // the store; that is fine — we already did the on-disk work.
866        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
867            && let Err(e) = store.mark_cleaned(store_id).await
868        {
869            tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned — will retry");
870            // Don't log activity; retry next tick will do the actual cleanup
871            return Ok(did_clean);
872        }
873        store_log_activity(
874            &Some(Arc::clone(store)),
875            repo,
876            task_id,
877            "branch_delete",
878            None,
879            None,
880            None,
881            None,
882            Some(&serde_json::json!({
883                "worktree": worktree_path.map(|p| p.display().to_string()),
884                "branch": branch,
885                "keep_remote_branch": keep_remote_branch,
886            })),
887        )
888        .await;
889    }
890
891    Ok(did_clean)
892}
893
894/// Result of saving a task branch's unpushed work before cleanup deletes it
895#[derive(Debug, PartialEq, Eq)]
896pub(crate) enum Rescue {
897    /// Every commit is on a remote ref already
898    Clean,
899    /// The commits were pushed to `rescue/<branch>`
900    Pushed {
901        branch: String,
902        commits: Vec<String>,
903    },
904    /// Git failed, so the worktree and branch must stay
905    Failed,
906}
907
908async fn git_in(dir: &Path, args: &[&str]) -> Option<Output> {
909    git_ops::git()
910        .await
911        .arg("-C")
912        .arg(dir)
913        .args(args)
914        .output_with_context()
915        .await
916        .ok()
917        .map(git_ops::sanitize_git_output)
918}
919
920/**
921 * Cleanup deletes the worktree and the local branch, which can be the only refs to commits the
922 * agent made before its run ended without a push (#3999). Uncommitted changes are committed
923 * first, then everything not on a remote ref goes to `rescue/<branch>`
924 **/
925pub(crate) async fn rescue_unpushed_work(
926    task_id: &str,
927    wt: Option<&Path>,
928    br: &str,
929    repo_root: &Path,
930) -> Rescue {
931    if let Some(wt) = wt
932        && fs::try_exists(wt.join(".git")).await.unwrap_or(false)
933    {
934        let Some(status) = git_in(wt, &["status", "--porcelain"]).await else {
935            tracing::warn!(task_id, branch = %br, "cannot read worktree status, keeping worktree");
936            return Rescue::Failed;
937        };
938        if !status.status.success() {
939            tracing::warn!(task_id, branch = %br, "git status failed, keeping worktree");
940            return Rescue::Failed;
941        }
942        if !status.stdout.is_empty() {
943            let committed = git_in(wt, &["add", "-A"])
944                .await
945                .is_some_and(|o| o.status.success())
946                && git_in(
947                    wt,
948                    &[
949                        "-c",
950                        "user.name=orch",
951                        "-c",
952                        "user.email=orch@localhost",
953                        "commit",
954                        "-m",
955                        "rescue: uncommitted changes saved by orch cleanup",
956                    ],
957                )
958                .await
959                .is_some_and(|o| o.status.success());
960            if !committed {
961                tracing::warn!(task_id, branch = %br, "cannot commit uncommitted changes, keeping worktree");
962                return Rescue::Failed;
963            }
964        }
965    }
966
967    let local_ref = format!("refs/heads/{br}");
968    let exists = git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
969        .await
970        .is_some_and(|o| o.status.success());
971    if !exists {
972        return Rescue::Clean;
973    }
974    // Commits on the local default branch survive the branch delete, so they do not count
975    let default_ref = format!(
976        "refs/heads/{}",
977        worktree::detect_default_branch(repo_root).await
978    );
979    // A bare clone has no remote-tracking refs, so ask the remote for the branch tip
980    let remote_tip = git_in(repo_root, &["ls-remote", "origin", &local_ref])
981        .await
982        .filter(|o| o.status.success())
983        .and_then(|o| {
984            String::from_utf8_lossy(&o.stdout)
985                .split_whitespace()
986                .next()
987                .map(str::to_string)
988        });
989    // A tip someone else pushed may be missing locally, and rev-list fails on an unknown object
990    let remote_tip = match remote_tip {
991        Some(tip) => {
992            let known = git_in(repo_root, &["cat-file", "-e", &format!("{tip}^{{commit}}")])
993                .await
994                .is_some_and(|o| o.status.success());
995            known.then_some(tip)
996        }
997        None => None,
998    };
999    let mut args = vec!["rev-list", "--oneline", &local_ref, "--not", "--remotes"];
1000    if let Some(tip) = remote_tip.as_deref() {
1001        args.push(tip);
1002    }
1003    let default_exists = default_ref != local_ref
1004        && git_in(
1005            repo_root,
1006            &["rev-parse", "--verify", "--quiet", &default_ref],
1007        )
1008        .await
1009        .is_some_and(|o| o.status.success());
1010    if default_exists {
1011        args.push(&default_ref);
1012    }
1013    let Some(list) = git_in(repo_root, &args)
1014        .await
1015        .filter(|o| o.status.success())
1016    else {
1017        tracing::warn!(task_id, branch = %br, "cannot list unpushed commits, keeping worktree and branch");
1018        return Rescue::Failed;
1019    };
1020    let commits: Vec<String> = String::from_utf8_lossy(&list.stdout)
1021        .lines()
1022        .map(str::to_string)
1023        .collect();
1024    if commits.is_empty() {
1025        return Rescue::Clean;
1026    }
1027
1028    let rescue = format!("rescue/{br}");
1029    let refspec = format!("{local_ref}:refs/heads/{rescue}");
1030    let pushed = git_in(
1031        repo_root,
1032        &["-c", "lfs.locksverify=false", "push", "origin", &refspec],
1033    )
1034    .await;
1035    if pushed.is_some_and(|o| o.status.success()) {
1036        // A bare clone has no fetchspec, so push does not create the tracking ref
1037        // Point that ref at the tip we just sent so delete_local_branch can see it
1038        let tracking = format!("refs/remotes/origin/{rescue}");
1039        if let Some(tip) = git_in(repo_root, &["rev-parse", &local_ref])
1040            .await
1041            .filter(|o| o.status.success())
1042        {
1043            let sha = String::from_utf8_lossy(&tip.stdout).trim().to_string();
1044            if !sha.is_empty() {
1045                let _ = git_in(repo_root, &["update-ref", &tracking, &sha]).await;
1046            }
1047        }
1048        tracing::warn!(task_id, branch = %br, rescue = %rescue, commits = commits.len(), "pushed unpushed commits to rescue branch");
1049        Rescue::Pushed {
1050            branch: rescue,
1051            commits,
1052        }
1053    } else {
1054        tracing::warn!(task_id, branch = %br, rescue = %rescue, "rescue push failed, keeping worktree and branch");
1055        Rescue::Failed
1056    }
1057}
1058
1059fn rescue_comment(rescue_branch: &str, commits: &[String]) -> String {
1060    let list = commits
1061        .iter()
1062        .map(|c| format!("- `{c}`"))
1063        .collect::<Vec<_>>()
1064        .join("\n");
1065    format!(
1066        "Cleanup found commits on this task's branch that were never pushed. They are saved on `{rescue_branch}`:\n\n{list}"
1067    )
1068}
1069
1070/// Save unpushed work and tell the task about it. Returns false when cleanup must not go on
1071async fn rescue_and_notify(
1072    task_id: &str,
1073    repo: &str,
1074    external_id: Option<&str>,
1075    wt: Option<&Path>,
1076    br: &str,
1077    repo_root: &Path,
1078) -> bool {
1079    match rescue_unpushed_work(task_id, wt, br, repo_root).await {
1080        Rescue::Clean => true,
1081        Rescue::Failed => false,
1082        Rescue::Pushed { branch, commits } => {
1083            if let Some(number) = external_id.filter(|n| n.parse::<u64>().is_ok()) {
1084                match GhHttp::new() {
1085                    Ok(gh) => {
1086                        if let Err(e) = gh
1087                            .add_comment(repo, number, &rescue_comment(&branch, &commits))
1088                            .await
1089                        {
1090                            tracing::warn!(task_id, err = %e, "failed to comment on rescue branch");
1091                        }
1092                    }
1093                    Err(e) => {
1094                        tracing::warn!(task_id, err = %e, "cannot comment on rescue branch");
1095                    }
1096                }
1097            }
1098            true
1099        }
1100    }
1101}
1102
1103/// Remove a git worktree directory and its local + remote branches.
1104/// When `keep_remote_branch` is true, only the worktree and local branch are
1105/// removed — the remote branch stays so that any linked PR remains open
1106/// (used for blocked tasks awaiting human review).
1107/// Returns `true` if the worktree directory is gone after the call
1108/// (either we removed it or it was already absent).
1109pub(crate) async fn remove_worktree_and_branch(
1110    task_id: &str,
1111    wt: &Path,
1112    branch: Option<&str>,
1113    repo_root: &Path,
1114    keep_remote_branch: bool,
1115) -> bool {
1116    remove_worktree_and_branch_for(task_id, wt, branch, repo_root, keep_remote_branch, false).await
1117}
1118
1119/// Same as `remove_worktree_and_branch`, but a merged PR skips the rescue. A squash leaves the
1120/// branch commits on no ref, so the rescue would push a branch for work that already landed
1121pub(crate) async fn remove_worktree_and_branch_for(
1122    task_id: &str,
1123    wt: &Path,
1124    branch: Option<&str>,
1125    repo_root: &Path,
1126    keep_remote_branch: bool,
1127    pr_merged: bool,
1128) -> bool {
1129    if !pr_merged
1130        && let Some(br) = branch.filter(|b| !b.is_empty())
1131        && rescue_unpushed_work(task_id, Some(wt), br, repo_root).await == Rescue::Failed
1132    {
1133        return false;
1134    }
1135
1136    remove_worktree_and_branch_unchecked(task_id, wt, branch, repo_root, keep_remote_branch).await
1137}
1138
1139/// The caller has already rescued the branch or knows its PR merged
1140async fn remove_worktree_and_branch_unchecked(
1141    task_id: &str,
1142    wt: &Path,
1143    branch: Option<&str>,
1144    repo_root: &Path,
1145    keep_remote_branch: bool,
1146) -> bool {
1147    let wt_str = wt.to_string_lossy().to_string();
1148    let repo_root_str = repo_root.to_string_lossy();
1149
1150    // Remove worktree FIRST, then delete the branch.
1151    // Git refuses to remove a worktree if its branch is already deleted.
1152    let remove_result = Command::new("git")
1153        .args([
1154            "-C",
1155            repo_root_str.as_ref(),
1156            "worktree",
1157            "remove",
1158            &wt_str,
1159            "--force",
1160        ])
1161        .output_with_context()
1162        .await;
1163
1164    match remove_result {
1165        Ok(output) if output.status.success() => {
1166            const MAX_RETRIES: usize = 3;
1167            const RETRY_DELAY_MS: u64 = 2000;
1168            tracing::info!(task_id, "worktree removed (git metadata)");
1169            // `git worktree remove` only removes git metadata — the physical directory
1170            // with working files may remain. Clean it up with retry logic, since macOS
1171            // sometimes needs a moment to release file handles from a recently-exited
1172            // process. Retrying with a delay addresses "Directory not empty" errors.
1173            let mut removed = false;
1174            for attempt in 0..MAX_RETRIES {
1175                match fs::remove_dir_all(wt).await {
1176                    Ok(()) => {
1177                        removed = true;
1178                        break;
1179                    }
1180                    Err(e) if e.kind() == ErrorKind::NotFound => {
1181                        // Directory was already removed by git or another cleanup pass.
1182                        removed = true;
1183                        break;
1184                    }
1185                    Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => {
1186                        if attempt < MAX_RETRIES - 1 {
1187                            tracing::debug!(
1188                                task_id,
1189                                path = %wt.display(),
1190                                attempt = attempt + 1,
1191                                "worktree directory not empty — retrying after delay"
1192                            );
1193                            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
1194                        } else {
1195                            tracing::warn!(
1196                                task_id,
1197                                path = %wt.display(),
1198                                err = %e,
1199                                "failed to remove worktree directory after {} attempts (Directory not empty)",
1200                                MAX_RETRIES
1201                            );
1202                        }
1203                    }
1204                    Err(e) => {
1205                        tracing::warn!(task_id, path = %wt.display(), err = %e, "failed to remove worktree directory");
1206                        break;
1207                    }
1208                }
1209            }
1210            if removed {
1211                tracing::info!(task_id, path = %wt.display(), "worktree physical directory removed");
1212            } else {
1213                tracing::warn!(
1214                    task_id,
1215                    path = %wt.display(),
1216                    "worktree directory still present after git worktree remove"
1217                );
1218            }
1219        }
1220        Ok(output) => {
1221            let stderr = String::from_utf8_lossy(&output.stderr);
1222            // "is not a working tree" means git's metadata is stale — the path
1223            // exists on disk but is no longer registered as a worktree. Treat
1224            // this as an already-gone worktree: physically remove the directory
1225            // and prune stale git metadata. Downgrade to debug — this is an
1226            // expected idempotency case, not a real error.
1227            if stderr.contains("is not a working tree") {
1228                tracing::debug!(
1229                    task_id,
1230                    path = %wt.display(),
1231                    "path is not a registered worktree (stale metadata) — removing directory and pruning"
1232                );
1233                if let Err(e) = fs::remove_dir_all(wt).await {
1234                    // Only warn if the directory actually still exists after removal attempt.
1235                    if match fs::try_exists(wt).await {
1236                        Ok(true) => true,
1237                        Ok(false) => false,
1238                        Err(e) => {
1239                            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree does not exist");
1240                            false
1241                        }
1242                    } {
1243                        tracing::warn!(
1244                            task_id,
1245                            path = %wt.display(),
1246                            err = %e,
1247                            "failed to remove stale worktree directory"
1248                        );
1249                    }
1250                } else {
1251                    tracing::debug!(task_id, path = %wt.display(), "stale worktree directory removed");
1252                }
1253                // Prune stale worktree entries from git metadata so the warning
1254                // stops recurring on future `git worktree list` / cleanup cycles.
1255                if let Err(e) = Command::new("git")
1256                    .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1257                    .output_with_context()
1258                    .await
1259                {
1260                    tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1261                }
1262            } else {
1263                tracing::warn!(task_id, err = %stderr, "failed to remove worktree");
1264            }
1265        }
1266        Err(e) => {
1267            tracing::warn!(task_id, err = %e, "failed to remove worktree");
1268        }
1269    }
1270
1271    // Prune stale .git/worktrees/ entries left behind after removal.
1272    if let Err(e) = Command::new("git")
1273        .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1274        .output_with_context()
1275        .await
1276    {
1277        tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1278    }
1279
1280    // Delete local and remote branch from the main repo root (worktree is already gone).
1281    // When keep_remote_branch is set, only delete the local branch — the remote stays
1282    // so any linked PR remains open for human review.
1283    if let Some(br) = branch {
1284        if keep_remote_branch {
1285            delete_local_branch(task_id, br, repo_root).await;
1286        } else {
1287            delete_branches(task_id, br, repo_root).await;
1288        }
1289    }
1290
1291    // Definitively check: is the worktree directory actually gone?
1292    match fs::try_exists(wt).await {
1293        Ok(false) => true, // confirmed gone
1294        Ok(true) => false, // still there
1295        Err(e) => {
1296            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree still present");
1297            false // conservative: don't claim success
1298        }
1299    }
1300}
1301
1302/// Whether `refs/heads/{br}` still exists in `repo_root`
1303pub(crate) async fn local_branch_exists(repo_root: &Path, br: &str) -> bool {
1304    let local_ref = format!("refs/heads/{br}");
1305    git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
1306        .await
1307        .is_some_and(|o| o.status.success())
1308}
1309
1310/// Commits reachable from `rev` and from none of `others`. `None` when git cannot answer
1311async fn commits_outside(repo_root: &Path, rev: &str, others: &[&str]) -> Option<u64> {
1312    let mut args = vec![
1313        "rev-list".to_string(),
1314        "--count".to_string(),
1315        rev.to_string(),
1316    ];
1317    // One `--not` then every other ref, a `--not` per ref would toggle and count the wrong set
1318    let mut exclude = Vec::new();
1319    for other in others {
1320        let exists = Command::new("git")
1321            .args([
1322                "-C",
1323                &repo_root.to_string_lossy(),
1324                "rev-parse",
1325                "--verify",
1326                "--quiet",
1327                other,
1328            ])
1329            .output_with_context()
1330            .await
1331            .is_ok_and(|o| o.status.success());
1332        if exists {
1333            exclude.push(other.to_string());
1334        }
1335    }
1336    if !exclude.is_empty() {
1337        args.push("--not".to_string());
1338        args.extend(exclude);
1339    }
1340    let output = Command::new("git")
1341        .arg("-C")
1342        .arg(repo_root)
1343        .args(&args)
1344        .output_with_context()
1345        .await
1346        .ok()?;
1347    if !output.status.success() {
1348        return None;
1349    }
1350    String::from_utf8_lossy(&output.stdout).trim().parse().ok()
1351}
1352
1353/// Delete the local branch only when origin or the default branch holds every commit on it
1354async fn delete_local_branch(task_id: &str, br: &str, repo_root: &Path) {
1355    let default_branch = worktree::detect_default_branch(repo_root).await;
1356    let remote_ref = format!("refs/remotes/origin/{br}");
1357    let default_ref = format!("refs/remotes/origin/{default_branch}");
1358    let local_ref = format!("refs/heads/{br}");
1359    let local_default_ref = format!("refs/heads/{default_branch}");
1360    let rescue_ref = format!("refs/remotes/origin/rescue/{br}");
1361    match commits_outside(
1362        repo_root,
1363        &local_ref,
1364        &[&remote_ref, &default_ref, &local_default_ref, &rescue_ref],
1365    )
1366    .await
1367    {
1368        Some(0) => {}
1369        Some(n) => {
1370            tracing::warn!(task_id, branch = %br, unpushed = n, "local branch holds commits origin does not have, keeping it");
1371            return;
1372        }
1373        None => {
1374            tracing::debug!(task_id, branch = %br, "local branch missing or unreadable, nothing to delete");
1375            return;
1376        }
1377    }
1378    let output = Command::new("git")
1379        .args(["-C", &repo_root.to_string_lossy(), "branch", "-D", br])
1380        .output_with_context()
1381        .await;
1382    match output {
1383        Ok(o) if o.status.success() => {
1384            tracing::info!(task_id, branch = %br, "local branch deleted");
1385        }
1386        Ok(o) => {
1387            let stderr = String::from_utf8_lossy(&o.stderr);
1388            tracing::debug!(task_id, err = %stderr, "local branch delete skipped (may not exist)");
1389        }
1390        Err(e) => {
1391            tracing::warn!(task_id, err = %e, "failed to delete local branch");
1392        }
1393    }
1394}
1395
1396/**
1397 * Delete local and remote branches for a task, but never the only copy of a commit.
1398 * The remote branch goes only when the default branch already contains all of it, which
1399 * covers fast-forward and merge-commit merges. A squash-merged branch stays, and GitHub's
1400 * "automatically delete head branches" removes it. Every other caller (startup rebase
1401 * failure, closed-issue reconciliation, orphan sweeps) may hold unmerged work (#3781)
1402 **/
1403pub(crate) async fn delete_branches(task_id: &str, br: &str, repo_root: &Path) {
1404    if br == SCREENSHOTS_BRANCH {
1405        return;
1406    }
1407    let default_branch = worktree::detect_default_branch(repo_root).await;
1408    let _ = git_ops::git()
1409        .await
1410        .args([
1411            "-C",
1412            &repo_root.to_string_lossy(),
1413            "fetch",
1414            "origin",
1415            &default_branch,
1416            br,
1417        ])
1418        .output_with_context()
1419        .await;
1420
1421    delete_local_branch(task_id, br, repo_root).await;
1422
1423    let remote_ref = format!("refs/remotes/origin/{br}");
1424    let default_ref = format!("refs/remotes/origin/{default_branch}");
1425    match commits_outside(repo_root, &remote_ref, &[&default_ref]).await {
1426        Some(0) => {}
1427        Some(n) => {
1428            tracing::info!(task_id, branch = %br, unmerged = n, "remote branch has commits the default branch lacks, keeping it");
1429            return;
1430        }
1431        None => {
1432            tracing::debug!(task_id, branch = %br, "remote branch missing or unreadable, nothing to delete");
1433            return;
1434        }
1435    }
1436
1437    let remote_delete = git_ops::git()
1438        .await
1439        .args([
1440            "-C",
1441            &repo_root.to_string_lossy(),
1442            "-c",
1443            "lfs.locksverify=false",
1444            "push",
1445            "origin",
1446            "--delete",
1447            br,
1448        ])
1449        .output_with_context()
1450        .await;
1451
1452    match remote_delete {
1453        Ok(output) if output.status.success() => {
1454            tracing::info!(task_id, branch = %br, "remote branch deleted");
1455            git_ops::clear_push_lease(repo_root, br).await;
1456        }
1457        Ok(output) => {
1458            let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
1459            tracing::debug!(task_id, err = %stderr, "remote branch delete skipped");
1460        }
1461        Err(e) => {
1462            tracing::warn!(task_id, err = %e, "failed to delete remote branch");
1463        }
1464    }
1465}
1466
1467/// Returns `true` if any active tmux pane has its cwd inside `worktree`.
1468///
1469/// Uses `tmux list-panes -a -F '#{pane_current_path}'` to enumerate all
1470/// pane working directories across every session. If tmux is not running
1471/// (command fails), we conservatively return `false` so cleanup can proceed.
1472async fn is_worktree_in_active_session(worktree: &Path) -> bool {
1473    let output = Command::new("tmux")
1474        .args(["list-panes", "-a", "-F", "#{pane_current_path}"])
1475        .output()
1476        .await;
1477
1478    match output {
1479        Ok(o) if o.status.success() => {
1480            let worktree_str = worktree.to_string_lossy();
1481            String::from_utf8_lossy(&o.stdout).lines().any(|line| {
1482                let pane_path = line.trim();
1483                // Match exact path or any path that starts with worktree + separator.
1484                pane_path == worktree_str.as_ref()
1485                    || pane_path.starts_with(&format!("{worktree_str}/"))
1486            })
1487        }
1488        // tmux not running or no sessions — treat as safe to clean.
1489        _ => false,
1490    }
1491}
1492
1493/// Returns how many hours old the worktree directory is, based on its mtime.
1494///
1495/// Returns `None` if the directory does not exist or the mtime cannot be read.
1496async fn worktree_age_hours(worktree: &Path) -> Option<u64> {
1497    // Use tokio::fs to avoid blocking the async runtime when reading metadata.
1498    let metadata = fs::metadata(worktree).await.ok()?;
1499    let modified = metadata.modified().ok()?;
1500    let age = SystemTime::now().duration_since(modified).ok()?;
1501    Some(age.as_secs() / 3600)
1502}
1503
1504/// Resolve the repo root for an orphaned worktree and remove it.
1505///
1506/// Used by `orch service doctor --fix` and `orch service prune` to clean up worktrees that are
1507/// no longer owned by any task. Returns `true` if the worktree directory is gone
1508/// after the call.
1509pub(crate) async fn cleanup_orphaned_worktree(task_id: &str, wt: &Path) -> bool {
1510    let repo_root = match resolve_repo_root_for_orphaned_worktree(wt).await {
1511        Ok(root) => root,
1512        Err(e) => {
1513            tracing::warn!(
1514                worktree = %wt.display(),
1515                error = %e,
1516                "cannot resolve repo root for orphaned worktree"
1517            );
1518            return false;
1519        }
1520    };
1521    remove_worktree_and_branch(task_id, wt, None, Path::new(&repo_root), false).await
1522}
1523
1524/// Resolve the main git repository root for an orphaned worktree.
1525///
1526/// First tries to parse the `.git` file's `gitdir:` pointer (two levels up from
1527/// the pointer gives the bare repo root, which is the orch convention). Falls back
1528/// to looking up registered projects by directory name when the `.git` file is
1529/// absent or unparseable.
1530pub(crate) async fn resolve_repo_root_for_orphaned_worktree(wt: &Path) -> anyhow::Result<String> {
1531    let git_file = wt.join(".git");
1532    if fs::try_exists(&git_file).await.unwrap_or(false)
1533        && let Ok(content) = fs::read_to_string(&git_file).await
1534    {
1535        for line in content.lines() {
1536            if let Some(gitdir) = line.strip_prefix("gitdir: ") {
1537                // gitdir points to: <repo>/.git/worktrees/<name> (non-bare)
1538                //                or <repo>.git/worktrees/<name>   (bare)
1539                // Two levels up reaches the .git dir / bare repo root.
1540                let gitdir_path = PathBuf::from(gitdir);
1541                if let Some(repo_root) = gitdir_path.parent().and_then(|p| p.parent()) {
1542                    return Ok(repo_root.display().to_string());
1543                }
1544            }
1545        }
1546    }
1547
1548    // Fallback: look up registered projects by the project-name component of the
1549    // worktree path (format: ~/.orch/worktrees/<project>/<branch>).
1550    let worktrees_dir = home::worktrees_dir()?;
1551    if let Ok(relative) = wt.strip_prefix(&worktrees_dir)
1552        && let Some(project_name) = relative.components().next()
1553    {
1554        let project_name = project_name.as_os_str().to_string_lossy();
1555
1556        if let Ok(paths) = config::get_project_paths() {
1557            for path_str in &paths {
1558                let path = Path::new(path_str);
1559                if let Some(name) = path.file_name()
1560                    && name.to_string_lossy() == project_name
1561                {
1562                    return Ok(path_str.clone());
1563                }
1564            }
1565        }
1566
1567        // Try bare clone at ~/.orch/projects/<owner>/<project>.git
1568        let parts: Vec<&str> = project_name.split('/').collect();
1569        let bare = if parts.len() == 2 {
1570            home::projects_dir()
1571                .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1572                .unwrap_or_default()
1573        } else {
1574            return Err(anyhow::anyhow!(
1575                "cannot determine owner for single-segment project name '{project_name}' — \
1576                     bare repo location is ambiguous; register the project with 'owner/repo' format"
1577            ));
1578        };
1579        if fs::try_exists(&bare).await.unwrap_or(false) {
1580            return Ok(bare.display().to_string());
1581        }
1582    }
1583
1584    anyhow::bail!(
1585        "cannot resolve repo root for orphaned worktree at {} — tried .git parsing and project lookup",
1586        wt.display()
1587    )
1588}
1589
1590/// Resolve the main git repository root from a worktree's .git file.
1591///
1592/// For cross-project worktree cleanup, we cannot rely on `resolve_repo_root` because
1593/// the worktree may belong to a different project than the current repo context.
1594/// Instead, we extract the repo root from the worktree's `.git` file which contains
1595/// a `gitdir:` pointer to the main repo's git directory.
1596pub async fn resolve_repo_root_from_worktree(wt: &Path) -> anyhow::Result<String> {
1597    // For worktrees, .git is a file (not a directory) containing gitdir path
1598    let git_file = wt.join(".git");
1599    if !fs::try_exists(&git_file).await.unwrap_or(false) {
1600        anyhow::bail!(".git file not found in worktree at {}", wt.display());
1601    }
1602
1603    let content = fs::read_to_string(&git_file)
1604        .await
1605        .map_err(|e| anyhow::anyhow!("failed to read .git file: {e}"))?;
1606
1607    // Parse gitdir line: "gitdir: /path/to/repo/.git/worktrees/<name>"
1608    for line in content.lines() {
1609        if let Some(gitdir) = line.strip_prefix("gitdir:") {
1610            let gitdir = gitdir.trim();
1611            // The main repo is the parent of the .git directory
1612            // gitdir points to: <repo>/.git/worktrees/<name>
1613            // We need to go up 3 levels: worktrees/<name> -> .git -> repo
1614            let gitdir_path = Path::new(gitdir);
1615            if let Some(repo_root) = gitdir_path
1616                .parent() // worktrees/<name>
1617                .and_then(|p| p.parent()) // .git
1618                .and_then(|p| p.parent())
1619            // repo root
1620            {
1621                return Ok(repo_root.display().to_string());
1622            }
1623        }
1624    }
1625
1626    anyhow::bail!(
1627        "cannot resolve repo root from worktree .git file at {}",
1628        wt.display()
1629    )
1630}
1631
1632/// Resolve the main git repository root path for a project.
1633///
1634/// Looks up the local project path from config, then verifies it's a git repo.
1635/// This avoids relying on cwd (which is undefined under launchd services).
1636pub(crate) async fn resolve_repo_root(repo: &str) -> anyhow::Result<String> {
1637    // Look up the local path from registered projects
1638    let paths = config::get_project_paths().unwrap_or_default();
1639    for path_str in &paths {
1640        let path = Path::new(path_str);
1641        let orch_yml = path.join(".orch.yml");
1642        let legacy = path.join(".orchestrator.yml");
1643        let config_file = if fs::try_exists(&orch_yml).await.unwrap_or(false) {
1644            orch_yml
1645        } else if fs::try_exists(&legacy).await.unwrap_or(false) {
1646            legacy
1647        } else {
1648            continue;
1649        };
1650        if let Ok(content) = fs::read_to_string(&config_file).await
1651            && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1652            && let Some(r) = doc
1653                .get("gh")
1654                .and_then(|gh| gh.get("repo"))
1655                .and_then(|r| r.as_str())
1656            && r == repo
1657        {
1658            return Ok(path_str.clone());
1659        }
1660    }
1661
1662    // Fallback: try bare clone in ~/.orch/projects/<owner>/<repo>.git
1663    let parts: Vec<&str> = repo.split('/').collect();
1664    let bare = if parts.len() == 2 {
1665        home::projects_dir()
1666            .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1667            .unwrap_or_default()
1668    } else {
1669        PathBuf::new()
1670    };
1671    if fs::try_exists(&bare).await.unwrap_or(false) {
1672        return Ok(bare.display().to_string());
1673    }
1674
1675    anyhow::bail!(
1676        "cannot find local path for project {repo} — \
1677         checked {} registered project(s) and bare clone at {}",
1678        paths.len(),
1679        bare.display()
1680    )
1681}
1682
1683/// Resolve the project directory for a given repo string.
1684///
1685/// Replicates the logic from `TaskRunner::resolve_project_dir()` without requiring
1686/// a Runner instance. Used by the orphan scanner to enumerate active worktrees.
1687fn resolve_project_dir_for_repo(repo: &str) -> anyhow::Result<PathBuf> {
1688    // Check PROJECT_DIR env var first
1689    if let Ok(dir) = env::var("PROJECT_DIR")
1690        && !dir.is_empty()
1691    {
1692        return Ok(PathBuf::from(dir));
1693    }
1694
1695    // Look up registered projects in config
1696    if let Ok(paths) = config::get_project_paths() {
1697        for path_str in &paths {
1698            let path = Path::new(path_str);
1699            let orch_yml = path.join(".orch.yml");
1700            let legacy = path.join(".orchestrator.yml");
1701            let config_file = if Path::new(&orch_yml).exists() {
1702                orch_yml
1703            } else if Path::new(&legacy).exists() {
1704                legacy
1705            } else {
1706                continue;
1707            };
1708            if let Ok(content) = std_fs::read_to_string(&config_file)
1709                && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1710                && let Some(r) = doc
1711                    .get("gh")
1712                    .and_then(|gh| gh.get("repo"))
1713                    .and_then(|r| r.as_str())
1714                && r == repo
1715            {
1716                return Ok(path_str.clone().into());
1717            }
1718        }
1719    }
1720
1721    anyhow::bail!("project dir not found for repo: {repo}")
1722}
1723
1724/// Check for merged PRs and update task status accordingly.
1725///
1726/// Queries status:in_review and status:needs_review tasks, checks if their PR
1727/// is merged, and updates status to done if merged.
1728pub(crate) async fn check_merged_prs(
1729    backend: &Arc<dyn ExternalBackend>,
1730    repo: &str,
1731    store: &Arc<TaskStore>,
1732    task_manager: &Arc<TaskManager>,
1733    in_review_tasks: &[ReviewTaskSnapshot],
1734    needs_review_tasks: &[ReviewTaskSnapshot],
1735) -> anyhow::Result<()> {
1736    let mut branch_map: HashMap<String, String> = HashMap::new();
1737    let all_review_tasks: Vec<&ReviewTaskSnapshot> = in_review_tasks
1738        .iter()
1739        .chain(needs_review_tasks.iter())
1740        .collect();
1741
1742    for task in &all_review_tasks {
1743        if !task.stored.branch.is_empty() {
1744            branch_map.insert(task.external.id.0.clone(), task.stored.branch.clone());
1745        }
1746    }
1747
1748    tracing::debug!(
1749        count = all_review_tasks.len(),
1750        "checking review tasks for merged PRs"
1751    );
1752
1753    // Collect (task_id, branch) pairs — skip tasks with no branch recorded.
1754    // Use the pre-built branch map; fall back to store lookup only for tasks
1755    // not in the map (e.g. internal tasks, backend fallback path).
1756    let mut task_branches: Vec<(String, String)> = Vec::new();
1757    for task in all_review_tasks {
1758        let task_id = &task.external.id.0;
1759        let branch = if let Some(b) = branch_map.get(task_id) {
1760            b.clone()
1761        } else {
1762            match store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
1763                .await
1764                .map(|t| t.branch)
1765            {
1766                Some(b) if !b.is_empty() => b,
1767                _ => {
1768                    tracing::debug!(task_id, "no branch info, skipping PR check");
1769                    continue;
1770                }
1771            }
1772        };
1773        task_branches.push((task_id.clone(), branch));
1774    }
1775
1776    if task_branches.is_empty() {
1777        return Ok(());
1778    }
1779
1780    // Single GraphQL call for all branches — N REST calls → 1 GraphQL call.
1781    let branches: Vec<String> = task_branches.iter().map(|(_, b)| b.clone()).collect();
1782    let merged_map = match backend.batch_is_pr_merged(&branches).await {
1783        Ok(map) => map,
1784        Err(e) => {
1785            tracing::warn!(
1786                err = %e,
1787                branch_count = branches.len(),
1788                "batch PR merge check failed, retrying affected branches individually"
1789            );
1790            fallback_is_pr_merged_by_branch(backend, &branches).await
1791        }
1792    };
1793
1794    for (task_id, branch) in task_branches {
1795        let is_merged = merged_map.get(&branch).copied().unwrap_or(false);
1796        if !is_merged {
1797            continue;
1798        }
1799
1800        tracing::info!(task_id, branch = %branch, "PR merged, marking task complete");
1801
1802        let stored = store::opt_store_get_task(&Some(Arc::clone(store)), repo, &task_id).await;
1803        if let Some(ref stored) = stored {
1804            let external = store_task_to_external(stored);
1805            if ci_bootstrap::hold_after_merge(&external, store, repo).await {
1806                tracing::info!(
1807                    task_id,
1808                    branch = %branch,
1809                    "CI bootstrap PR merged, holding done until the ruleset exists"
1810                );
1811                if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1812                    tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1813                }
1814                continue;
1815            }
1816        }
1817
1818        let id = ExternalId(task_id.clone());
1819        if let Err(e) = task_manager.update_task_status(&id, Status::Done).await {
1820            tracing::warn!(task_id, err = %e, "failed to update status to done");
1821            continue;
1822        }
1823
1824        if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1825            tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1826        }
1827
1828        let comment = format!("PR merged, marking task complete{}", engine::orch_footer());
1829        if let Err(e) = backend.post_comment(&id, &comment).await {
1830            tracing::warn!(task_id, err = %e, "failed to post comment");
1831        }
1832    }
1833
1834    Ok(())
1835}
1836
1837async fn fallback_is_pr_merged_by_branch(
1838    backend: &Arc<dyn ExternalBackend>,
1839    branches: &[String],
1840) -> HashMap<String, bool> {
1841    let futs: Vec<_> = branches
1842        .iter()
1843        .map(|branch| {
1844            let backend = Arc::clone(backend);
1845            let branch = branch.clone();
1846            async move {
1847                match backend.is_pr_merged(&branch).await {
1848                    Ok(is_merged) => Some((branch, is_merged)),
1849                    Err(err) => {
1850                        tracing::warn!(
1851                            branch,
1852                            err = %err,
1853                            "fallback PR merge check failed for branch"
1854                        );
1855                        None
1856                    }
1857                }
1858            }
1859        })
1860        .collect();
1861
1862    future::join_all(futs).await.into_iter().flatten().collect()
1863}
1864
1865#[cfg(test)]
1866mod tests {
1867    use super::*;
1868    use crate::engine::tasks;
1869    use crate::store::{NewTask, TaskStore};
1870    use async_trait::async_trait;
1871    use std::collections::HashMap;
1872    use std::process;
1873    use std::sync::Mutex;
1874    use std::sync::atomic::{AtomicUsize, Ordering};
1875
1876    // worktree_age_hours
1877
1878    #[tokio::test]
1879    async fn worktree_age_hours_returns_none_for_missing_dir() {
1880        let tmp = tempfile::tempdir().unwrap();
1881        let missing = tmp.path().join("does-not-exist");
1882        assert!(worktree_age_hours(&missing).await.is_none());
1883    }
1884
1885    #[tokio::test]
1886    async fn worktree_age_hours_returns_some_for_existing_dir() {
1887        let tmp = tempfile::tempdir().unwrap();
1888        // A freshly created directory should have age ~0 hours.
1889        let age = worktree_age_hours(tmp.path()).await;
1890        assert!(age.is_some());
1891        assert_eq!(age.unwrap(), 0, "freshly created dir should have age 0h");
1892    }
1893
1894    // JanitorOptions::from_config defaults
1895
1896    #[test]
1897    fn janitor_options_default_values() {
1898        let opts = JanitorOptions::default();
1899        assert_eq!(opts.ttl_hours, 24);
1900        assert!(!opts.dry_run);
1901    }
1902
1903    // cleanup_task_worktree_with_opts TTL guard (skips if git missing)
1904
1905    /// Create a minimal git repo + worktree; return (tmp, worktree_dir).
1906    fn setup_test_repo() -> Option<(tempfile::TempDir, PathBuf)> {
1907        let tmp = tempfile::tempdir().ok()?;
1908
1909        // Create a real git repo (not bare) so we can add worktrees.
1910        let repo_dir = tmp.path().join("repo.git");
1911        std_fs::create_dir_all(&repo_dir).ok()?;
1912
1913        let init = process::Command::new("git")
1914            .args([
1915                "init",
1916                "--initial-branch=main",
1917                repo_dir
1918                    .to_str()
1919                    .expect("test repo path contains non-UTF-8 characters"),
1920            ])
1921            .output()
1922            .ok()?;
1923        if !init.status.success() {
1924            // Older git may not have --initial-branch; fall back.
1925            let _ = process::Command::new("git")
1926                .args([
1927                    "init",
1928                    repo_dir
1929                        .to_str()
1930                        .expect("test repo path contains non-UTF-8 characters"),
1931                ])
1932                .output()
1933                .ok()?;
1934        }
1935
1936        // Create an initial commit so the repo has a HEAD.
1937        let readme = repo_dir.join("README.md");
1938        std_fs::write(&readme, "# test").ok()?;
1939        process::Command::new("git")
1940            .args(["-C", repo_dir.to_str()?, "add", "."])
1941            .output()
1942            .ok()?;
1943        process::Command::new("git")
1944            .args([
1945                "-C",
1946                repo_dir.to_str()?,
1947                "-c",
1948                "user.email=test@test.com",
1949                "-c",
1950                "user.name=Test",
1951                "-c",
1952                "commit.gpgsign=false",
1953                "commit",
1954                "--allow-empty",
1955                "-m",
1956                "init",
1957            ])
1958            .output()
1959            .ok()?;
1960
1961        // Create a worktree branch.
1962        let wt_dir = tmp
1963            .path()
1964            .join("worktrees")
1965            .join("repo.git")
1966            .join("gh-issue-42-test");
1967        std_fs::create_dir_all(wt_dir.parent()?).ok()?;
1968        let wt_out = process::Command::new("git")
1969            .args([
1970                "-C",
1971                repo_dir.to_str()?,
1972                "worktree",
1973                "add",
1974                "-b",
1975                "gh-issue-42-test",
1976                wt_dir.to_str()?,
1977            ])
1978            .output()
1979            .ok()?;
1980
1981        if !wt_out.status.success() {
1982            return None;
1983        }
1984
1985        Some((tmp, wt_dir))
1986    }
1987
1988    #[tokio::test]
1989    async fn janitor_skips_young_worktree() {
1990        let Some((tmp, wt_dir)) = setup_test_repo() else {
1991            eprintln!("skipping test: git not available");
1992            return;
1993        };
1994
1995        // The worktree was just created → age is 0h < ttl_hours=24 → must be skipped.
1996        assert!(wt_dir.exists(), "worktree should exist before janitor runs");
1997
1998        // Run the TTL guard check directly (without a full engine).
1999        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2000        assert!(
2001            age < 24,
2002            "freshly created worktree should be younger than 24h (age={age}h)"
2003        );
2004
2005        // Explicitly verify: with ttl_hours=24 the worktree stays.
2006        let opts = JanitorOptions {
2007            ttl_hours: 24,
2008            dry_run: true,
2009        };
2010        // Age < ttl → should skip (worktree_age_hours < opts.ttl_hours).
2011        assert!(age < opts.ttl_hours, "janitor should skip young worktree");
2012
2013        drop(tmp); // cleanup tempdir
2014    }
2015
2016    #[tokio::test]
2017    async fn janitor_eligible_when_ttl_zero() {
2018        let Some((tmp, wt_dir)) = setup_test_repo() else {
2019            eprintln!("skipping test: git not available");
2020            return;
2021        };
2022
2023        // With ttl_hours=0 even a brand-new worktree is eligible.
2024        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2025        let opts = JanitorOptions {
2026            ttl_hours: 0,
2027            dry_run: true,
2028        };
2029        assert!(
2030            age >= opts.ttl_hours,
2031            "worktree should be eligible when ttl=0 (age={age}h)"
2032        );
2033
2034        drop(tmp);
2035    }
2036
2037    #[tokio::test]
2038    async fn dry_run_does_not_remove_worktree() {
2039        // Verify that with dry_run=true the worktree directory is NOT removed.
2040        let Some((tmp, wt_dir)) = setup_test_repo() else {
2041            eprintln!("skipping test: git not available");
2042            return;
2043        };
2044
2045        assert!(wt_dir.exists(), "worktree must exist before test");
2046
2047        // With dry_run=true and ttl_hours=0 the janitor would normally remove the
2048        // worktree, but dry_run prevents it.
2049        let opts = JanitorOptions {
2050            ttl_hours: 0,
2051            dry_run: true,
2052        };
2053
2054        // The janitor needs a repo_root to run git commands.  We point it at the
2055        // bare repo we created inside tmp (even though it's not registered in config).
2056        // Because resolve_repo_root will fail for our synthetic repo slug, we test the
2057        // inner helpers directly here.
2058        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2059        let in_use = false; // no tmux running in tests
2060        let skip = age < opts.ttl_hours || in_use;
2061
2062        if !skip && opts.dry_run {
2063            // dry-run: log only, don't touch filesystem.
2064            tracing::debug!("[dry-run] would remove {}", wt_dir.display());
2065        } else if !skip {
2066            // This branch would do actual removal — but we're in dry_run mode above.
2067        }
2068
2069        // Worktree must still be there.
2070        assert!(
2071            wt_dir.exists(),
2072            "dry-run must not remove the worktree directory"
2073        );
2074
2075        drop(tmp);
2076    }
2077
2078    #[test]
2079    fn fallback_branch_path_construction() {
2080        // Verify the new fallback logic: when worktree path is absent but
2081        // branch is known, we should construct the path correctly.
2082        let tmp = tempfile::tempdir().unwrap();
2083        let wt_dir = tmp.path().join("myrepo").join("gh-issue-7-fix");
2084        std_fs::create_dir_all(&wt_dir).unwrap();
2085
2086        // Simulate: worktree=None, branch=Some("gh-issue-7-fix"), repo="owner/myrepo"
2087        let branch = "gh-issue-7-fix";
2088        let repo = "owner/myrepo";
2089        let worktrees_base = tmp.path();
2090
2091        let project = repo
2092            .rsplit('/')
2093            .next()
2094            .unwrap_or(repo)
2095            .trim_end_matches(".git");
2096        let candidate = worktrees_base.join(project).join(branch);
2097
2098        assert_eq!(candidate, wt_dir);
2099        assert!(
2100            candidate.exists(),
2101            "fallback path should point to existing dir"
2102        );
2103    }
2104
2105    /// A task with no worktree and no branch should return Ok(false) —
2106    /// nothing was cleaned, so git pull should NOT be triggered.
2107    #[tokio::test]
2108    async fn cleanup_returns_false_when_nothing_to_clean() {
2109        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2110
2111        // Create a done task with no worktree or branch fields set
2112        let id = store
2113            .create(&NewTask {
2114                external_id: Some("999".to_string()),
2115                repo: "owner/repo".to_string(),
2116                origin: "github".to_string(),
2117                title: "Already cleaned".to_string(),
2118                ..Default::default()
2119            })
2120            .await
2121            .unwrap();
2122        store.update_status(id, TaskStatus::Done).await.unwrap();
2123
2124        let opts = JanitorOptions {
2125            ttl_hours: 0,
2126            dry_run: false,
2127        };
2128
2129        let result = cleanup_task_worktree_with_opts("999", "owner/repo", &store, &opts).await;
2130        // resolve_repo_root will fail for synthetic repo — that's OK, the point is
2131        // that when nothing needs cleaning the function should not report true.
2132        // In production this means no git pull is triggered.
2133        if let Ok(cleaned) = result {
2134            assert!(
2135                !cleaned,
2136                "cleanup should return false when there is no worktree or branch to clean"
2137            );
2138        } else {
2139            // resolve_repo_root fails for test repo — acceptable.
2140            // The bug was that even with no worktree/branch, Ok(true) was returned.
2141        }
2142    }
2143
2144    /// When the stored worktree path no longer exists on disk and there is no
2145    /// branch recorded, cleanup must mark the task as cleaned so subsequent
2146    /// runs do not keep retrying (idempotency regression test for #1021).
2147    #[tokio::test]
2148    async fn cleanup_marks_cleaned_when_stored_path_is_gone() {
2149        let tmp = tempfile::tempdir().unwrap();
2150        // A path that does NOT exist on disk.
2151        let stale_wt = tmp.path().join("stale-worktree");
2152        assert!(!stale_wt.exists(), "stale path must not exist");
2153
2154        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2155
2156        let id = store
2157            .create(&NewTask {
2158                external_id: Some("1005".to_string()),
2159                repo: "owner/repo".to_string(),
2160                origin: "github".to_string(),
2161                title: "Stale worktree task".to_string(),
2162                ..Default::default()
2163            })
2164            .await
2165            .unwrap();
2166        store.update_status(id, TaskStatus::Done).await.unwrap();
2167        // Set the stale worktree path on the task.
2168        store
2169            .set_fields(
2170                id,
2171                &[("worktree", stale_wt.to_string_lossy().as_ref().into())],
2172            )
2173            .await
2174            .unwrap();
2175
2176        // Verify the task starts out NOT cleaned.
2177        let before = store.get(id).await.unwrap();
2178        assert!(!before.worktree_cleaned, "task should not be cleaned yet");
2179        assert!(
2180            !before.worktree.is_empty(),
2181            "task must have a worktree path set"
2182        );
2183
2184        let opts = JanitorOptions {
2185            ttl_hours: 0,
2186            dry_run: false,
2187        };
2188
2189        let result = cleanup_task_worktree_with_opts("1005", "owner/repo", &store, &opts).await;
2190
2191        // The function should succeed (Ok) — stale path is not an error.
2192        // It returns Ok(false) because there was nothing to actively remove.
2193        match result {
2194            Ok(_) => {}
2195            Err(e) => panic!("cleanup should not error for stale path: {e}"),
2196        }
2197
2198        // The task must now be marked cleaned so the next cycle skips it.
2199        let after = store.get(id).await.unwrap();
2200        assert!(
2201            after.worktree_cleaned,
2202            "task must be marked cleaned when stored worktree path is already gone"
2203        );
2204    }
2205
2206    /// `remove_worktree_and_branch` returns true when the worktree directory
2207    /// is successfully removed (regression test for #1143).
2208    #[tokio::test]
2209    async fn remove_worktree_returns_true_on_success() {
2210        let Some((tmp, wt_dir)) = setup_test_repo() else {
2211            eprintln!("skipping test: git not available");
2212            return;
2213        };
2214
2215        let repo_dir = tmp.path().join("repo.git");
2216        assert!(wt_dir.exists(), "worktree must exist before removal");
2217
2218        let removed =
2219            remove_worktree_and_branch("42", &wt_dir, Some("gh-issue-42-test"), &repo_dir, false)
2220                .await;
2221
2222        assert!(
2223            removed,
2224            "remove_worktree_and_branch should return true when directory is gone"
2225        );
2226        assert!(!wt_dir.exists(), "worktree directory should be removed");
2227    }
2228
2229    /// Successful worktree cleanup must still delete the local branch.
2230    /// This guards against early-return regressions in `remove_worktree_and_branch`.
2231    #[tokio::test]
2232    async fn remove_worktree_success_still_deletes_local_branch() {
2233        let Some((tmp, wt_dir)) = setup_test_repo() else {
2234            eprintln!("skipping test: git not available");
2235            return;
2236        };
2237
2238        let repo_dir = tmp.path().join("repo.git");
2239        let repo_dir_str = repo_dir
2240            .to_str()
2241            .expect("test repo path contains non-UTF-8 characters");
2242        let branch = "gh-issue-42-test";
2243
2244        let before = process::Command::new("git")
2245            .args([
2246                "-C",
2247                repo_dir_str,
2248                "show-ref",
2249                "--verify",
2250                "--quiet",
2251                &format!("refs/heads/{branch}"),
2252            ])
2253            .status()
2254            .unwrap();
2255        assert!(
2256            before.success(),
2257            "test setup must create local branch before cleanup"
2258        );
2259
2260        let removed =
2261            remove_worktree_and_branch("42", &wt_dir, Some(branch), &repo_dir, false).await;
2262        assert!(removed, "worktree should be removed");
2263
2264        let after = process::Command::new("git")
2265            .args([
2266                "-C",
2267                repo_dir_str,
2268                "show-ref",
2269                "--verify",
2270                "--quiet",
2271                &format!("refs/heads/{branch}"),
2272            ])
2273            .status()
2274            .unwrap();
2275        assert!(
2276            !after.success(),
2277            "local branch should be deleted after successful worktree cleanup"
2278        );
2279    }
2280
2281    /// `remove_worktree_and_branch` returns false when the worktree directory
2282    /// cannot be removed (regression test for #1143 — prevents orphaned worktrees
2283    /// from being permanently marked as cleaned).
2284    #[tokio::test]
2285    async fn remove_worktree_returns_false_when_dir_persists() {
2286        let tmp = tempfile::tempdir().unwrap();
2287        // Create a plain directory (not a real git worktree) — git will fail
2288        // to remove it, so the directory stays on disk.
2289        let fake_wt = tmp.path().join("not-a-worktree");
2290        std_fs::create_dir_all(&fake_wt).unwrap();
2291
2292        // repo_root doesn't matter much — it just needs to exist for the -C flag.
2293        // Using tmp itself; git commands will fail but that's the point.
2294        let removed = remove_worktree_and_branch("99", &fake_wt, None, tmp.path(), false).await;
2295
2296        // The directory still exists because git couldn't remove it.
2297        // Before the fix, this would still have been marked as cleaned.
2298        assert!(
2299            !removed,
2300            "remove_worktree_and_branch should return false when directory still exists"
2301        );
2302        assert!(fake_wt.exists(), "directory should still be on disk");
2303    }
2304
2305    struct MergeCheckMockBackend {
2306        batch_result: Mutex<anyhow::Result<HashMap<String, bool>>>,
2307        single_results: HashMap<String, anyhow::Result<bool>>,
2308        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2309        single_checked_branches: Arc<Mutex<Vec<String>>>,
2310    }
2311
2312    impl MergeCheckMockBackend {
2313        fn with_batch_error(
2314            error: anyhow::Error,
2315            single_results: HashMap<String, anyhow::Result<bool>>,
2316        ) -> Self {
2317            Self {
2318                batch_result: Mutex::new(Err(error)),
2319                single_results,
2320                posted_comments: Arc::new(Mutex::new(Vec::new())),
2321                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2322            }
2323        }
2324
2325        fn with_merged(branch: &str) -> Self {
2326            Self {
2327                batch_result: Mutex::new(Ok(HashMap::from([(branch.to_string(), true)]))),
2328                single_results: HashMap::new(),
2329                posted_comments: Arc::new(Mutex::new(Vec::new())),
2330                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2331            }
2332        }
2333    }
2334
2335    #[async_trait]
2336    impl ExternalBackend for MergeCheckMockBackend {
2337        fn name(&self) -> &'static str {
2338            "mock"
2339        }
2340
2341        async fn create_task(
2342            &self,
2343            _title: &str,
2344            _body: &str,
2345            _labels: &[String],
2346        ) -> anyhow::Result<ExternalId> {
2347            anyhow::bail!("not implemented")
2348        }
2349
2350        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2351            anyhow::bail!("not implemented")
2352        }
2353
2354        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2355            Ok(vec![])
2356        }
2357
2358        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2359            Ok(vec![])
2360        }
2361
2362        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2363            self.posted_comments
2364                .lock()
2365                .unwrap()
2366                .push((id.0.clone(), body.to_string()));
2367            Ok(())
2368        }
2369
2370        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2371            Ok(())
2372        }
2373
2374        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2375            Ok(())
2376        }
2377
2378        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2379            Ok(vec![])
2380        }
2381
2382        async fn create_sub_task(
2383            &self,
2384            _parent: &ExternalId,
2385            _title: &str,
2386            _body: &str,
2387            _labels: &[String],
2388        ) -> anyhow::Result<ExternalId> {
2389            anyhow::bail!("not implemented")
2390        }
2391
2392        async fn health_check(&self) -> anyhow::Result<()> {
2393            Ok(())
2394        }
2395
2396        async fn is_pr_merged(&self, branch: &str) -> anyhow::Result<bool> {
2397            self.single_checked_branches
2398                .lock()
2399                .unwrap()
2400                .push(branch.to_string());
2401            match self.single_results.get(branch) {
2402                Some(Ok(is_merged)) => Ok(*is_merged),
2403                Some(Err(err)) => Err(anyhow::anyhow!(err.to_string())),
2404                None => Ok(false),
2405            }
2406        }
2407
2408        async fn batch_is_pr_merged(
2409            &self,
2410            _branches: &[String],
2411        ) -> anyhow::Result<HashMap<String, bool>> {
2412            match &*self.batch_result.lock().unwrap() {
2413                Ok(map) => Ok(map.clone()),
2414                Err(err) => Err(anyhow::anyhow!(err.to_string())),
2415            }
2416        }
2417    }
2418
2419    #[tokio::test]
2420    async fn check_merged_prs_falls_back_to_single_branch_checks_on_batch_error() {
2421        let single_results = HashMap::from([
2422            ("branch-1".to_string(), Ok(true)),
2423            ("branch-2".to_string(), Ok(false)),
2424        ]);
2425        let backend = Arc::new(MergeCheckMockBackend::with_batch_error(
2426            anyhow::anyhow!("missing /data/repository in GraphQL response: no error details"),
2427            single_results,
2428        ));
2429        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2430
2431        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2432        let task1 = store
2433            .create(&NewTask {
2434                external_id: Some("101".to_string()),
2435                repo: "owner/repo".to_string(),
2436                origin: "github".to_string(),
2437                title: "Task 101".to_string(),
2438                ..Default::default()
2439            })
2440            .await
2441            .unwrap();
2442        store
2443            .update_status(task1, TaskStatus::NeedsReview)
2444            .await
2445            .unwrap();
2446        store
2447            .set_fields(task1, &[("branch", serde_json::json!("branch-1"))])
2448            .await
2449            .unwrap();
2450        // No .git file, so cleanup force-removes the directory
2451        let wt_dir = tempfile::tempdir().unwrap();
2452        let wt_path = wt_dir.path().join("branch-1");
2453        std_fs::create_dir_all(&wt_path).unwrap();
2454        store
2455            .set_fields(
2456                task1,
2457                &[("worktree", serde_json::json!(wt_path.to_string_lossy()))],
2458            )
2459            .await
2460            .unwrap();
2461
2462        let task2 = store
2463            .create(&NewTask {
2464                external_id: Some("102".to_string()),
2465                repo: "owner/repo".to_string(),
2466                origin: "github".to_string(),
2467                title: "Task 102".to_string(),
2468                ..Default::default()
2469            })
2470            .await
2471            .unwrap();
2472        store
2473            .update_status(task2, TaskStatus::InReview)
2474            .await
2475            .unwrap();
2476        store
2477            .set_fields(task2, &[("branch", serde_json::json!("branch-2"))])
2478            .await
2479            .unwrap();
2480
2481        let task_manager = Arc::new(TaskManager::with_store(
2482            Arc::clone(&backend_dyn),
2483            Arc::clone(&store),
2484            "owner/repo".to_string(),
2485        ));
2486
2487        let in_review_tasks = store
2488            .list_by_status("owner/repo", TaskStatus::InReview)
2489            .await
2490            .unwrap()
2491            .into_iter()
2492            .map(|stored| ReviewTaskSnapshot {
2493                external: tasks::store_task_to_external(&stored),
2494                stored,
2495            })
2496            .collect::<Vec<_>>();
2497        let needs_review_tasks = store
2498            .list_by_status("owner/repo", TaskStatus::NeedsReview)
2499            .await
2500            .unwrap()
2501            .into_iter()
2502            .map(|stored| ReviewTaskSnapshot {
2503                external: tasks::store_task_to_external(&stored),
2504                stored,
2505            })
2506            .collect::<Vec<_>>();
2507
2508        check_merged_prs(
2509            &backend_dyn,
2510            "owner/repo",
2511            &store,
2512            &task_manager,
2513            &in_review_tasks,
2514            &needs_review_tasks,
2515        )
2516        .await
2517        .unwrap();
2518
2519        let checked = backend.single_checked_branches.lock().unwrap().clone();
2520        assert_eq!(
2521            checked,
2522            vec!["branch-2".to_string(), "branch-1".to_string()]
2523        );
2524
2525        let updated_task1 = store.get(task1).await.unwrap();
2526        let updated_task2 = store.get(task2).await.unwrap();
2527        assert_eq!(updated_task1.status, TaskStatus::Done);
2528        assert!(updated_task1.worktree_cleaned);
2529        assert!(!wt_path.exists());
2530        assert!(!updated_task2.worktree_cleaned);
2531        assert_eq!(updated_task2.status, TaskStatus::InReview);
2532
2533        let comments = backend.posted_comments.lock().unwrap().clone();
2534        assert_eq!(comments.len(), 1);
2535        assert_eq!(comments[0].0, "101");
2536        assert!(comments[0].1.contains("PR merged, marking task complete"));
2537    }
2538
2539    #[tokio::test]
2540    async fn check_merged_prs_skips_rescue_while_ci_bootstrap_holds() {
2541        let (tmp, clone) = origin_and_clone_with_feat();
2542        let wt = add_worktree(&clone, tmp.path(), "feat");
2543        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2544        // hold_after_merge reads the plan marker in the body, not the title
2545        let id = store_approved_task(
2546            &store,
2547            "owner/repo",
2548            "task-12",
2549            "CI-01 Set up CI and tests",
2550            "<!-- orch-plan: CI-01 -->",
2551            &wt,
2552        )
2553        .await;
2554        let stored = store.get(id).await.unwrap();
2555
2556        let backend = Arc::new(MergeCheckMockBackend::with_merged("feat"));
2557        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2558        let task_manager = Arc::new(TaskManager::with_store(
2559            Arc::clone(&backend_dyn),
2560            Arc::clone(&store),
2561            "owner/repo".to_string(),
2562        ));
2563
2564        check_merged_prs(
2565            &backend_dyn,
2566            "owner/repo",
2567            &store,
2568            &task_manager,
2569            &[ReviewTaskSnapshot {
2570                external: tasks::store_task_to_external(&stored),
2571                stored,
2572            }],
2573            &[],
2574        )
2575        .await
2576        .unwrap();
2577
2578        let after = store.get(id).await.unwrap();
2579        assert_eq!(after.status, TaskStatus::Approved);
2580        assert!(after.worktree_cleaned);
2581        assert!(!wt.exists());
2582        assert!(!remote_has(&clone, "rescue/feat"));
2583        assert!(backend.posted_comments.lock().unwrap().is_empty());
2584    }
2585
2586    struct ReconciliationMockBackend {
2587        reconciliation_calls: Arc<AtomicUsize>,
2588    }
2589
2590    #[async_trait]
2591    impl ExternalBackend for ReconciliationMockBackend {
2592        fn name(&self) -> &'static str {
2593            "mock"
2594        }
2595
2596        async fn create_task(
2597            &self,
2598            _title: &str,
2599            _body: &str,
2600            _labels: &[String],
2601        ) -> anyhow::Result<ExternalId> {
2602            anyhow::bail!("not implemented")
2603        }
2604
2605        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2606            anyhow::bail!("not implemented")
2607        }
2608
2609        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2610            Ok(vec![])
2611        }
2612
2613        async fn list_reconciliation_candidates(&self) -> anyhow::Result<Vec<ExternalTask>> {
2614            self.reconciliation_calls.fetch_add(1, Ordering::SeqCst);
2615            Ok(vec![])
2616        }
2617
2618        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2619            Ok(vec![])
2620        }
2621
2622        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2623            Ok(())
2624        }
2625
2626        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2627            Ok(())
2628        }
2629
2630        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2631            Ok(())
2632        }
2633
2634        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2635            Ok(vec![])
2636        }
2637
2638        async fn create_sub_task(
2639            &self,
2640            _parent: &ExternalId,
2641            _title: &str,
2642            _body: &str,
2643            _labels: &[String],
2644        ) -> anyhow::Result<ExternalId> {
2645            anyhow::bail!("not implemented")
2646        }
2647
2648        async fn health_check(&self) -> anyhow::Result<()> {
2649            Ok(())
2650        }
2651    }
2652
2653    #[tokio::test]
2654    async fn cleanup_reconciliation_is_rate_limited_per_repo() {
2655        let repo = "owner/repo-cleanup-rate-limit";
2656        RECONCILIATION_LAST_RUN.remove(repo);
2657
2658        let backend = Arc::new(ReconciliationMockBackend {
2659            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2660        });
2661        let backend_dyn: Arc<dyn ExternalBackend> =
2662            Arc::<ReconciliationMockBackend>::clone(&backend);
2663        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2664        let task_manager = Arc::new(TaskManager::with_store(
2665            Arc::clone(&backend_dyn),
2666            Arc::clone(&store),
2667            repo.to_string(),
2668        ));
2669        let opts = JanitorOptions {
2670            ttl_hours: 0,
2671            dry_run: true,
2672        };
2673
2674        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2675            .await
2676            .unwrap();
2677        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2678            .await
2679            .unwrap();
2680
2681        assert_eq!(
2682            backend.reconciliation_calls.load(Ordering::SeqCst),
2683            1,
2684            "cleanup should not re-list reconciliation candidates on every sync pass"
2685        );
2686
2687        RECONCILIATION_LAST_RUN.remove(repo);
2688    }
2689
2690    /// Cleanup only collects done tasks. A blocked task waits on a person and resumes from its
2691    /// worktree and branch, so it must not even be considered (#3781)
2692    #[tokio::test]
2693    async fn cleanup_skips_blocked_internal_tasks_and_cleans_done_ones() {
2694        let repo = "owner/repo-cleanup-blocked";
2695        RECONCILIATION_LAST_RUN.remove(repo);
2696        let backend_dyn: Arc<dyn ExternalBackend> = Arc::new(ReconciliationMockBackend {
2697            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2698        });
2699        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2700        let task_manager = Arc::new(TaskManager::with_store(
2701            Arc::clone(&backend_dyn),
2702            Arc::clone(&store),
2703            repo.to_string(),
2704        ));
2705
2706        let mut ids = Vec::new();
2707        for (title, status) in [("blocked", TaskStatus::Blocked), ("done", TaskStatus::Done)] {
2708            let id = store
2709                .create_internal(repo, title, "", "test", title, None)
2710                .await
2711                .unwrap();
2712            store
2713                .set_fields(
2714                    id,
2715                    &[
2716                        (
2717                            "worktree",
2718                            serde_json::json!(format!("/nonexistent/{title}")),
2719                        ),
2720                        ("branch", serde_json::json!("")),
2721                    ],
2722                )
2723                .await
2724                .unwrap();
2725            store.update_status(id, status).await.unwrap();
2726            ids.push(id);
2727        }
2728
2729        let opts = JanitorOptions {
2730            ttl_hours: 0,
2731            dry_run: false,
2732        };
2733        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2734            .await
2735            .unwrap();
2736
2737        assert!(
2738            !store.get(ids[0]).await.unwrap().worktree_cleaned,
2739            "blocked task must not be considered for cleanup"
2740        );
2741        assert!(
2742            store.get(ids[1]).await.unwrap().worktree_cleaned,
2743            "done task is cleaned, which proves the pass ran"
2744        );
2745        RECONCILIATION_LAST_RUN.remove(repo);
2746    }
2747
2748    fn git_ok(dir: &Path, args: &[&str]) -> String {
2749        let out = process::Command::new("git")
2750            .args([
2751                "-c",
2752                "commit.gpgsign=false",
2753                "-c",
2754                "user.name=T",
2755                "-c",
2756                "user.email=t@t",
2757            ])
2758            .args(args)
2759            .current_dir(dir)
2760            .output()
2761            .unwrap();
2762        assert!(
2763            out.status.success(),
2764            "git {args:?}: {}",
2765            String::from_utf8_lossy(&out.stderr)
2766        );
2767        String::from_utf8_lossy(&out.stdout).to_string()
2768    }
2769
2770    /// A clone of a bare origin with main pushed and branch `feat` one commit ahead
2771    fn origin_and_clone_with_feat() -> (tempfile::TempDir, PathBuf) {
2772        let tmp = tempfile::tempdir().unwrap();
2773        let bare = tmp.path().join("origin.git");
2774        std_fs::create_dir_all(&bare).unwrap();
2775        git_ok(&bare, &["init", "--bare", "-b", "main"]);
2776        git_ok(tmp.path(), &["clone", bare.to_str().unwrap(), "clone"]);
2777        let clone = tmp.path().join("clone");
2778        std_fs::write(clone.join("base.txt"), "base").unwrap();
2779        git_ok(&clone, &["add", "."]);
2780        git_ok(&clone, &["commit", "-m", "base"]);
2781        git_ok(&clone, &["push", "origin", "main"]);
2782        git_ok(&clone, &["checkout", "-b", "feat"]);
2783        std_fs::write(clone.join("work.txt"), "work").unwrap();
2784        git_ok(&clone, &["add", "."]);
2785        git_ok(&clone, &["commit", "-m", "work"]);
2786        git_ok(&clone, &["checkout", "main"]);
2787        (tmp, clone)
2788    }
2789
2790    fn remote_has(clone: &Path, branch: &str) -> bool {
2791        !git_ok(clone, &["ls-remote", "--heads", "origin", branch])
2792            .trim()
2793            .is_empty()
2794    }
2795
2796    fn local_has(clone: &Path, branch: &str) -> bool {
2797        process::Command::new("git")
2798            .args([
2799                "show-ref",
2800                "--verify",
2801                "--quiet",
2802                &format!("refs/heads/{branch}"),
2803            ])
2804            .current_dir(clone)
2805            .status()
2806            .unwrap()
2807            .success()
2808    }
2809
2810    /// Deleting a pushed branch whose commits main lacks would destroy the only copy of the
2811    /// work, as startup cleanup did to a blocked task (#3781)
2812    #[tokio::test]
2813    async fn delete_branches_keeps_an_unmerged_remote_branch() {
2814        let (_tmp, clone) = origin_and_clone_with_feat();
2815        git_ok(&clone, &["push", "origin", "feat"]);
2816
2817        delete_branches("1", "feat", &clone).await;
2818
2819        assert!(
2820            remote_has(&clone, "feat"),
2821            "unmerged remote branch must survive"
2822        );
2823        assert!(
2824            !local_has(&clone, "feat"),
2825            "local copy is on origin, so it can go"
2826        );
2827    }
2828
2829    #[tokio::test]
2830    async fn delete_branches_leaves_the_screenshots_branch_alone() {
2831        let (_tmp, clone) = origin_and_clone_with_feat();
2832        git_ok(&clone, &["branch", SCREENSHOTS_BRANCH, "feat"]);
2833        git_ok(&clone, &["push", "origin", SCREENSHOTS_BRANCH]);
2834        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2835        git_ok(&clone, &["push", "origin", "main"]);
2836
2837        delete_branches("1", SCREENSHOTS_BRANCH, &clone).await;
2838
2839        assert!(remote_has(&clone, SCREENSHOTS_BRANCH));
2840        assert!(local_has(&clone, SCREENSHOTS_BRANCH));
2841    }
2842
2843    #[tokio::test]
2844    async fn delete_branches_removes_a_branch_main_already_contains() {
2845        let (_tmp, clone) = origin_and_clone_with_feat();
2846        git_ok(&clone, &["push", "origin", "feat"]);
2847        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2848        git_ok(&clone, &["push", "origin", "main"]);
2849
2850        delete_branches("1", "feat", &clone).await;
2851
2852        assert!(!remote_has(&clone, "feat"));
2853        assert!(!local_has(&clone, "feat"));
2854    }
2855
2856    fn add_worktree(clone: &Path, tmp: &Path, branch: &str) -> PathBuf {
2857        let wt = tmp.join("wt");
2858        git_ok(clone, &["worktree", "add", wt.to_str().unwrap(), branch]);
2859        wt
2860    }
2861
2862    async fn store_approved_task(
2863        store: &TaskStore,
2864        repo: &str,
2865        external_id: &str,
2866        title: &str,
2867        body: &str,
2868        worktree: &Path,
2869    ) -> i64 {
2870        let id = store
2871            .create(&NewTask {
2872                external_id: Some(external_id.to_string()),
2873                repo: repo.to_string(),
2874                origin: "github".to_string(),
2875                title: title.to_string(),
2876                body: body.to_string(),
2877                ..Default::default()
2878            })
2879            .await
2880            .unwrap();
2881        store.update_status(id, TaskStatus::Approved).await.unwrap();
2882        store
2883            .set_fields(
2884                id,
2885                &[
2886                    ("branch", serde_json::json!("feat")),
2887                    (
2888                        "worktree",
2889                        serde_json::json!(worktree.to_string_lossy().as_ref()),
2890                    ),
2891                    ("pr_number", serde_json::json!(7)),
2892                ],
2893            )
2894            .await
2895            .unwrap();
2896        id
2897    }
2898
2899    #[tokio::test]
2900    async fn cleanup_pushes_unpushed_commits_to_a_rescue_branch() {
2901        let (tmp, clone) = origin_and_clone_with_feat();
2902        let wt = add_worktree(&clone, tmp.path(), "feat");
2903
2904        let sha = git_ok(&clone, &["rev-parse", "--short", "feat"]);
2905        let sha = sha.trim();
2906        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2907            Rescue::Pushed { branch, commits } => {
2908                assert_eq!(branch, "rescue/feat");
2909                assert!(rescue_comment(&branch, &commits).contains(sha));
2910            }
2911            other => panic!("expected a rescue push, got {other:?}"),
2912        }
2913
2914        let tip = git_ok(&clone, &["rev-parse", "feat"]);
2915        let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2916        assert_eq!(
2917            tracking.trim(),
2918            tip.trim(),
2919            "rescue push must point the tracking ref at the tip it sent"
2920        );
2921
2922        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2923
2924        assert!(removed);
2925        assert!(remote_has(&clone, "rescue/feat"));
2926        assert!(!local_has(&clone, "feat"));
2927    }
2928
2929    #[tokio::test]
2930    async fn rescue_push_sets_tracking_ref_without_a_fetchspec() {
2931        let (tmp, clone) = origin_and_clone_with_feat();
2932        let wt = add_worktree(&clone, tmp.path(), "feat");
2933        git_ok(&clone, &["config", "--unset-all", "remote.origin.fetch"]);
2934
2935        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2936            Rescue::Pushed { branch, .. } => assert_eq!(branch, "rescue/feat"),
2937            other => panic!("expected a rescue push, got {other:?}"),
2938        }
2939
2940        let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2941        let tip = git_ok(&clone, &["rev-parse", "feat"]);
2942        assert_eq!(tracking.trim(), tip.trim());
2943    }
2944
2945    #[tokio::test]
2946    async fn cleanup_commits_uncommitted_changes_before_the_rescue() {
2947        let (tmp, clone) = origin_and_clone_with_feat();
2948        git_ok(&clone, &["push", "origin", "feat"]);
2949        let wt = add_worktree(&clone, tmp.path(), "feat");
2950        std_fs::write(wt.join("dirty.txt"), "dirty").unwrap();
2951
2952        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2953
2954        assert!(removed);
2955        assert!(remote_has(&clone, "rescue/feat"));
2956    }
2957
2958    #[tokio::test]
2959    async fn cleanup_makes_no_rescue_branch_for_a_pushed_branch() {
2960        let (tmp, clone) = origin_and_clone_with_feat();
2961        git_ok(&clone, &["push", "origin", "feat"]);
2962        let wt = add_worktree(&clone, tmp.path(), "feat");
2963
2964        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2965
2966        assert!(removed);
2967        assert!(!remote_has(&clone, "rescue/feat"));
2968        assert!(!local_has(&clone, "feat"));
2969    }
2970
2971    #[tokio::test]
2972    async fn rescue_skips_a_branch_on_the_remote_without_a_tracking_ref() {
2973        let (_tmp, clone) = origin_and_clone_with_feat();
2974        git_ok(&clone, &["push", "origin", "feat"]);
2975        git_ok(&clone, &["update-ref", "-d", "refs/remotes/origin/feat"]);
2976
2977        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
2978
2979        assert_eq!(rescue, Rescue::Clean);
2980        assert!(!remote_has(&clone, "rescue/feat"));
2981    }
2982
2983    #[tokio::test]
2984    async fn merged_pr_cleanup_pushes_no_rescue_branch() {
2985        let (tmp, clone) = origin_and_clone_with_feat();
2986        let wt = add_worktree(&clone, tmp.path(), "feat");
2987
2988        let removed =
2989            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, true).await;
2990
2991        assert!(removed);
2992        assert!(!wt.exists());
2993        assert!(!remote_has(&clone, "rescue/feat"));
2994    }
2995
2996    #[tokio::test]
2997    async fn cleanup_skips_rescue_for_an_approved_merged_pr() {
2998        let (tmp, clone) = origin_and_clone_with_feat();
2999        let wt = add_worktree(&clone, tmp.path(), "feat");
3000        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3001        store_approved_task(&store, "owner/repo", "task-12", "CI-01 Set up CI", "", &wt).await;
3002
3003        cleanup_merged_pr_worktree("task-12", "owner/repo", &store)
3004            .await
3005            .unwrap();
3006
3007        let task = store
3008            .get(
3009                store
3010                    .resolve_task_id("owner/repo", "task-12")
3011                    .await
3012                    .unwrap()
3013                    .unwrap(),
3014            )
3015            .await
3016            .unwrap();
3017        assert_eq!(task.status, TaskStatus::Approved);
3018        assert!(task.worktree_cleaned);
3019        assert!(!wt.exists());
3020        assert!(!remote_has(&clone, "rescue/feat"));
3021    }
3022
3023    #[tokio::test]
3024    async fn cleanup_rescues_an_approved_task_when_the_pr_is_not_known_merged() {
3025        let (tmp, clone) = origin_and_clone_with_feat();
3026        let wt = add_worktree(&clone, tmp.path(), "feat");
3027        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3028        store_approved_task(&store, "owner/repo", "task-12", "fix something", "", &wt).await;
3029
3030        cleanup_task_worktree("task-12", "owner/repo", &store)
3031            .await
3032            .unwrap();
3033
3034        assert!(remote_has(&clone, "rescue/feat"));
3035    }
3036
3037    #[tokio::test]
3038    async fn unmerged_cleanup_still_pushes_a_rescue_branch() {
3039        let (tmp, clone) = origin_and_clone_with_feat();
3040        let wt = add_worktree(&clone, tmp.path(), "feat");
3041
3042        let removed =
3043            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, false).await;
3044
3045        assert!(removed);
3046        assert!(remote_has(&clone, "rescue/feat"));
3047    }
3048
3049    #[tokio::test]
3050    async fn rescue_ignores_a_remote_tip_missing_locally() {
3051        let (tmp, clone) = origin_and_clone_with_feat();
3052        git_ok(&clone, &["push", "origin", "feat"]);
3053        let other = tmp.path().join("other");
3054        git_ok(
3055            tmp.path(),
3056            &["clone", "-b", "feat", "origin.git", other.to_str().unwrap()],
3057        );
3058        std_fs::write(other.join("extra.txt"), "extra").unwrap();
3059        git_ok(&other, &["add", "."]);
3060        git_ok(&other, &["commit", "-m", "extra"]);
3061        git_ok(&other, &["push", "origin", "feat"]);
3062
3063        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3064
3065        assert_ne!(rescue, Rescue::Failed);
3066    }
3067
3068    #[tokio::test]
3069    async fn cleanup_keeps_worktree_and_branch_when_the_rescue_push_fails() {
3070        let (tmp, clone) = origin_and_clone_with_feat();
3071        let wt = add_worktree(&clone, tmp.path(), "feat");
3072        git_ok(
3073            &clone,
3074            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3075        );
3076
3077        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3078
3079        assert!(!removed);
3080        assert!(wt.exists());
3081        assert!(local_has(&clone, "feat"));
3082    }
3083
3084    #[tokio::test]
3085    async fn delete_branches_keeps_a_local_branch_with_unpushed_commits() {
3086        let (_tmp, clone) = origin_and_clone_with_feat();
3087
3088        delete_branches("1", "feat", &clone).await;
3089
3090        assert!(
3091            local_has(&clone, "feat"),
3092            "unpushed commits must not be deleted"
3093        );
3094    }
3095}