Skip to main content

orch/engine/
cleanup.rs

1//! Worktree cleanup and merged-PR detection.
2//!
3//! Contains the post-merge cleanup pipeline: removing git worktrees,
4//! deleting local/remote branches, pulling main, and detecting
5//! already-merged PRs so their tasks can be marked done.
6
7use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
8use crate::cmd::CommandErrorContext;
9use crate::engine::ci_bootstrap;
10use crate::engine::tasks::{TaskManager, store_task_to_external};
11use crate::store;
12use crate::store::TaskStatus;
13use crate::store::TaskStore;
14use crate::store::store_log_activity;
15use dashmap::{DashMap, DashSet};
16use futures::future;
17use std::collections::HashMap;
18use std::collections::HashSet;
19use std::env;
20use std::fs as std_fs;
21use std::io::ErrorKind;
22use std::path::Path;
23use std::path::PathBuf;
24use std::process::Output;
25use std::sync::Arc;
26use std::sync::LazyLock;
27use std::time::SystemTime;
28use std::time::{Duration, Instant};
29use tokio::fs;
30use tokio::process::Command;
31use tokio::time::sleep;
32use tokio::time::timeout;
33
34use super::sync::ReviewTaskSnapshot;
35use crate::config;
36use crate::engine;
37use crate::engine::auto_merge;
38use crate::engine::runner::cow;
39use crate::engine::runner::git_ops;
40use crate::engine::runner::screenshots::SCREENSHOTS_BRANCH;
41use crate::engine::runner::worktree;
42use crate::github::http::GhHttp;
43use crate::home;
44
45/// Reconciliation calls can be slow on repos with many issues or during GitHub instability.
46/// Cleanup runs in a background task (tokio::spawn) so it does not block the main tick loop.
47/// 30 s gives enough headroom for paginated list_all_tasks calls while still bounding the
48/// overall background-task duration.
49const RECONCILIATION_LIST_TIMEOUT: Duration = Duration::from_secs(30);
50/// Closed-issue reconciliation is a best-effort cleanup fallback, not a per-sync requirement.
51/// Run it on a slower cadence so repeated GitHub slowness does not consume 30 s every sync.
52const RECONCILIATION_INTERVAL: Duration = Duration::from_mins(15);
53
54const RUN_OUTPUT_RETENTION_DAYS: i32 = 30;
55const RUN_PRUNE_INTERVAL_SECS: i64 = 24 * 60 * 60;
56const RUN_PRUNE_KV_KEY: &str = "task_runs_pruned_at";
57
58/// Per-repo in-flight guard for closed-issue reconciliation/cleanup.
59/// Prevents overlapping cleanup runs from piling up across sync cycles.
60static CLEANUP_IN_FLIGHT: LazyLock<DashSet<String>> = LazyLock::new(DashSet::new);
61/// Per-repo reconciliation cadence tracker.
62static RECONCILIATION_LAST_RUN: LazyLock<DashMap<String, Instant>> = LazyLock::new(DashMap::new);
63
64struct CleanupInFlightGuard {
65    repo: String,
66}
67
68impl CleanupInFlightGuard {
69    fn try_acquire(repo: &str) -> Option<Self> {
70        if CLEANUP_IN_FLIGHT.insert(repo.to_string()) {
71            Some(Self {
72                repo: repo.to_string(),
73            })
74        } else {
75            None
76        }
77    }
78}
79
80impl Drop for CleanupInFlightGuard {
81    fn drop(&mut self) {
82        CLEANUP_IN_FLIGHT.remove(&self.repo);
83    }
84}
85
86fn should_run_reconciliation(repo: &str) -> bool {
87    let now = Instant::now();
88    if let Some(mut last_run) = RECONCILIATION_LAST_RUN.get_mut(repo) {
89        if last_run.elapsed() < RECONCILIATION_INTERVAL {
90            return false;
91        }
92        *last_run = now;
93        true
94    } else {
95        RECONCILIATION_LAST_RUN.insert(repo.to_string(), now);
96        true
97    }
98}
99
100/// Options controlling the worktree janitor.
101#[derive(Debug, Clone)]
102pub struct JanitorOptions {
103    /// Minimum age (in hours) of the worktree directory before it is eligible for removal.
104    ///
105    /// A worktree whose filesystem mtime is more recent than this threshold is skipped
106    /// even if the task is already in a terminal state. This provides a safety buffer
107    /// against races (e.g. review agent still writing when the janitor runs).
108    ///
109    /// Default: 24.  Configurable via `workflow.worktree_janitor_ttl_hours` in config.
110    pub ttl_hours: u64,
111
112    /// When `true`, log what would be done but skip all destructive operations.
113    ///
114    /// Configurable via `workflow.worktree_janitor_dry_run: true` in config.
115    pub dry_run: bool,
116}
117
118impl Default for JanitorOptions {
119    fn default() -> Self {
120        Self {
121            ttl_hours: 24,
122            dry_run: false,
123        }
124    }
125}
126
127impl JanitorOptions {
128    /// Read options from the global config, falling back to defaults.
129    pub fn from_config() -> Self {
130        let ttl_hours = config::get("workflow.worktree_janitor_ttl_hours")
131            .ok()
132            .and_then(|v| v.parse::<u64>().ok())
133            .unwrap_or(24);
134
135        let dry_run = config::get("workflow.worktree_janitor_dry_run").is_ok_and(|v| v == "true");
136
137        Self { ttl_hours, dry_run }
138    }
139}
140
141/// Cleanup worktrees for completed tasks.
142///
143/// Targets tasks that are finished: status:done OR closed on GitHub
144/// (e.g. when a PR merge auto-closes the issue via "Fixes #N").
145///
146/// Note: This is a fallback. Primary cleanup happens inline in
147/// `auto_merge_pr` via `cleanup_task_worktree`. This catches edge cases
148/// where the inline cleanup missed (e.g., manual merges, auto-closed issues).
149pub(crate) async fn cleanup_done_worktrees(
150    backend: &Arc<dyn ExternalBackend>,
151    repo: &str,
152    task_manager: &Arc<TaskManager>,
153    store: &Arc<TaskStore>,
154) -> anyhow::Result<()> {
155    let opts = JanitorOptions::from_config();
156    cleanup_done_worktrees_with_opts(backend, repo, task_manager, store, &opts).await
157}
158
159/// Clear the stored output of runs for tasks done longer than `RUN_OUTPUT_RETENTION_DAYS`, once a day
160pub(crate) async fn prune_old_run_output(store: &TaskStore) -> anyhow::Result<()> {
161    let now = chrono::Utc::now().timestamp();
162    let last = store
163        .kv_get(RUN_PRUNE_KV_KEY)
164        .await?
165        .and_then(|v| v.parse::<i64>().ok())
166        .unwrap_or(0);
167    if now - last < RUN_PRUNE_INTERVAL_SECS {
168        return Ok(());
169    }
170    store.kv_set(RUN_PRUNE_KV_KEY, &now.to_string()).await?;
171
172    let pruned = store.prune_old_runs(RUN_OUTPUT_RETENTION_DAYS).await?;
173    if pruned > 0 {
174        tracing::info!(pruned, "cleared the output of old task runs");
175    }
176    Ok(())
177}
178
179async fn reconcile_closed_tasks(
180    task_ids: &mut Vec<String>,
181    all_tasks: &[ExternalTask],
182    task_manager: &TaskManager,
183    store: &Arc<TaskStore>,
184    repo: &str,
185) {
186    const MAX_RECONCILE_PER_CYCLE: usize = 10;
187
188    let done_set: HashSet<String> = task_ids.iter().cloned().collect();
189    let mut closed_not_in_store = Vec::new();
190    for task in all_tasks {
191        if task.state != "closed" || done_set.contains(&task.id.0) {
192            continue;
193        }
194        if ci_bootstrap::hold_after_merge(task, store, repo).await {
195            continue;
196        }
197        closed_not_in_store.push(task);
198    }
199
200    // Split: issues that already have status:done label just need worktree
201    // cleanup (no API call needed), vs those needing label reconciliation.
202    let mut already_labeled: Vec<String> = Vec::new();
203    let mut needs_reconcile: Vec<&ExternalTask> = Vec::new();
204    for task in &closed_not_in_store {
205        if task.labels.iter().any(|l| l == "status:done") {
206            already_labeled.push(task.id.0.clone());
207        } else {
208            needs_reconcile.push(task);
209        }
210    }
211
212    // Add already-labeled issues directly (no API call).
213    if !already_labeled.is_empty() {
214        tracing::debug!(
215            count = already_labeled.len(),
216            "closed issues already labeled status:done — adding to cleanup set"
217        );
218        task_ids.extend(already_labeled);
219    }
220
221    // Reconcile issues missing the done label — cap to avoid rate-limit
222    // exhaustion. Remaining issues will be picked up in subsequent cycles.
223    if needs_reconcile.len() > MAX_RECONCILE_PER_CYCLE {
224        tracing::info!(
225            total = needs_reconcile.len(),
226            batch = MAX_RECONCILE_PER_CYCLE,
227            "capping closed-issue reconciliation to avoid rate-limit exhaustion"
228        );
229    }
230    let mut futs = Vec::new();
231    for task in needs_reconcile.iter().take(MAX_RECONCILE_PER_CYCLE) {
232        let id = task.id.clone();
233        let task_id_str = task.id.0.clone();
234        let labels = task.labels.clone();
235        futs.push(async move {
236            tracing::info!(
237                task_id = task_id_str,
238                labels = ?labels,
239                "closed issue with stale status label — reconciling to done and cleaning up"
240            );
241            match task_manager.update_task_status(&id, Status::Done).await {
242                Ok(()) => Some(task_id_str),
243                Err(e) => {
244                    tracing::warn!(
245                        task_id = task_id_str,
246                        err = %e,
247                        "failed to reconcile closed issue status to done"
248                    );
249                    None
250                }
251            }
252        });
253    }
254    let results = future::join_all(futs).await;
255    task_ids.extend(results.into_iter().flatten());
256}
257
258/// Cleanup worktrees for completed tasks with explicit options.
259///
260/// Separated from `cleanup_done_worktrees` so that integration tests can
261/// inject specific options (e.g. `ttl_hours: 0`, `dry_run: true`) without
262/// touching the global config.
263pub(crate) async fn cleanup_done_worktrees_with_opts(
264    backend: &Arc<dyn ExternalBackend>,
265    repo: &str,
266    task_manager: &Arc<TaskManager>,
267    store: &Arc<TaskStore>,
268    opts: &JanitorOptions,
269) -> anyhow::Result<()> {
270    let Some(_guard) = CleanupInFlightGuard::try_acquire(repo) else {
271        tracing::debug!(
272            repo = %repo,
273            "cleanup already in progress for repo — skipping overlapping run"
274        );
275        return Ok(());
276    };
277
278    // Read done tasks from the store first; fall back to backend before first sync.
279    let done_tasks = task_manager.list_all_by_status(Status::Done).await?;
280    tracing::debug!(count = done_tasks.len(), "checking done tasks for cleanup");
281
282    // Collect task IDs from external done tasks.
283    let mut task_ids: Vec<String> = done_tasks.iter().map(|t| t.id.0.clone()).collect();
284
285    // Also include closed-but-not-done external tasks.
286    // When a PR merge auto-closes a GitHub issue (via "Fixes #N"), the issue
287    // state becomes "closed" but orch never updates the status label to "done".
288    // These orphaned worktrees accumulate unless we catch them here.
289    // Note: This still queries the backend because GitHub `state` (open/closed)
290    // is backend-specific and not tracked in the store.
291    //
292    // Use list_reconciliation_candidates() (closed issues only, bounded recent window) directly.
293    // list_all_tasks() fetches every issue ever created (state=all, all pages) and
294    // consistently times out on repos with many issues — it is not appropriate for
295    // a cleanup fallback path.
296    if should_run_reconciliation(repo) {
297        match timeout(
298            RECONCILIATION_LIST_TIMEOUT,
299            backend.list_reconciliation_candidates(),
300        )
301        .await
302        {
303            Ok(Ok(candidates)) if !candidates.is_empty() => {
304                tracing::debug!(
305                    count = candidates.len(),
306                    "reconciling closed issues from candidates"
307                );
308                reconcile_closed_tasks(&mut task_ids, &candidates, task_manager, store, repo).await;
309            }
310            Ok(Ok(_)) => {
311                tracing::debug!("no reconciliation candidates found");
312            }
313            Ok(Err(e)) => {
314                tracing::warn!(err = %e, "failed to list reconciliation candidates");
315            }
316            Err(_) => {
317                tracing::warn!(
318                    timeout_secs = RECONCILIATION_LIST_TIMEOUT.as_secs(),
319                    retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
320                    "timed out listing reconciliation candidates"
321                );
322            }
323        }
324    } else {
325        tracing::debug!(
326            retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
327            "skipping closed-issue reconciliation until interval elapses"
328        );
329    }
330
331    // Also include internal done tasks.
332    if let Ok(internal_done) = task_manager.list_internal_by_status(TaskStatus::Done).await {
333        task_ids.extend(internal_done.iter().map(|t| t.id.0.clone()));
334    }
335
336    tracing::debug!(
337        count = task_ids.len(),
338        "checking all terminal tasks for cleanup"
339    );
340
341    if let Err(e) = auto_merge::sweep_ci_timer_keys(store, repo, &task_ids).await {
342        tracing::warn!(err = %e, "failed to sweep CI timer keys");
343    }
344
345    // Prefetch all cleaned external IDs in one query instead of N+1 per-task lookups.
346    // NOTE: This is now primarily a hint to skip processing, NOT a correctness
347    // guarantee — tasks with external_id NOT in the store at all (or mismatched
348    // repo) will still be cleaned by iterating all worktree directories.
349    let already_cleaned = store.cleaned_external_ids(repo).await.unwrap_or_default();
350
351    let mut cleaned_any = false;
352    for task_id in &task_ids {
353        // Skip if already cleaned (checked via prefetched set).
354        if already_cleaned.contains(task_id) {
355            continue;
356        }
357
358        match cleanup_task_worktree_with_opts(task_id, repo, store, opts).await {
359            Ok(true) => cleaned_any = true,
360            Ok(false) => {
361                // Skipped (TTL guard, tmux guard, or nothing to remove) — do not
362                // pull main; we did not change the repo state.
363            }
364            Err(e) => {
365                tracing::warn!(task_id, err = %e, "worktree cleanup failed for task");
366            }
367        }
368    }
369
370    // ── Orphaned worktree scanner ─────────────────────────────────────────────
371    //
372    // Even if every task in our list was already cleaned (or had no worktree),
373    // there may be worktree directories on disk that are not owned by any task.
374    // This happens when a task was ingested into the store but its worktree_cleaned
375    // flag was never set — e.g. because the task was never in `list_all_by_status`
376    // (unknown repo / missing from store at cleanup time), or the cleanup skipped
377    // due to TTL guard and never retried.
378    //
379    // We scan all worktree directories in this repo's worktrees subdirectory and
380    // remove any that are not referenced by an existing task. This catches the
381    // "done but worktree not cleaned" case that `orch service doctor` reports.
382    let worktrees_base = match home::worktrees_dir() {
383        Ok(base) => Some(base),
384        Err(e) => {
385            tracing::debug!(err = %e, "worktrees_dir() unavailable — skipping orphan scan");
386            None
387        }
388    };
389
390    if let Some(base) = worktrees_base {
391        let project = repo
392            .rsplit('/')
393            .next()
394            .unwrap_or(repo)
395            .trim_end_matches(".git");
396        let repo_worktrees = base.join(project);
397
398        // Build set of all known worktree paths for this repo (all statuses).
399        let mut combined: HashSet<String> = HashSet::new();
400        if let Ok(tasks) = store.list_all(repo).await {
401            for t in tasks {
402                if !t.worktree.is_empty() {
403                    combined.insert(t.worktree);
404                }
405            }
406        }
407
408        // Also include active worktrees (in case task is in-memory but has a dir).
409        if let Ok(project_dir) = resolve_project_dir_for_repo(repo)
410            && let Ok(active_wts) = worktree::list_project_worktrees(&project_dir).await
411        {
412            for wt in active_wts {
413                combined.insert(wt.to_string_lossy().to_string());
414            }
415        }
416
417        // Scan all branch directories and remove orphaned ones.
418        if let Ok(mut entries) = fs::read_dir(&repo_worktrees).await {
419            while let Ok(Some(entry)) = entries.next_entry().await {
420                let Ok(entry_meta) = entry.metadata().await else {
421                    continue;
422                };
423                if !entry_meta.is_dir() {
424                    continue;
425                }
426                let wt_path = entry.path().to_string_lossy().to_string();
427                if combined.contains(&wt_path) {
428                    continue;
429                }
430                tracing::info!(worktree = %wt_path, "found orphaned worktree — attempting cleanup");
431                match resolve_repo_root_for_orphaned_worktree(&entry.path()).await {
432                    Ok(repo_root) => {
433                        let removed = remove_worktree_and_branch(
434                            "orphan-scan",
435                            &entry.path(),
436                            None,
437                            Path::new(&repo_root),
438                            false,
439                        )
440                        .await;
441                        if removed {
442                            tracing::info!(worktree = %wt_path, "orphaned worktree removed");
443                        } else {
444                            tracing::warn!(worktree = %wt_path, "orphaned worktree could not be removed");
445                        }
446                    }
447                    Err(e) => {
448                        tracing::warn!(worktree = %wt_path, err = %e, "could not resolve repo root for orphaned worktree");
449                    }
450                }
451            }
452        }
453    }
454
455    // Prune stale worktree metadata and pull main after cleanup.
456    if cleaned_any
457        && !opts.dry_run
458        && let Ok(repo_root) = resolve_repo_root(repo).await
459    {
460        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
461
462        // Prune stale .git/worktrees/ entries whose directories no longer exist.
463        let prune_result = timeout(
464            GIT_TIMEOUT,
465            Command::new("git")
466                .args(["-C", &repo_root, "worktree", "prune"])
467                .output_with_context(),
468        )
469        .await;
470        match prune_result {
471            Ok(Ok(output)) if output.status.success() => {
472                tracing::debug!(%repo, "pruned stale worktree metadata");
473            }
474            Ok(Ok(output)) => {
475                let stderr = String::from_utf8_lossy(&output.stderr);
476                tracing::warn!(%repo, err = %stderr, "git worktree prune failed");
477            }
478            Ok(Err(e)) => {
479                tracing::warn!(%repo, err = %e, "git worktree prune failed");
480            }
481            Err(_) => {
482                tracing::warn!(%repo, "git worktree prune timed out after 60s");
483            }
484        }
485
486        // Fetch instead of pull: refresh `origin/*` refs without touching
487        // the operator's checked-out branch or working tree. `pull` would
488        // operate on whatever HEAD happens to be (often a feature branch,
489        // not main) and would silently no-op on divergence.
490        let fetch_result = timeout(
491            GIT_TIMEOUT,
492            git_ops::git()
493                .await
494                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
495                .output_with_context(),
496        )
497        .await;
498        match fetch_result {
499            Ok(Ok(output)) if output.status.success() => {
500                tracing::info!(%repo, "fetched origin after worktree cleanup");
501            }
502            Ok(Ok(output)) => {
503                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
504                tracing::debug!(%repo, err = %stderr, "git fetch skipped after cleanup");
505            }
506            Ok(Err(e)) => {
507                tracing::debug!(%repo, err = %e, "git fetch failed after cleanup");
508            }
509            Err(_) => {
510                tracing::debug!(%repo, "git fetch timed out after 60s");
511            }
512        }
513    }
514
515    Ok(())
516}
517
518/// Cleanup a single task's worktree and branches.
519///
520/// Removes the git worktree, deletes local + remote branches,
521/// fetches origin to refresh remote tracking refs, and marks the task as cleaned.
522///
523/// This function is used for inline post-merge cleanup (called from
524/// auto-merge flows) and must attempt immediate removal — it constructs
525/// janitor options with `ttl_hours = 0` so the janitor age guard does not
526/// postpone removing freshly-created worktrees.
527pub(crate) async fn cleanup_task_worktree(
528    task_id: &str,
529    repo: &str,
530    store: &Arc<TaskStore>,
531) -> anyhow::Result<()> {
532    cleanup_task_worktree_impl(task_id, repo, store, false).await
533}
534
535/// Cleanup after a merged pull request, including while `hold_after_merge` keeps the task approved
536pub(crate) async fn cleanup_merged_pr_worktree(
537    task_id: &str,
538    repo: &str,
539    store: &Arc<TaskStore>,
540) -> anyhow::Result<()> {
541    cleanup_task_worktree_impl(task_id, repo, store, true).await
542}
543
544async fn cleanup_task_worktree_impl(
545    task_id: &str,
546    repo: &str,
547    store: &Arc<TaskStore>,
548    known_pr_merged: bool,
549) -> anyhow::Result<()> {
550    let opts = JanitorOptions {
551        ttl_hours: 0,
552        ..Default::default()
553    };
554    let cleaned =
555        cleanup_task_worktree_with_opts_impl(task_id, repo, store, &opts, known_pr_merged).await?;
556
557    // Fetch origin once after inline cleanup (post-merge single-task path).
558    // Only fetch if cleanup actually happened (not skipped due to active session).
559    // Fetch (not pull) so the operator's checked-out branch and working tree
560    // are never touched — see the matching comment in cleanup_orphaned_worktrees.
561    if cleaned && let Ok(repo_root) = resolve_repo_root(repo).await {
562        const GIT_TIMEOUT: Duration = Duration::from_secs(60);
563        let fetch_result = timeout(
564            GIT_TIMEOUT,
565            git_ops::git()
566                .await
567                .args(["-C", &repo_root, "fetch", "--prune", "origin"])
568                .output_with_context(),
569        )
570        .await;
571        match fetch_result {
572            Ok(Ok(output)) if output.status.success() => {
573                tracing::debug!(%repo, "fetched origin after task cleanup");
574            }
575            Ok(Ok(output)) => {
576                let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
577                tracing::debug!(%repo, err = %stderr, "git fetch skipped after task cleanup");
578            }
579            Ok(Err(e)) => {
580                tracing::debug!(%repo, err = %e, "git fetch failed after task cleanup");
581            }
582            Err(_) => {
583                tracing::debug!(%repo, "git fetch timed out after 60s");
584            }
585        }
586    }
587
588    Ok(())
589}
590
591/// Cleanup a single task's worktree and branches with explicit janitor options.
592///
593/// Returns `Ok(true)` when cleanup was actually performed and the task was
594/// marked as cleaned.  Returns `Ok(false)` when cleanup was intentionally
595/// skipped (TTL guard, active tmux session guard, or nothing on disk to remove).
596/// Returns `Err` on unexpected failures (e.g. cannot resolve the repo root).
597pub(crate) async fn cleanup_task_worktree_with_opts(
598    task_id: &str,
599    repo: &str,
600    store: &Arc<TaskStore>,
601    opts: &JanitorOptions,
602) -> anyhow::Result<bool> {
603    cleanup_task_worktree_with_opts_impl(task_id, repo, store, opts, false).await
604}
605
606async fn cleanup_task_worktree_with_opts_impl(
607    task_id: &str,
608    repo: &str,
609    store: &Arc<TaskStore>,
610    opts: &JanitorOptions,
611    known_pr_merged: bool,
612) -> anyhow::Result<bool> {
613    let (worktree, branch, keep_remote_branch, merged, external_id, pr_merged) =
614        store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
615            .await
616            .map_or((None, None, false, false, None, false), |t| {
617                // Blocked tasks with a PR should keep their remote branch so the
618                // PR stays open for human review.
619                let keep = t.status == TaskStatus::Blocked && t.pr_number.is_some();
620                let merged = t.status == TaskStatus::Done || known_pr_merged;
621                (
622                    Some(t.worktree),
623                    Some(t.branch),
624                    keep,
625                    merged,
626                    t.external_id,
627                    // A squash leaves the branch commits on no ref, so skip rescue when the PR merged
628                    known_pr_merged || (t.status == TaskStatus::Done && t.pr_number.is_some()),
629                )
630            });
631
632    let worktree_path = worktree.as_ref().map(PathBuf::from);
633
634    // Get worktrees base path
635    let worktrees_base = home::worktrees_dir().unwrap_or_else(|_| PathBuf::from(".orch/worktrees"));
636
637    // Determine which directory to remove.
638    //
639    // Priority:
640    //   1. stored "worktree" path, if it exists on disk.
641    //   2. Construct from worktrees_base + project name + branch, if it exists.
642    let worktree_to_remove = if let Some(ref wt) = worktree_path {
643        if match fs::try_exists(wt).await {
644            Ok(true) => true,
645            Ok(false) => false,
646            Err(e) => {
647                tracing::warn!(task_id, worktree = %wt.display(), err = %e, "try_exists failed on stored worktree path — trying branch-based fallback");
648                false
649            }
650        } {
651            Some(wt.clone())
652        } else {
653            // The recorded worktree path doesn't exist — try reconstructing from branch.
654            if let Some(ref b) = branch {
655                let project = repo
656                    .rsplit('/')
657                    .next()
658                    .unwrap_or(repo)
659                    .trim_end_matches(".git");
660                let candidate = worktrees_base.join(project).join(b);
661                if match fs::try_exists(&candidate).await {
662                    Ok(true) => true,
663                    Ok(false) => false,
664                    Err(e) => {
665                        tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
666                        false
667                    }
668                } {
669                    Some(candidate)
670                } else {
671                    None
672                }
673            } else {
674                None
675            }
676        }
677    } else if let Some(ref b) = branch {
678        // No worktree path in store — try branch-based path.
679        let project = repo
680            .rsplit('/')
681            .next()
682            .unwrap_or(repo)
683            .trim_end_matches(".git");
684        let candidate = worktrees_base.join(project).join(b);
685        if match fs::try_exists(&candidate).await {
686            Ok(true) => true,
687            Ok(false) => false,
688            Err(e) => {
689                tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
690                false
691            }
692        } {
693            Some(candidate)
694        } else {
695            None
696        }
697    } else {
698        None
699    };
700
701    let mut did_clean = false;
702
703    // If there's nothing to remove (no worktree path and no branch),
704    // treat as a no-op and return Ok(false) without attempting to
705    // resolve the repo root. Resolving the repo root can fail for
706    // projects not registered in config and that should not make
707    // cleanup a hard error when there is nothing to do.
708    let branch_nonempty = branch.as_ref().filter(|b| !b.is_empty());
709    if worktree_to_remove.is_none() && branch_nonempty.is_none() {
710        // If the stored worktree path is non-empty but doesn't exist on disk,
711        // the worktree is already gone — mark it cleaned so we stop retrying.
712        if worktree_path
713            .as_ref()
714            .is_some_and(|p| !p.as_os_str().is_empty())
715        {
716            tracing::debug!(
717                task_id,
718                worktree = ?worktree_path,
719                "stored worktree path no longer exists on disk — marking cleaned"
720            );
721            if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
722                && let Err(e) = store.mark_cleaned(store_id).await
723            {
724                tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned in store — will retry");
725            }
726        }
727        return Ok(false);
728    }
729
730    if let Some(wt) = worktree_to_remove {
731        // TTL guard: skip if the worktree directory is too young.
732        if let Some(age_hours) = worktree_age_hours(&wt).await
733            && age_hours < opts.ttl_hours
734        {
735            tracing::debug!(
736                task_id,
737                worktree = %wt.display(),
738                age_hours,
739                ttl_hours = opts.ttl_hours,
740                "worktree too young for cleanup, skipping"
741            );
742            return Ok(false);
743        }
744
745        // Tmux guard: skip if any active pane still has its cwd inside this worktree.
746        if is_worktree_in_active_session(&wt).await {
747            tracing::warn!(
748                task_id,
749                worktree = %wt.display(),
750                "worktree is referenced by an active tmux session — skipping cleanup"
751            );
752            return Ok(false);
753        }
754
755        // Validate git metadata before attempting any git operations.
756        // If the .git file points to a deleted gitdir the worktree is
757        // unrecoverable — force-remove the directory and skip git commands.
758        let gitdir_valid = worktree::validate_worktree_gitdir(&wt).await;
759
760        if opts.dry_run {
761            tracing::info!(
762                task_id,
763                worktree = %wt.display(),
764                branch = ?branch,
765                gitdir_valid,
766                "[dry-run] would remove worktree and branch"
767            );
768        } else if !gitdir_valid {
769            tracing::warn!(
770                task_id,
771                worktree = %wt.display(),
772                "worktree has broken git metadata — force-removing directory without git operations"
773            );
774            match fs::remove_dir_all(&wt).await {
775                Ok(()) => {
776                    did_clean = true;
777                }
778                Err(e) => {
779                    tracing::warn!(
780                        task_id,
781                        worktree = %wt.display(),
782                        err = %e,
783                        "failed to force-remove broken worktree directory"
784                    );
785                }
786            }
787        } else {
788            tracing::info!(task_id, worktree = %wt.display(), "removing worktree");
789            // Derive repo root from the worktree itself (for cross-project support).
790            // If the worktree belongs to a different project than the current repo
791            // context (e.g., internal task with worktree in another project), we can
792            // still find the repo root by examining the worktree's .git file.
793            let repo_root = match resolve_repo_root_from_worktree(&wt).await {
794                Ok(root) => root,
795                Err(_) => resolve_repo_root(repo).await?,
796            };
797            if let Some(br) = branch.as_deref().filter(|b| !b.is_empty())
798                && !pr_merged
799                && !rescue_and_notify(
800                    task_id,
801                    repo,
802                    external_id.as_deref(),
803                    Some(&wt),
804                    br,
805                    Path::new(&repo_root),
806                )
807                .await
808            {
809                return Ok(false);
810            }
811            if merged {
812                cow::warm_from_merged(
813                    Path::new(&repo_root),
814                    &worktree::project_name(Path::new(&repo_root)),
815                    &wt,
816                )
817                .await;
818            }
819            let removed = remove_worktree_and_branch_unchecked(
820                task_id,
821                &wt,
822                branch.as_deref(),
823                Path::new(&repo_root),
824                keep_remote_branch,
825            )
826            .await;
827            if removed {
828                did_clean = true;
829            } else {
830                tracing::warn!(
831                    task_id,
832                    worktree = %wt.display(),
833                    "worktree directory still exists after removal attempt — not marking as cleaned"
834                );
835            }
836        }
837    } else if let Some(ref br) = branch {
838        // Worktree directory is already gone, but the branch may still exist
839        // on the remote. Delete it to avoid orphaned branches.
840        if !opts.dry_run && !keep_remote_branch {
841            tracing::debug!(task_id, branch = %br, "no worktree on disk, cleaning up branch only");
842            // Resolve repo root lazily — branch cleanup needs it.
843            let repo_root = resolve_repo_root(repo).await?;
844            if !pr_merged
845                && !rescue_and_notify(
846                    task_id,
847                    repo,
848                    external_id.as_deref(),
849                    None,
850                    br,
851                    Path::new(&repo_root),
852                )
853                .await
854            {
855                return Ok(false);
856            }
857            delete_branches(task_id, br, Path::new(&repo_root)).await;
858            did_clean = true;
859        }
860    }
861
862    if did_clean {
863        // Mark as cleaned in store so we don't retry next tick.
864        // If resolve_task_id returns None the task is not (or no longer) in
865        // the store; that is fine — we already did the on-disk work.
866        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
867            && let Err(e) = store.mark_cleaned(store_id).await
868        {
869            tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned — will retry");
870            // Don't log activity; retry next tick will do the actual cleanup
871            return Ok(did_clean);
872        }
873        store_log_activity(
874            &Some(Arc::clone(store)),
875            repo,
876            task_id,
877            "branch_delete",
878            None,
879            None,
880            None,
881            None,
882            Some(&serde_json::json!({
883                "worktree": worktree_path.map(|p| p.display().to_string()),
884                "branch": branch,
885                "keep_remote_branch": keep_remote_branch,
886            })),
887        )
888        .await;
889    }
890
891    Ok(did_clean)
892}
893
894/// Result of saving a task branch's unpushed work before cleanup deletes it
895#[derive(Debug, PartialEq, Eq)]
896pub(crate) enum Rescue {
897    /// Every commit is on a remote ref already
898    Clean,
899    /// The commits were pushed to `rescue/<branch>`
900    Pushed {
901        branch: String,
902        commits: Vec<String>,
903    },
904    /// Git failed, so the worktree and branch must stay
905    Failed,
906}
907
908async fn git_in(dir: &Path, args: &[&str]) -> Option<Output> {
909    git_ops::git()
910        .await
911        .arg("-C")
912        .arg(dir)
913        .args(args)
914        .output_with_context()
915        .await
916        .ok()
917        .map(git_ops::sanitize_git_output)
918}
919
920/**
921 * Cleanup deletes the worktree and the local branch, which can be the only refs to commits the
922 * agent made before its run ended without a push (#3999). Uncommitted changes are committed
923 * first, then everything not on a remote ref goes to `rescue/<branch>`
924 **/
925pub(crate) async fn rescue_unpushed_work(
926    task_id: &str,
927    wt: Option<&Path>,
928    br: &str,
929    repo_root: &Path,
930) -> Rescue {
931    if let Some(wt) = wt
932        && fs::try_exists(wt.join(".git")).await.unwrap_or(false)
933    {
934        let Some(status) = git_in(wt, &["status", "--porcelain"]).await else {
935            tracing::warn!(task_id, branch = %br, "cannot read worktree status, keeping worktree");
936            return Rescue::Failed;
937        };
938        if !status.status.success() {
939            tracing::warn!(task_id, branch = %br, "git status failed, keeping worktree");
940            return Rescue::Failed;
941        }
942        if !status.stdout.is_empty() {
943            let committed = git_in(wt, &["add", "-A"])
944                .await
945                .is_some_and(|o| o.status.success())
946                && git_in(
947                    wt,
948                    &[
949                        "-c",
950                        "user.name=orch",
951                        "-c",
952                        "user.email=orch@localhost",
953                        "commit",
954                        "-m",
955                        "rescue: uncommitted changes saved by orch cleanup",
956                    ],
957                )
958                .await
959                .is_some_and(|o| o.status.success());
960            if !committed {
961                tracing::warn!(task_id, branch = %br, "cannot commit uncommitted changes, keeping worktree");
962                return Rescue::Failed;
963            }
964        }
965    }
966
967    let local_ref = format!("refs/heads/{br}");
968    let exists = git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
969        .await
970        .is_some_and(|o| o.status.success());
971    if !exists {
972        return Rescue::Clean;
973    }
974    // Commits on the local default branch survive the branch delete, so they do not count
975    let default_ref = format!(
976        "refs/heads/{}",
977        worktree::detect_default_branch(repo_root).await
978    );
979    // A bare clone has no remote-tracking refs, so ask the remote for the branch tip
980    let remote_tip = git_in(repo_root, &["ls-remote", "origin", &local_ref])
981        .await
982        .filter(|o| o.status.success())
983        .and_then(|o| {
984            String::from_utf8_lossy(&o.stdout)
985                .split_whitespace()
986                .next()
987                .map(str::to_string)
988        });
989    // A tip someone else pushed may be missing locally, and rev-list fails on an unknown object
990    let remote_tip = match remote_tip {
991        Some(tip) => {
992            let known = git_in(repo_root, &["cat-file", "-e", &format!("{tip}^{{commit}}")])
993                .await
994                .is_some_and(|o| o.status.success());
995            known.then_some(tip)
996        }
997        None => None,
998    };
999    let mut args = vec!["rev-list", "--oneline", &local_ref, "--not", "--remotes"];
1000    if let Some(tip) = remote_tip.as_deref() {
1001        args.push(tip);
1002    }
1003    let default_exists = default_ref != local_ref
1004        && git_in(
1005            repo_root,
1006            &["rev-parse", "--verify", "--quiet", &default_ref],
1007        )
1008        .await
1009        .is_some_and(|o| o.status.success());
1010    if default_exists {
1011        args.push(&default_ref);
1012    }
1013    let Some(list) = git_in(repo_root, &args)
1014        .await
1015        .filter(|o| o.status.success())
1016    else {
1017        tracing::warn!(task_id, branch = %br, "cannot list unpushed commits, keeping worktree and branch");
1018        return Rescue::Failed;
1019    };
1020    let commits: Vec<String> = String::from_utf8_lossy(&list.stdout)
1021        .lines()
1022        .map(str::to_string)
1023        .collect();
1024    if commits.is_empty() {
1025        return Rescue::Clean;
1026    }
1027
1028    let rescue = format!("rescue/{br}");
1029    let refspec = format!("{local_ref}:refs/heads/{rescue}");
1030    let pushed = git_in(
1031        repo_root,
1032        &["-c", "lfs.locksverify=false", "push", "origin", &refspec],
1033    )
1034    .await;
1035    if pushed.is_some_and(|o| o.status.success()) {
1036        tracing::warn!(task_id, branch = %br, rescue = %rescue, commits = commits.len(), "pushed unpushed commits to rescue branch");
1037        Rescue::Pushed {
1038            branch: rescue,
1039            commits,
1040        }
1041    } else {
1042        tracing::warn!(task_id, branch = %br, rescue = %rescue, "rescue push failed, keeping worktree and branch");
1043        Rescue::Failed
1044    }
1045}
1046
1047fn rescue_comment(rescue_branch: &str, commits: &[String]) -> String {
1048    let list = commits
1049        .iter()
1050        .map(|c| format!("- `{c}`"))
1051        .collect::<Vec<_>>()
1052        .join("\n");
1053    format!(
1054        "Cleanup found commits on this task's branch that were never pushed. They are saved on `{rescue_branch}`:\n\n{list}"
1055    )
1056}
1057
1058/// Save unpushed work and tell the task about it. Returns false when cleanup must not go on
1059async fn rescue_and_notify(
1060    task_id: &str,
1061    repo: &str,
1062    external_id: Option<&str>,
1063    wt: Option<&Path>,
1064    br: &str,
1065    repo_root: &Path,
1066) -> bool {
1067    match rescue_unpushed_work(task_id, wt, br, repo_root).await {
1068        Rescue::Clean => true,
1069        Rescue::Failed => false,
1070        Rescue::Pushed { branch, commits } => {
1071            if let Some(number) = external_id.filter(|n| n.parse::<u64>().is_ok()) {
1072                match GhHttp::new() {
1073                    Ok(gh) => {
1074                        if let Err(e) = gh
1075                            .add_comment(repo, number, &rescue_comment(&branch, &commits))
1076                            .await
1077                        {
1078                            tracing::warn!(task_id, err = %e, "failed to comment on rescue branch");
1079                        }
1080                    }
1081                    Err(e) => {
1082                        tracing::warn!(task_id, err = %e, "cannot comment on rescue branch");
1083                    }
1084                }
1085            }
1086            true
1087        }
1088    }
1089}
1090
1091/// Remove a git worktree directory and its local + remote branches.
1092/// When `keep_remote_branch` is true, only the worktree and local branch are
1093/// removed — the remote branch stays so that any linked PR remains open
1094/// (used for blocked tasks awaiting human review).
1095/// Returns `true` if the worktree directory is gone after the call
1096/// (either we removed it or it was already absent).
1097pub(crate) async fn remove_worktree_and_branch(
1098    task_id: &str,
1099    wt: &Path,
1100    branch: Option<&str>,
1101    repo_root: &Path,
1102    keep_remote_branch: bool,
1103) -> bool {
1104    remove_worktree_and_branch_for(task_id, wt, branch, repo_root, keep_remote_branch, false).await
1105}
1106
1107/// Same as `remove_worktree_and_branch`, but a merged PR skips the rescue. A squash leaves the
1108/// branch commits on no ref, so the rescue would push a branch for work that already landed
1109pub(crate) async fn remove_worktree_and_branch_for(
1110    task_id: &str,
1111    wt: &Path,
1112    branch: Option<&str>,
1113    repo_root: &Path,
1114    keep_remote_branch: bool,
1115    pr_merged: bool,
1116) -> bool {
1117    if !pr_merged
1118        && let Some(br) = branch.filter(|b| !b.is_empty())
1119        && rescue_unpushed_work(task_id, Some(wt), br, repo_root).await == Rescue::Failed
1120    {
1121        return false;
1122    }
1123
1124    remove_worktree_and_branch_unchecked(task_id, wt, branch, repo_root, keep_remote_branch).await
1125}
1126
1127/// The caller has already rescued the branch or knows its PR merged
1128async fn remove_worktree_and_branch_unchecked(
1129    task_id: &str,
1130    wt: &Path,
1131    branch: Option<&str>,
1132    repo_root: &Path,
1133    keep_remote_branch: bool,
1134) -> bool {
1135    let wt_str = wt.to_string_lossy().to_string();
1136    let repo_root_str = repo_root.to_string_lossy();
1137
1138    // Remove worktree FIRST, then delete the branch.
1139    // Git refuses to remove a worktree if its branch is already deleted.
1140    let remove_result = Command::new("git")
1141        .args([
1142            "-C",
1143            repo_root_str.as_ref(),
1144            "worktree",
1145            "remove",
1146            &wt_str,
1147            "--force",
1148        ])
1149        .output_with_context()
1150        .await;
1151
1152    match remove_result {
1153        Ok(output) if output.status.success() => {
1154            const MAX_RETRIES: usize = 3;
1155            const RETRY_DELAY_MS: u64 = 2000;
1156            tracing::info!(task_id, "worktree removed (git metadata)");
1157            // `git worktree remove` only removes git metadata — the physical directory
1158            // with working files may remain. Clean it up with retry logic, since macOS
1159            // sometimes needs a moment to release file handles from a recently-exited
1160            // process. Retrying with a delay addresses "Directory not empty" errors.
1161            let mut removed = false;
1162            for attempt in 0..MAX_RETRIES {
1163                match fs::remove_dir_all(wt).await {
1164                    Ok(()) => {
1165                        removed = true;
1166                        break;
1167                    }
1168                    Err(e) if e.kind() == ErrorKind::NotFound => {
1169                        // Directory was already removed by git or another cleanup pass.
1170                        removed = true;
1171                        break;
1172                    }
1173                    Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => {
1174                        if attempt < MAX_RETRIES - 1 {
1175                            tracing::debug!(
1176                                task_id,
1177                                path = %wt.display(),
1178                                attempt = attempt + 1,
1179                                "worktree directory not empty — retrying after delay"
1180                            );
1181                            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
1182                        } else {
1183                            tracing::warn!(
1184                                task_id,
1185                                path = %wt.display(),
1186                                err = %e,
1187                                "failed to remove worktree directory after {} attempts (Directory not empty)",
1188                                MAX_RETRIES
1189                            );
1190                        }
1191                    }
1192                    Err(e) => {
1193                        tracing::warn!(task_id, path = %wt.display(), err = %e, "failed to remove worktree directory");
1194                        break;
1195                    }
1196                }
1197            }
1198            if removed {
1199                tracing::info!(task_id, path = %wt.display(), "worktree physical directory removed");
1200            } else {
1201                tracing::warn!(
1202                    task_id,
1203                    path = %wt.display(),
1204                    "worktree directory still present after git worktree remove"
1205                );
1206            }
1207        }
1208        Ok(output) => {
1209            let stderr = String::from_utf8_lossy(&output.stderr);
1210            // "is not a working tree" means git's metadata is stale — the path
1211            // exists on disk but is no longer registered as a worktree. Treat
1212            // this as an already-gone worktree: physically remove the directory
1213            // and prune stale git metadata. Downgrade to debug — this is an
1214            // expected idempotency case, not a real error.
1215            if stderr.contains("is not a working tree") {
1216                tracing::debug!(
1217                    task_id,
1218                    path = %wt.display(),
1219                    "path is not a registered worktree (stale metadata) — removing directory and pruning"
1220                );
1221                if let Err(e) = fs::remove_dir_all(wt).await {
1222                    // Only warn if the directory actually still exists after removal attempt.
1223                    if match fs::try_exists(wt).await {
1224                        Ok(true) => true,
1225                        Ok(false) => false,
1226                        Err(e) => {
1227                            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree does not exist");
1228                            false
1229                        }
1230                    } {
1231                        tracing::warn!(
1232                            task_id,
1233                            path = %wt.display(),
1234                            err = %e,
1235                            "failed to remove stale worktree directory"
1236                        );
1237                    }
1238                } else {
1239                    tracing::debug!(task_id, path = %wt.display(), "stale worktree directory removed");
1240                }
1241                // Prune stale worktree entries from git metadata so the warning
1242                // stops recurring on future `git worktree list` / cleanup cycles.
1243                if let Err(e) = Command::new("git")
1244                    .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1245                    .output_with_context()
1246                    .await
1247                {
1248                    tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1249                }
1250            } else {
1251                tracing::warn!(task_id, err = %stderr, "failed to remove worktree");
1252            }
1253        }
1254        Err(e) => {
1255            tracing::warn!(task_id, err = %e, "failed to remove worktree");
1256        }
1257    }
1258
1259    // Prune stale .git/worktrees/ entries left behind after removal.
1260    if let Err(e) = Command::new("git")
1261        .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1262        .output_with_context()
1263        .await
1264    {
1265        tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1266    }
1267
1268    // Delete local and remote branch from the main repo root (worktree is already gone).
1269    // When keep_remote_branch is set, only delete the local branch — the remote stays
1270    // so any linked PR remains open for human review.
1271    if let Some(br) = branch {
1272        if keep_remote_branch {
1273            delete_local_branch(task_id, br, repo_root).await;
1274        } else {
1275            delete_branches(task_id, br, repo_root).await;
1276        }
1277    }
1278
1279    // Definitively check: is the worktree directory actually gone?
1280    match fs::try_exists(wt).await {
1281        Ok(false) => true, // confirmed gone
1282        Ok(true) => false, // still there
1283        Err(e) => {
1284            tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree still present");
1285            false // conservative: don't claim success
1286        }
1287    }
1288}
1289
1290/// Commits reachable from `rev` and from none of `others`. `None` when git cannot answer
1291async fn commits_outside(repo_root: &Path, rev: &str, others: &[&str]) -> Option<u64> {
1292    let mut args = vec![
1293        "rev-list".to_string(),
1294        "--count".to_string(),
1295        rev.to_string(),
1296    ];
1297    for other in others {
1298        let exists = Command::new("git")
1299            .args([
1300                "-C",
1301                &repo_root.to_string_lossy(),
1302                "rev-parse",
1303                "--verify",
1304                "--quiet",
1305                other,
1306            ])
1307            .output_with_context()
1308            .await
1309            .is_ok_and(|o| o.status.success());
1310        if exists {
1311            args.push("--not".to_string());
1312            args.push(other.to_string());
1313        }
1314    }
1315    let output = Command::new("git")
1316        .arg("-C")
1317        .arg(repo_root)
1318        .args(&args)
1319        .output_with_context()
1320        .await
1321        .ok()?;
1322    if !output.status.success() {
1323        return None;
1324    }
1325    String::from_utf8_lossy(&output.stdout).trim().parse().ok()
1326}
1327
1328/// Delete the local branch only when origin or the default branch holds every commit on it
1329async fn delete_local_branch(task_id: &str, br: &str, repo_root: &Path) {
1330    let default_branch = worktree::detect_default_branch(repo_root).await;
1331    let remote_ref = format!("refs/remotes/origin/{br}");
1332    let default_ref = format!("refs/remotes/origin/{default_branch}");
1333    let local_ref = format!("refs/heads/{br}");
1334    let local_default_ref = format!("refs/heads/{default_branch}");
1335    let rescue_ref = format!("refs/remotes/origin/rescue/{br}");
1336    match commits_outside(
1337        repo_root,
1338        &local_ref,
1339        &[&remote_ref, &default_ref, &local_default_ref, &rescue_ref],
1340    )
1341    .await
1342    {
1343        Some(0) => {}
1344        Some(n) => {
1345            tracing::warn!(task_id, branch = %br, unpushed = n, "local branch holds commits origin does not have, keeping it");
1346            return;
1347        }
1348        None => {
1349            tracing::debug!(task_id, branch = %br, "local branch missing or unreadable, nothing to delete");
1350            return;
1351        }
1352    }
1353    let output = Command::new("git")
1354        .args(["-C", &repo_root.to_string_lossy(), "branch", "-D", br])
1355        .output_with_context()
1356        .await;
1357    match output {
1358        Ok(o) if o.status.success() => {
1359            tracing::info!(task_id, branch = %br, "local branch deleted");
1360        }
1361        Ok(o) => {
1362            let stderr = String::from_utf8_lossy(&o.stderr);
1363            tracing::debug!(task_id, err = %stderr, "local branch delete skipped (may not exist)");
1364        }
1365        Err(e) => {
1366            tracing::warn!(task_id, err = %e, "failed to delete local branch");
1367        }
1368    }
1369}
1370
1371/**
1372 * Delete local and remote branches for a task, but never the only copy of a commit.
1373 * The remote branch goes only when the default branch already contains all of it, which
1374 * covers fast-forward and merge-commit merges. A squash-merged branch stays, and GitHub's
1375 * "automatically delete head branches" removes it. Every other caller (startup rebase
1376 * failure, closed-issue reconciliation, orphan sweeps) may hold unmerged work (#3781)
1377 **/
1378pub(crate) async fn delete_branches(task_id: &str, br: &str, repo_root: &Path) {
1379    if br == SCREENSHOTS_BRANCH {
1380        return;
1381    }
1382    let default_branch = worktree::detect_default_branch(repo_root).await;
1383    let _ = git_ops::git()
1384        .await
1385        .args([
1386            "-C",
1387            &repo_root.to_string_lossy(),
1388            "fetch",
1389            "origin",
1390            &default_branch,
1391            br,
1392        ])
1393        .output_with_context()
1394        .await;
1395
1396    delete_local_branch(task_id, br, repo_root).await;
1397
1398    let remote_ref = format!("refs/remotes/origin/{br}");
1399    let default_ref = format!("refs/remotes/origin/{default_branch}");
1400    match commits_outside(repo_root, &remote_ref, &[&default_ref]).await {
1401        Some(0) => {}
1402        Some(n) => {
1403            tracing::info!(task_id, branch = %br, unmerged = n, "remote branch has commits the default branch lacks, keeping it");
1404            return;
1405        }
1406        None => {
1407            tracing::debug!(task_id, branch = %br, "remote branch missing or unreadable, nothing to delete");
1408            return;
1409        }
1410    }
1411
1412    let remote_delete = git_ops::git()
1413        .await
1414        .args([
1415            "-C",
1416            &repo_root.to_string_lossy(),
1417            "-c",
1418            "lfs.locksverify=false",
1419            "push",
1420            "origin",
1421            "--delete",
1422            br,
1423        ])
1424        .output_with_context()
1425        .await;
1426
1427    match remote_delete {
1428        Ok(output) if output.status.success() => {
1429            tracing::info!(task_id, branch = %br, "remote branch deleted");
1430            git_ops::clear_push_lease(repo_root, br).await;
1431        }
1432        Ok(output) => {
1433            let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
1434            tracing::debug!(task_id, err = %stderr, "remote branch delete skipped");
1435        }
1436        Err(e) => {
1437            tracing::warn!(task_id, err = %e, "failed to delete remote branch");
1438        }
1439    }
1440}
1441
1442/// Returns `true` if any active tmux pane has its cwd inside `worktree`.
1443///
1444/// Uses `tmux list-panes -a -F '#{pane_current_path}'` to enumerate all
1445/// pane working directories across every session. If tmux is not running
1446/// (command fails), we conservatively return `false` so cleanup can proceed.
1447async fn is_worktree_in_active_session(worktree: &Path) -> bool {
1448    let output = Command::new("tmux")
1449        .args(["list-panes", "-a", "-F", "#{pane_current_path}"])
1450        .output()
1451        .await;
1452
1453    match output {
1454        Ok(o) if o.status.success() => {
1455            let worktree_str = worktree.to_string_lossy();
1456            String::from_utf8_lossy(&o.stdout).lines().any(|line| {
1457                let pane_path = line.trim();
1458                // Match exact path or any path that starts with worktree + separator.
1459                pane_path == worktree_str.as_ref()
1460                    || pane_path.starts_with(&format!("{worktree_str}/"))
1461            })
1462        }
1463        // tmux not running or no sessions — treat as safe to clean.
1464        _ => false,
1465    }
1466}
1467
1468/// Returns how many hours old the worktree directory is, based on its mtime.
1469///
1470/// Returns `None` if the directory does not exist or the mtime cannot be read.
1471async fn worktree_age_hours(worktree: &Path) -> Option<u64> {
1472    // Use tokio::fs to avoid blocking the async runtime when reading metadata.
1473    let metadata = fs::metadata(worktree).await.ok()?;
1474    let modified = metadata.modified().ok()?;
1475    let age = SystemTime::now().duration_since(modified).ok()?;
1476    Some(age.as_secs() / 3600)
1477}
1478
1479/// Resolve the repo root for an orphaned worktree and remove it.
1480///
1481/// Used by `orch service doctor --fix` and `orch service prune` to clean up worktrees that are
1482/// no longer owned by any task. Returns `true` if the worktree directory is gone
1483/// after the call.
1484pub(crate) async fn cleanup_orphaned_worktree(task_id: &str, wt: &Path) -> bool {
1485    let repo_root = match resolve_repo_root_for_orphaned_worktree(wt).await {
1486        Ok(root) => root,
1487        Err(e) => {
1488            tracing::warn!(
1489                worktree = %wt.display(),
1490                error = %e,
1491                "cannot resolve repo root for orphaned worktree"
1492            );
1493            return false;
1494        }
1495    };
1496    remove_worktree_and_branch(task_id, wt, None, Path::new(&repo_root), false).await
1497}
1498
1499/// Resolve the main git repository root for an orphaned worktree.
1500///
1501/// First tries to parse the `.git` file's `gitdir:` pointer (two levels up from
1502/// the pointer gives the bare repo root, which is the orch convention). Falls back
1503/// to looking up registered projects by directory name when the `.git` file is
1504/// absent or unparseable.
1505pub(crate) async fn resolve_repo_root_for_orphaned_worktree(wt: &Path) -> anyhow::Result<String> {
1506    let git_file = wt.join(".git");
1507    if fs::try_exists(&git_file).await.unwrap_or(false)
1508        && let Ok(content) = fs::read_to_string(&git_file).await
1509    {
1510        for line in content.lines() {
1511            if let Some(gitdir) = line.strip_prefix("gitdir: ") {
1512                // gitdir points to: <repo>/.git/worktrees/<name> (non-bare)
1513                //                or <repo>.git/worktrees/<name>   (bare)
1514                // Two levels up reaches the .git dir / bare repo root.
1515                let gitdir_path = PathBuf::from(gitdir);
1516                if let Some(repo_root) = gitdir_path.parent().and_then(|p| p.parent()) {
1517                    return Ok(repo_root.display().to_string());
1518                }
1519            }
1520        }
1521    }
1522
1523    // Fallback: look up registered projects by the project-name component of the
1524    // worktree path (format: ~/.orch/worktrees/<project>/<branch>).
1525    let worktrees_dir = home::worktrees_dir()?;
1526    if let Ok(relative) = wt.strip_prefix(&worktrees_dir)
1527        && let Some(project_name) = relative.components().next()
1528    {
1529        let project_name = project_name.as_os_str().to_string_lossy();
1530
1531        if let Ok(paths) = config::get_project_paths() {
1532            for path_str in &paths {
1533                let path = Path::new(path_str);
1534                if let Some(name) = path.file_name()
1535                    && name.to_string_lossy() == project_name
1536                {
1537                    return Ok(path_str.clone());
1538                }
1539            }
1540        }
1541
1542        // Try bare clone at ~/.orch/projects/<owner>/<project>.git
1543        let parts: Vec<&str> = project_name.split('/').collect();
1544        let bare = if parts.len() == 2 {
1545            home::projects_dir()
1546                .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1547                .unwrap_or_default()
1548        } else {
1549            return Err(anyhow::anyhow!(
1550                "cannot determine owner for single-segment project name '{project_name}' — \
1551                     bare repo location is ambiguous; register the project with 'owner/repo' format"
1552            ));
1553        };
1554        if fs::try_exists(&bare).await.unwrap_or(false) {
1555            return Ok(bare.display().to_string());
1556        }
1557    }
1558
1559    anyhow::bail!(
1560        "cannot resolve repo root for orphaned worktree at {} — tried .git parsing and project lookup",
1561        wt.display()
1562    )
1563}
1564
1565/// Resolve the main git repository root from a worktree's .git file.
1566///
1567/// For cross-project worktree cleanup, we cannot rely on `resolve_repo_root` because
1568/// the worktree may belong to a different project than the current repo context.
1569/// Instead, we extract the repo root from the worktree's `.git` file which contains
1570/// a `gitdir:` pointer to the main repo's git directory.
1571pub async fn resolve_repo_root_from_worktree(wt: &Path) -> anyhow::Result<String> {
1572    // For worktrees, .git is a file (not a directory) containing gitdir path
1573    let git_file = wt.join(".git");
1574    if !fs::try_exists(&git_file).await.unwrap_or(false) {
1575        anyhow::bail!(".git file not found in worktree at {}", wt.display());
1576    }
1577
1578    let content = fs::read_to_string(&git_file)
1579        .await
1580        .map_err(|e| anyhow::anyhow!("failed to read .git file: {e}"))?;
1581
1582    // Parse gitdir line: "gitdir: /path/to/repo/.git/worktrees/<name>"
1583    for line in content.lines() {
1584        if let Some(gitdir) = line.strip_prefix("gitdir:") {
1585            let gitdir = gitdir.trim();
1586            // The main repo is the parent of the .git directory
1587            // gitdir points to: <repo>/.git/worktrees/<name>
1588            // We need to go up 3 levels: worktrees/<name> -> .git -> repo
1589            let gitdir_path = Path::new(gitdir);
1590            if let Some(repo_root) = gitdir_path
1591                .parent() // worktrees/<name>
1592                .and_then(|p| p.parent()) // .git
1593                .and_then(|p| p.parent())
1594            // repo root
1595            {
1596                return Ok(repo_root.display().to_string());
1597            }
1598        }
1599    }
1600
1601    anyhow::bail!(
1602        "cannot resolve repo root from worktree .git file at {}",
1603        wt.display()
1604    )
1605}
1606
1607/// Resolve the main git repository root path for a project.
1608///
1609/// Looks up the local project path from config, then verifies it's a git repo.
1610/// This avoids relying on cwd (which is undefined under launchd services).
1611pub(crate) async fn resolve_repo_root(repo: &str) -> anyhow::Result<String> {
1612    // Look up the local path from registered projects
1613    let paths = config::get_project_paths().unwrap_or_default();
1614    for path_str in &paths {
1615        let path = Path::new(path_str);
1616        let orch_yml = path.join(".orch.yml");
1617        let legacy = path.join(".orchestrator.yml");
1618        let config_file = if fs::try_exists(&orch_yml).await.unwrap_or(false) {
1619            orch_yml
1620        } else if fs::try_exists(&legacy).await.unwrap_or(false) {
1621            legacy
1622        } else {
1623            continue;
1624        };
1625        if let Ok(content) = fs::read_to_string(&config_file).await
1626            && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1627            && let Some(r) = doc
1628                .get("gh")
1629                .and_then(|gh| gh.get("repo"))
1630                .and_then(|r| r.as_str())
1631            && r == repo
1632        {
1633            return Ok(path_str.clone());
1634        }
1635    }
1636
1637    // Fallback: try bare clone in ~/.orch/projects/<owner>/<repo>.git
1638    let parts: Vec<&str> = repo.split('/').collect();
1639    let bare = if parts.len() == 2 {
1640        home::projects_dir()
1641            .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1642            .unwrap_or_default()
1643    } else {
1644        PathBuf::new()
1645    };
1646    if fs::try_exists(&bare).await.unwrap_or(false) {
1647        return Ok(bare.display().to_string());
1648    }
1649
1650    anyhow::bail!(
1651        "cannot find local path for project {repo} — \
1652         checked {} registered project(s) and bare clone at {}",
1653        paths.len(),
1654        bare.display()
1655    )
1656}
1657
1658/// Resolve the project directory for a given repo string.
1659///
1660/// Replicates the logic from `TaskRunner::resolve_project_dir()` without requiring
1661/// a Runner instance. Used by the orphan scanner to enumerate active worktrees.
1662fn resolve_project_dir_for_repo(repo: &str) -> anyhow::Result<PathBuf> {
1663    // Check PROJECT_DIR env var first
1664    if let Ok(dir) = env::var("PROJECT_DIR")
1665        && !dir.is_empty()
1666    {
1667        return Ok(PathBuf::from(dir));
1668    }
1669
1670    // Look up registered projects in config
1671    if let Ok(paths) = config::get_project_paths() {
1672        for path_str in &paths {
1673            let path = Path::new(path_str);
1674            let orch_yml = path.join(".orch.yml");
1675            let legacy = path.join(".orchestrator.yml");
1676            let config_file = if Path::new(&orch_yml).exists() {
1677                orch_yml
1678            } else if Path::new(&legacy).exists() {
1679                legacy
1680            } else {
1681                continue;
1682            };
1683            if let Ok(content) = std_fs::read_to_string(&config_file)
1684                && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1685                && let Some(r) = doc
1686                    .get("gh")
1687                    .and_then(|gh| gh.get("repo"))
1688                    .and_then(|r| r.as_str())
1689                && r == repo
1690            {
1691                return Ok(path_str.clone().into());
1692            }
1693        }
1694    }
1695
1696    anyhow::bail!("project dir not found for repo: {repo}")
1697}
1698
1699/// Check for merged PRs and update task status accordingly.
1700///
1701/// Queries status:in_review and status:needs_review tasks, checks if their PR
1702/// is merged, and updates status to done if merged.
1703pub(crate) async fn check_merged_prs(
1704    backend: &Arc<dyn ExternalBackend>,
1705    repo: &str,
1706    store: &Arc<TaskStore>,
1707    task_manager: &Arc<TaskManager>,
1708    in_review_tasks: &[ReviewTaskSnapshot],
1709    needs_review_tasks: &[ReviewTaskSnapshot],
1710) -> anyhow::Result<()> {
1711    let mut branch_map: HashMap<String, String> = HashMap::new();
1712    let all_review_tasks: Vec<&ReviewTaskSnapshot> = in_review_tasks
1713        .iter()
1714        .chain(needs_review_tasks.iter())
1715        .collect();
1716
1717    for task in &all_review_tasks {
1718        if !task.stored.branch.is_empty() {
1719            branch_map.insert(task.external.id.0.clone(), task.stored.branch.clone());
1720        }
1721    }
1722
1723    tracing::debug!(
1724        count = all_review_tasks.len(),
1725        "checking review tasks for merged PRs"
1726    );
1727
1728    // Collect (task_id, branch) pairs — skip tasks with no branch recorded.
1729    // Use the pre-built branch map; fall back to store lookup only for tasks
1730    // not in the map (e.g. internal tasks, backend fallback path).
1731    let mut task_branches: Vec<(String, String)> = Vec::new();
1732    for task in all_review_tasks {
1733        let task_id = &task.external.id.0;
1734        let branch = if let Some(b) = branch_map.get(task_id) {
1735            b.clone()
1736        } else {
1737            match store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
1738                .await
1739                .map(|t| t.branch)
1740            {
1741                Some(b) if !b.is_empty() => b,
1742                _ => {
1743                    tracing::debug!(task_id, "no branch info, skipping PR check");
1744                    continue;
1745                }
1746            }
1747        };
1748        task_branches.push((task_id.clone(), branch));
1749    }
1750
1751    if task_branches.is_empty() {
1752        return Ok(());
1753    }
1754
1755    // Single GraphQL call for all branches — N REST calls → 1 GraphQL call.
1756    let branches: Vec<String> = task_branches.iter().map(|(_, b)| b.clone()).collect();
1757    let merged_map = match backend.batch_is_pr_merged(&branches).await {
1758        Ok(map) => map,
1759        Err(e) => {
1760            tracing::warn!(
1761                err = %e,
1762                branch_count = branches.len(),
1763                "batch PR merge check failed, retrying affected branches individually"
1764            );
1765            fallback_is_pr_merged_by_branch(backend, &branches).await
1766        }
1767    };
1768
1769    for (task_id, branch) in task_branches {
1770        let is_merged = merged_map.get(&branch).copied().unwrap_or(false);
1771        if !is_merged {
1772            continue;
1773        }
1774
1775        tracing::info!(task_id, branch = %branch, "PR merged, marking task complete");
1776
1777        let stored = store::opt_store_get_task(&Some(Arc::clone(store)), repo, &task_id).await;
1778        if let Some(ref stored) = stored {
1779            let external = store_task_to_external(stored);
1780            if ci_bootstrap::hold_after_merge(&external, store, repo).await {
1781                tracing::info!(
1782                    task_id,
1783                    branch = %branch,
1784                    "CI bootstrap PR merged, holding done until the ruleset exists"
1785                );
1786                if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1787                    tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1788                }
1789                continue;
1790            }
1791        }
1792
1793        let id = ExternalId(task_id.clone());
1794        if let Err(e) = task_manager.update_task_status(&id, Status::Done).await {
1795            tracing::warn!(task_id, err = %e, "failed to update status to done");
1796            continue;
1797        }
1798
1799        if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1800            tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1801        }
1802
1803        let comment = format!("PR merged, marking task complete{}", engine::orch_footer());
1804        if let Err(e) = backend.post_comment(&id, &comment).await {
1805            tracing::warn!(task_id, err = %e, "failed to post comment");
1806        }
1807    }
1808
1809    Ok(())
1810}
1811
1812async fn fallback_is_pr_merged_by_branch(
1813    backend: &Arc<dyn ExternalBackend>,
1814    branches: &[String],
1815) -> HashMap<String, bool> {
1816    let futs: Vec<_> = branches
1817        .iter()
1818        .map(|branch| {
1819            let backend = Arc::clone(backend);
1820            let branch = branch.clone();
1821            async move {
1822                match backend.is_pr_merged(&branch).await {
1823                    Ok(is_merged) => Some((branch, is_merged)),
1824                    Err(err) => {
1825                        tracing::warn!(
1826                            branch,
1827                            err = %err,
1828                            "fallback PR merge check failed for branch"
1829                        );
1830                        None
1831                    }
1832                }
1833            }
1834        })
1835        .collect();
1836
1837    future::join_all(futs).await.into_iter().flatten().collect()
1838}
1839
1840#[cfg(test)]
1841mod tests {
1842    use super::*;
1843    use crate::engine::tasks;
1844    use crate::store::{NewTask, TaskStore};
1845    use async_trait::async_trait;
1846    use std::collections::HashMap;
1847    use std::process;
1848    use std::sync::Mutex;
1849    use std::sync::atomic::{AtomicUsize, Ordering};
1850
1851    // worktree_age_hours
1852
1853    #[tokio::test]
1854    async fn worktree_age_hours_returns_none_for_missing_dir() {
1855        let tmp = tempfile::tempdir().unwrap();
1856        let missing = tmp.path().join("does-not-exist");
1857        assert!(worktree_age_hours(&missing).await.is_none());
1858    }
1859
1860    #[tokio::test]
1861    async fn worktree_age_hours_returns_some_for_existing_dir() {
1862        let tmp = tempfile::tempdir().unwrap();
1863        // A freshly created directory should have age ~0 hours.
1864        let age = worktree_age_hours(tmp.path()).await;
1865        assert!(age.is_some());
1866        assert_eq!(age.unwrap(), 0, "freshly created dir should have age 0h");
1867    }
1868
1869    // JanitorOptions::from_config defaults
1870
1871    #[test]
1872    fn janitor_options_default_values() {
1873        let opts = JanitorOptions::default();
1874        assert_eq!(opts.ttl_hours, 24);
1875        assert!(!opts.dry_run);
1876    }
1877
1878    // cleanup_task_worktree_with_opts TTL guard (skips if git missing)
1879
1880    /// Create a minimal git repo + worktree; return (tmp, worktree_dir).
1881    fn setup_test_repo() -> Option<(tempfile::TempDir, PathBuf)> {
1882        let tmp = tempfile::tempdir().ok()?;
1883
1884        // Create a real git repo (not bare) so we can add worktrees.
1885        let repo_dir = tmp.path().join("repo.git");
1886        std_fs::create_dir_all(&repo_dir).ok()?;
1887
1888        let init = process::Command::new("git")
1889            .args([
1890                "init",
1891                "--initial-branch=main",
1892                repo_dir
1893                    .to_str()
1894                    .expect("test repo path contains non-UTF-8 characters"),
1895            ])
1896            .output()
1897            .ok()?;
1898        if !init.status.success() {
1899            // Older git may not have --initial-branch; fall back.
1900            let _ = process::Command::new("git")
1901                .args([
1902                    "init",
1903                    repo_dir
1904                        .to_str()
1905                        .expect("test repo path contains non-UTF-8 characters"),
1906                ])
1907                .output()
1908                .ok()?;
1909        }
1910
1911        // Create an initial commit so the repo has a HEAD.
1912        let readme = repo_dir.join("README.md");
1913        std_fs::write(&readme, "# test").ok()?;
1914        process::Command::new("git")
1915            .args(["-C", repo_dir.to_str()?, "add", "."])
1916            .output()
1917            .ok()?;
1918        process::Command::new("git")
1919            .args([
1920                "-C",
1921                repo_dir.to_str()?,
1922                "-c",
1923                "user.email=test@test.com",
1924                "-c",
1925                "user.name=Test",
1926                "-c",
1927                "commit.gpgsign=false",
1928                "commit",
1929                "--allow-empty",
1930                "-m",
1931                "init",
1932            ])
1933            .output()
1934            .ok()?;
1935
1936        // Create a worktree branch.
1937        let wt_dir = tmp
1938            .path()
1939            .join("worktrees")
1940            .join("repo.git")
1941            .join("gh-issue-42-test");
1942        std_fs::create_dir_all(wt_dir.parent()?).ok()?;
1943        let wt_out = process::Command::new("git")
1944            .args([
1945                "-C",
1946                repo_dir.to_str()?,
1947                "worktree",
1948                "add",
1949                "-b",
1950                "gh-issue-42-test",
1951                wt_dir.to_str()?,
1952            ])
1953            .output()
1954            .ok()?;
1955
1956        if !wt_out.status.success() {
1957            return None;
1958        }
1959
1960        Some((tmp, wt_dir))
1961    }
1962
1963    #[tokio::test]
1964    async fn janitor_skips_young_worktree() {
1965        let Some((tmp, wt_dir)) = setup_test_repo() else {
1966            eprintln!("skipping test: git not available");
1967            return;
1968        };
1969
1970        // The worktree was just created → age is 0h < ttl_hours=24 → must be skipped.
1971        assert!(wt_dir.exists(), "worktree should exist before janitor runs");
1972
1973        // Run the TTL guard check directly (without a full engine).
1974        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
1975        assert!(
1976            age < 24,
1977            "freshly created worktree should be younger than 24h (age={age}h)"
1978        );
1979
1980        // Explicitly verify: with ttl_hours=24 the worktree stays.
1981        let opts = JanitorOptions {
1982            ttl_hours: 24,
1983            dry_run: true,
1984        };
1985        // Age < ttl → should skip (worktree_age_hours < opts.ttl_hours).
1986        assert!(age < opts.ttl_hours, "janitor should skip young worktree");
1987
1988        drop(tmp); // cleanup tempdir
1989    }
1990
1991    #[tokio::test]
1992    async fn janitor_eligible_when_ttl_zero() {
1993        let Some((tmp, wt_dir)) = setup_test_repo() else {
1994            eprintln!("skipping test: git not available");
1995            return;
1996        };
1997
1998        // With ttl_hours=0 even a brand-new worktree is eligible.
1999        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2000        let opts = JanitorOptions {
2001            ttl_hours: 0,
2002            dry_run: true,
2003        };
2004        assert!(
2005            age >= opts.ttl_hours,
2006            "worktree should be eligible when ttl=0 (age={age}h)"
2007        );
2008
2009        drop(tmp);
2010    }
2011
2012    #[tokio::test]
2013    async fn dry_run_does_not_remove_worktree() {
2014        // Verify that with dry_run=true the worktree directory is NOT removed.
2015        let Some((tmp, wt_dir)) = setup_test_repo() else {
2016            eprintln!("skipping test: git not available");
2017            return;
2018        };
2019
2020        assert!(wt_dir.exists(), "worktree must exist before test");
2021
2022        // With dry_run=true and ttl_hours=0 the janitor would normally remove the
2023        // worktree, but dry_run prevents it.
2024        let opts = JanitorOptions {
2025            ttl_hours: 0,
2026            dry_run: true,
2027        };
2028
2029        // The janitor needs a repo_root to run git commands.  We point it at the
2030        // bare repo we created inside tmp (even though it's not registered in config).
2031        // Because resolve_repo_root will fail for our synthetic repo slug, we test the
2032        // inner helpers directly here.
2033        let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2034        let in_use = false; // no tmux running in tests
2035        let skip = age < opts.ttl_hours || in_use;
2036
2037        if !skip && opts.dry_run {
2038            // dry-run: log only, don't touch filesystem.
2039            tracing::debug!("[dry-run] would remove {}", wt_dir.display());
2040        } else if !skip {
2041            // This branch would do actual removal — but we're in dry_run mode above.
2042        }
2043
2044        // Worktree must still be there.
2045        assert!(
2046            wt_dir.exists(),
2047            "dry-run must not remove the worktree directory"
2048        );
2049
2050        drop(tmp);
2051    }
2052
2053    #[test]
2054    fn fallback_branch_path_construction() {
2055        // Verify the new fallback logic: when worktree path is absent but
2056        // branch is known, we should construct the path correctly.
2057        let tmp = tempfile::tempdir().unwrap();
2058        let wt_dir = tmp.path().join("myrepo").join("gh-issue-7-fix");
2059        std_fs::create_dir_all(&wt_dir).unwrap();
2060
2061        // Simulate: worktree=None, branch=Some("gh-issue-7-fix"), repo="owner/myrepo"
2062        let branch = "gh-issue-7-fix";
2063        let repo = "owner/myrepo";
2064        let worktrees_base = tmp.path();
2065
2066        let project = repo
2067            .rsplit('/')
2068            .next()
2069            .unwrap_or(repo)
2070            .trim_end_matches(".git");
2071        let candidate = worktrees_base.join(project).join(branch);
2072
2073        assert_eq!(candidate, wt_dir);
2074        assert!(
2075            candidate.exists(),
2076            "fallback path should point to existing dir"
2077        );
2078    }
2079
2080    /// A task with no worktree and no branch should return Ok(false) —
2081    /// nothing was cleaned, so git pull should NOT be triggered.
2082    #[tokio::test]
2083    async fn cleanup_returns_false_when_nothing_to_clean() {
2084        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2085
2086        // Create a done task with no worktree or branch fields set
2087        let id = store
2088            .create(&NewTask {
2089                external_id: Some("999".to_string()),
2090                repo: "owner/repo".to_string(),
2091                origin: "github".to_string(),
2092                title: "Already cleaned".to_string(),
2093                ..Default::default()
2094            })
2095            .await
2096            .unwrap();
2097        store.update_status(id, TaskStatus::Done).await.unwrap();
2098
2099        let opts = JanitorOptions {
2100            ttl_hours: 0,
2101            dry_run: false,
2102        };
2103
2104        let result = cleanup_task_worktree_with_opts("999", "owner/repo", &store, &opts).await;
2105        // resolve_repo_root will fail for synthetic repo — that's OK, the point is
2106        // that when nothing needs cleaning the function should not report true.
2107        // In production this means no git pull is triggered.
2108        if let Ok(cleaned) = result {
2109            assert!(
2110                !cleaned,
2111                "cleanup should return false when there is no worktree or branch to clean"
2112            );
2113        } else {
2114            // resolve_repo_root fails for test repo — acceptable.
2115            // The bug was that even with no worktree/branch, Ok(true) was returned.
2116        }
2117    }
2118
2119    /// When the stored worktree path no longer exists on disk and there is no
2120    /// branch recorded, cleanup must mark the task as cleaned so subsequent
2121    /// runs do not keep retrying (idempotency regression test for #1021).
2122    #[tokio::test]
2123    async fn cleanup_marks_cleaned_when_stored_path_is_gone() {
2124        let tmp = tempfile::tempdir().unwrap();
2125        // A path that does NOT exist on disk.
2126        let stale_wt = tmp.path().join("stale-worktree");
2127        assert!(!stale_wt.exists(), "stale path must not exist");
2128
2129        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2130
2131        let id = store
2132            .create(&NewTask {
2133                external_id: Some("1005".to_string()),
2134                repo: "owner/repo".to_string(),
2135                origin: "github".to_string(),
2136                title: "Stale worktree task".to_string(),
2137                ..Default::default()
2138            })
2139            .await
2140            .unwrap();
2141        store.update_status(id, TaskStatus::Done).await.unwrap();
2142        // Set the stale worktree path on the task.
2143        store
2144            .set_fields(
2145                id,
2146                &[("worktree", stale_wt.to_string_lossy().as_ref().into())],
2147            )
2148            .await
2149            .unwrap();
2150
2151        // Verify the task starts out NOT cleaned.
2152        let before = store.get(id).await.unwrap();
2153        assert!(!before.worktree_cleaned, "task should not be cleaned yet");
2154        assert!(
2155            !before.worktree.is_empty(),
2156            "task must have a worktree path set"
2157        );
2158
2159        let opts = JanitorOptions {
2160            ttl_hours: 0,
2161            dry_run: false,
2162        };
2163
2164        let result = cleanup_task_worktree_with_opts("1005", "owner/repo", &store, &opts).await;
2165
2166        // The function should succeed (Ok) — stale path is not an error.
2167        // It returns Ok(false) because there was nothing to actively remove.
2168        match result {
2169            Ok(_) => {}
2170            Err(e) => panic!("cleanup should not error for stale path: {e}"),
2171        }
2172
2173        // The task must now be marked cleaned so the next cycle skips it.
2174        let after = store.get(id).await.unwrap();
2175        assert!(
2176            after.worktree_cleaned,
2177            "task must be marked cleaned when stored worktree path is already gone"
2178        );
2179    }
2180
2181    /// `remove_worktree_and_branch` returns true when the worktree directory
2182    /// is successfully removed (regression test for #1143).
2183    #[tokio::test]
2184    async fn remove_worktree_returns_true_on_success() {
2185        let Some((tmp, wt_dir)) = setup_test_repo() else {
2186            eprintln!("skipping test: git not available");
2187            return;
2188        };
2189
2190        let repo_dir = tmp.path().join("repo.git");
2191        assert!(wt_dir.exists(), "worktree must exist before removal");
2192
2193        let removed =
2194            remove_worktree_and_branch("42", &wt_dir, Some("gh-issue-42-test"), &repo_dir, false)
2195                .await;
2196
2197        assert!(
2198            removed,
2199            "remove_worktree_and_branch should return true when directory is gone"
2200        );
2201        assert!(!wt_dir.exists(), "worktree directory should be removed");
2202    }
2203
2204    /// Successful worktree cleanup must still delete the local branch.
2205    /// This guards against early-return regressions in `remove_worktree_and_branch`.
2206    #[tokio::test]
2207    async fn remove_worktree_success_still_deletes_local_branch() {
2208        let Some((tmp, wt_dir)) = setup_test_repo() else {
2209            eprintln!("skipping test: git not available");
2210            return;
2211        };
2212
2213        let repo_dir = tmp.path().join("repo.git");
2214        let repo_dir_str = repo_dir
2215            .to_str()
2216            .expect("test repo path contains non-UTF-8 characters");
2217        let branch = "gh-issue-42-test";
2218
2219        let before = process::Command::new("git")
2220            .args([
2221                "-C",
2222                repo_dir_str,
2223                "show-ref",
2224                "--verify",
2225                "--quiet",
2226                &format!("refs/heads/{branch}"),
2227            ])
2228            .status()
2229            .unwrap();
2230        assert!(
2231            before.success(),
2232            "test setup must create local branch before cleanup"
2233        );
2234
2235        let removed =
2236            remove_worktree_and_branch("42", &wt_dir, Some(branch), &repo_dir, false).await;
2237        assert!(removed, "worktree should be removed");
2238
2239        let after = process::Command::new("git")
2240            .args([
2241                "-C",
2242                repo_dir_str,
2243                "show-ref",
2244                "--verify",
2245                "--quiet",
2246                &format!("refs/heads/{branch}"),
2247            ])
2248            .status()
2249            .unwrap();
2250        assert!(
2251            !after.success(),
2252            "local branch should be deleted after successful worktree cleanup"
2253        );
2254    }
2255
2256    /// `remove_worktree_and_branch` returns false when the worktree directory
2257    /// cannot be removed (regression test for #1143 — prevents orphaned worktrees
2258    /// from being permanently marked as cleaned).
2259    #[tokio::test]
2260    async fn remove_worktree_returns_false_when_dir_persists() {
2261        let tmp = tempfile::tempdir().unwrap();
2262        // Create a plain directory (not a real git worktree) — git will fail
2263        // to remove it, so the directory stays on disk.
2264        let fake_wt = tmp.path().join("not-a-worktree");
2265        std_fs::create_dir_all(&fake_wt).unwrap();
2266
2267        // repo_root doesn't matter much — it just needs to exist for the -C flag.
2268        // Using tmp itself; git commands will fail but that's the point.
2269        let removed = remove_worktree_and_branch("99", &fake_wt, None, tmp.path(), false).await;
2270
2271        // The directory still exists because git couldn't remove it.
2272        // Before the fix, this would still have been marked as cleaned.
2273        assert!(
2274            !removed,
2275            "remove_worktree_and_branch should return false when directory still exists"
2276        );
2277        assert!(fake_wt.exists(), "directory should still be on disk");
2278    }
2279
2280    struct MergeCheckMockBackend {
2281        batch_result: Mutex<anyhow::Result<HashMap<String, bool>>>,
2282        single_results: HashMap<String, anyhow::Result<bool>>,
2283        posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2284        single_checked_branches: Arc<Mutex<Vec<String>>>,
2285    }
2286
2287    impl MergeCheckMockBackend {
2288        fn with_batch_error(
2289            error: anyhow::Error,
2290            single_results: HashMap<String, anyhow::Result<bool>>,
2291        ) -> Self {
2292            Self {
2293                batch_result: Mutex::new(Err(error)),
2294                single_results,
2295                posted_comments: Arc::new(Mutex::new(Vec::new())),
2296                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2297            }
2298        }
2299
2300        fn with_merged(branch: &str) -> Self {
2301            Self {
2302                batch_result: Mutex::new(Ok(HashMap::from([(branch.to_string(), true)]))),
2303                single_results: HashMap::new(),
2304                posted_comments: Arc::new(Mutex::new(Vec::new())),
2305                single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2306            }
2307        }
2308    }
2309
2310    #[async_trait]
2311    impl ExternalBackend for MergeCheckMockBackend {
2312        fn name(&self) -> &'static str {
2313            "mock"
2314        }
2315
2316        async fn create_task(
2317            &self,
2318            _title: &str,
2319            _body: &str,
2320            _labels: &[String],
2321        ) -> anyhow::Result<ExternalId> {
2322            anyhow::bail!("not implemented")
2323        }
2324
2325        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2326            anyhow::bail!("not implemented")
2327        }
2328
2329        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2330            Ok(vec![])
2331        }
2332
2333        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2334            Ok(vec![])
2335        }
2336
2337        async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2338            self.posted_comments
2339                .lock()
2340                .unwrap()
2341                .push((id.0.clone(), body.to_string()));
2342            Ok(())
2343        }
2344
2345        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2346            Ok(())
2347        }
2348
2349        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2350            Ok(())
2351        }
2352
2353        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2354            Ok(vec![])
2355        }
2356
2357        async fn create_sub_task(
2358            &self,
2359            _parent: &ExternalId,
2360            _title: &str,
2361            _body: &str,
2362            _labels: &[String],
2363        ) -> anyhow::Result<ExternalId> {
2364            anyhow::bail!("not implemented")
2365        }
2366
2367        async fn health_check(&self) -> anyhow::Result<()> {
2368            Ok(())
2369        }
2370
2371        async fn is_pr_merged(&self, branch: &str) -> anyhow::Result<bool> {
2372            self.single_checked_branches
2373                .lock()
2374                .unwrap()
2375                .push(branch.to_string());
2376            match self.single_results.get(branch) {
2377                Some(Ok(is_merged)) => Ok(*is_merged),
2378                Some(Err(err)) => Err(anyhow::anyhow!(err.to_string())),
2379                None => Ok(false),
2380            }
2381        }
2382
2383        async fn batch_is_pr_merged(
2384            &self,
2385            _branches: &[String],
2386        ) -> anyhow::Result<HashMap<String, bool>> {
2387            match &*self.batch_result.lock().unwrap() {
2388                Ok(map) => Ok(map.clone()),
2389                Err(err) => Err(anyhow::anyhow!(err.to_string())),
2390            }
2391        }
2392    }
2393
2394    #[tokio::test]
2395    async fn check_merged_prs_falls_back_to_single_branch_checks_on_batch_error() {
2396        let single_results = HashMap::from([
2397            ("branch-1".to_string(), Ok(true)),
2398            ("branch-2".to_string(), Ok(false)),
2399        ]);
2400        let backend = Arc::new(MergeCheckMockBackend::with_batch_error(
2401            anyhow::anyhow!("missing /data/repository in GraphQL response: no error details"),
2402            single_results,
2403        ));
2404        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2405
2406        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2407        let task1 = store
2408            .create(&NewTask {
2409                external_id: Some("101".to_string()),
2410                repo: "owner/repo".to_string(),
2411                origin: "github".to_string(),
2412                title: "Task 101".to_string(),
2413                ..Default::default()
2414            })
2415            .await
2416            .unwrap();
2417        store
2418            .update_status(task1, TaskStatus::NeedsReview)
2419            .await
2420            .unwrap();
2421        store
2422            .set_fields(task1, &[("branch", serde_json::json!("branch-1"))])
2423            .await
2424            .unwrap();
2425        // No .git file, so cleanup force-removes the directory
2426        let wt_dir = tempfile::tempdir().unwrap();
2427        let wt_path = wt_dir.path().join("branch-1");
2428        std_fs::create_dir_all(&wt_path).unwrap();
2429        store
2430            .set_fields(
2431                task1,
2432                &[("worktree", serde_json::json!(wt_path.to_string_lossy()))],
2433            )
2434            .await
2435            .unwrap();
2436
2437        let task2 = store
2438            .create(&NewTask {
2439                external_id: Some("102".to_string()),
2440                repo: "owner/repo".to_string(),
2441                origin: "github".to_string(),
2442                title: "Task 102".to_string(),
2443                ..Default::default()
2444            })
2445            .await
2446            .unwrap();
2447        store
2448            .update_status(task2, TaskStatus::InReview)
2449            .await
2450            .unwrap();
2451        store
2452            .set_fields(task2, &[("branch", serde_json::json!("branch-2"))])
2453            .await
2454            .unwrap();
2455
2456        let task_manager = Arc::new(TaskManager::with_store(
2457            Arc::clone(&backend_dyn),
2458            Arc::clone(&store),
2459            "owner/repo".to_string(),
2460        ));
2461
2462        let in_review_tasks = store
2463            .list_by_status("owner/repo", TaskStatus::InReview)
2464            .await
2465            .unwrap()
2466            .into_iter()
2467            .map(|stored| ReviewTaskSnapshot {
2468                external: tasks::store_task_to_external(&stored),
2469                stored,
2470            })
2471            .collect::<Vec<_>>();
2472        let needs_review_tasks = store
2473            .list_by_status("owner/repo", TaskStatus::NeedsReview)
2474            .await
2475            .unwrap()
2476            .into_iter()
2477            .map(|stored| ReviewTaskSnapshot {
2478                external: tasks::store_task_to_external(&stored),
2479                stored,
2480            })
2481            .collect::<Vec<_>>();
2482
2483        check_merged_prs(
2484            &backend_dyn,
2485            "owner/repo",
2486            &store,
2487            &task_manager,
2488            &in_review_tasks,
2489            &needs_review_tasks,
2490        )
2491        .await
2492        .unwrap();
2493
2494        let checked = backend.single_checked_branches.lock().unwrap().clone();
2495        assert_eq!(
2496            checked,
2497            vec!["branch-2".to_string(), "branch-1".to_string()]
2498        );
2499
2500        let updated_task1 = store.get(task1).await.unwrap();
2501        let updated_task2 = store.get(task2).await.unwrap();
2502        assert_eq!(updated_task1.status, TaskStatus::Done);
2503        assert!(updated_task1.worktree_cleaned);
2504        assert!(!wt_path.exists());
2505        assert!(!updated_task2.worktree_cleaned);
2506        assert_eq!(updated_task2.status, TaskStatus::InReview);
2507
2508        let comments = backend.posted_comments.lock().unwrap().clone();
2509        assert_eq!(comments.len(), 1);
2510        assert_eq!(comments[0].0, "101");
2511        assert!(comments[0].1.contains("PR merged, marking task complete"));
2512    }
2513
2514    #[tokio::test]
2515    async fn check_merged_prs_skips_rescue_while_ci_bootstrap_holds() {
2516        let (tmp, clone) = origin_and_clone_with_feat();
2517        let wt = add_worktree(&clone, tmp.path(), "feat");
2518        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2519        // hold_after_merge reads the plan marker in the body, not the title
2520        let id = store_approved_task(
2521            &store,
2522            "owner/repo",
2523            "task-12",
2524            "CI-01 Set up CI and tests",
2525            "<!-- orch-plan: CI-01 -->",
2526            &wt,
2527        )
2528        .await;
2529        let stored = store.get(id).await.unwrap();
2530
2531        let backend = Arc::new(MergeCheckMockBackend::with_merged("feat"));
2532        let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2533        let task_manager = Arc::new(TaskManager::with_store(
2534            Arc::clone(&backend_dyn),
2535            Arc::clone(&store),
2536            "owner/repo".to_string(),
2537        ));
2538
2539        check_merged_prs(
2540            &backend_dyn,
2541            "owner/repo",
2542            &store,
2543            &task_manager,
2544            &[ReviewTaskSnapshot {
2545                external: tasks::store_task_to_external(&stored),
2546                stored,
2547            }],
2548            &[],
2549        )
2550        .await
2551        .unwrap();
2552
2553        let after = store.get(id).await.unwrap();
2554        assert_eq!(after.status, TaskStatus::Approved);
2555        assert!(after.worktree_cleaned);
2556        assert!(!wt.exists());
2557        assert!(!remote_has(&clone, "rescue/feat"));
2558        assert!(backend.posted_comments.lock().unwrap().is_empty());
2559    }
2560
2561    struct ReconciliationMockBackend {
2562        reconciliation_calls: Arc<AtomicUsize>,
2563    }
2564
2565    #[async_trait]
2566    impl ExternalBackend for ReconciliationMockBackend {
2567        fn name(&self) -> &'static str {
2568            "mock"
2569        }
2570
2571        async fn create_task(
2572            &self,
2573            _title: &str,
2574            _body: &str,
2575            _labels: &[String],
2576        ) -> anyhow::Result<ExternalId> {
2577            anyhow::bail!("not implemented")
2578        }
2579
2580        async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2581            anyhow::bail!("not implemented")
2582        }
2583
2584        async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2585            Ok(vec![])
2586        }
2587
2588        async fn list_reconciliation_candidates(&self) -> anyhow::Result<Vec<ExternalTask>> {
2589            self.reconciliation_calls.fetch_add(1, Ordering::SeqCst);
2590            Ok(vec![])
2591        }
2592
2593        async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2594            Ok(vec![])
2595        }
2596
2597        async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2598            Ok(())
2599        }
2600
2601        async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2602            Ok(())
2603        }
2604
2605        async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2606            Ok(())
2607        }
2608
2609        async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2610            Ok(vec![])
2611        }
2612
2613        async fn create_sub_task(
2614            &self,
2615            _parent: &ExternalId,
2616            _title: &str,
2617            _body: &str,
2618            _labels: &[String],
2619        ) -> anyhow::Result<ExternalId> {
2620            anyhow::bail!("not implemented")
2621        }
2622
2623        async fn health_check(&self) -> anyhow::Result<()> {
2624            Ok(())
2625        }
2626    }
2627
2628    #[tokio::test]
2629    async fn cleanup_reconciliation_is_rate_limited_per_repo() {
2630        let repo = "owner/repo-cleanup-rate-limit";
2631        RECONCILIATION_LAST_RUN.remove(repo);
2632
2633        let backend = Arc::new(ReconciliationMockBackend {
2634            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2635        });
2636        let backend_dyn: Arc<dyn ExternalBackend> =
2637            Arc::<ReconciliationMockBackend>::clone(&backend);
2638        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2639        let task_manager = Arc::new(TaskManager::with_store(
2640            Arc::clone(&backend_dyn),
2641            Arc::clone(&store),
2642            repo.to_string(),
2643        ));
2644        let opts = JanitorOptions {
2645            ttl_hours: 0,
2646            dry_run: true,
2647        };
2648
2649        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2650            .await
2651            .unwrap();
2652        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2653            .await
2654            .unwrap();
2655
2656        assert_eq!(
2657            backend.reconciliation_calls.load(Ordering::SeqCst),
2658            1,
2659            "cleanup should not re-list reconciliation candidates on every sync pass"
2660        );
2661
2662        RECONCILIATION_LAST_RUN.remove(repo);
2663    }
2664
2665    /// Cleanup only collects done tasks. A blocked task waits on a person and resumes from its
2666    /// worktree and branch, so it must not even be considered (#3781)
2667    #[tokio::test]
2668    async fn cleanup_skips_blocked_internal_tasks_and_cleans_done_ones() {
2669        let repo = "owner/repo-cleanup-blocked";
2670        RECONCILIATION_LAST_RUN.remove(repo);
2671        let backend_dyn: Arc<dyn ExternalBackend> = Arc::new(ReconciliationMockBackend {
2672            reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2673        });
2674        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2675        let task_manager = Arc::new(TaskManager::with_store(
2676            Arc::clone(&backend_dyn),
2677            Arc::clone(&store),
2678            repo.to_string(),
2679        ));
2680
2681        let mut ids = Vec::new();
2682        for (title, status) in [("blocked", TaskStatus::Blocked), ("done", TaskStatus::Done)] {
2683            let id = store
2684                .create_internal(repo, title, "", "test", title, None)
2685                .await
2686                .unwrap();
2687            store
2688                .set_fields(
2689                    id,
2690                    &[
2691                        (
2692                            "worktree",
2693                            serde_json::json!(format!("/nonexistent/{title}")),
2694                        ),
2695                        ("branch", serde_json::json!("")),
2696                    ],
2697                )
2698                .await
2699                .unwrap();
2700            store.update_status(id, status).await.unwrap();
2701            ids.push(id);
2702        }
2703
2704        let opts = JanitorOptions {
2705            ttl_hours: 0,
2706            dry_run: false,
2707        };
2708        cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2709            .await
2710            .unwrap();
2711
2712        assert!(
2713            !store.get(ids[0]).await.unwrap().worktree_cleaned,
2714            "blocked task must not be considered for cleanup"
2715        );
2716        assert!(
2717            store.get(ids[1]).await.unwrap().worktree_cleaned,
2718            "done task is cleaned, which proves the pass ran"
2719        );
2720        RECONCILIATION_LAST_RUN.remove(repo);
2721    }
2722
2723    fn git_ok(dir: &Path, args: &[&str]) -> String {
2724        let out = process::Command::new("git")
2725            .args([
2726                "-c",
2727                "commit.gpgsign=false",
2728                "-c",
2729                "user.name=T",
2730                "-c",
2731                "user.email=t@t",
2732            ])
2733            .args(args)
2734            .current_dir(dir)
2735            .output()
2736            .unwrap();
2737        assert!(
2738            out.status.success(),
2739            "git {args:?}: {}",
2740            String::from_utf8_lossy(&out.stderr)
2741        );
2742        String::from_utf8_lossy(&out.stdout).to_string()
2743    }
2744
2745    /// A clone of a bare origin with main pushed and branch `feat` one commit ahead
2746    fn origin_and_clone_with_feat() -> (tempfile::TempDir, PathBuf) {
2747        let tmp = tempfile::tempdir().unwrap();
2748        let bare = tmp.path().join("origin.git");
2749        std_fs::create_dir_all(&bare).unwrap();
2750        git_ok(&bare, &["init", "--bare", "-b", "main"]);
2751        git_ok(tmp.path(), &["clone", bare.to_str().unwrap(), "clone"]);
2752        let clone = tmp.path().join("clone");
2753        std_fs::write(clone.join("base.txt"), "base").unwrap();
2754        git_ok(&clone, &["add", "."]);
2755        git_ok(&clone, &["commit", "-m", "base"]);
2756        git_ok(&clone, &["push", "origin", "main"]);
2757        git_ok(&clone, &["checkout", "-b", "feat"]);
2758        std_fs::write(clone.join("work.txt"), "work").unwrap();
2759        git_ok(&clone, &["add", "."]);
2760        git_ok(&clone, &["commit", "-m", "work"]);
2761        git_ok(&clone, &["checkout", "main"]);
2762        (tmp, clone)
2763    }
2764
2765    fn remote_has(clone: &Path, branch: &str) -> bool {
2766        !git_ok(clone, &["ls-remote", "--heads", "origin", branch])
2767            .trim()
2768            .is_empty()
2769    }
2770
2771    fn local_has(clone: &Path, branch: &str) -> bool {
2772        process::Command::new("git")
2773            .args([
2774                "show-ref",
2775                "--verify",
2776                "--quiet",
2777                &format!("refs/heads/{branch}"),
2778            ])
2779            .current_dir(clone)
2780            .status()
2781            .unwrap()
2782            .success()
2783    }
2784
2785    /// Deleting a pushed branch whose commits main lacks would destroy the only copy of the
2786    /// work, as startup cleanup did to a blocked task (#3781)
2787    #[tokio::test]
2788    async fn delete_branches_keeps_an_unmerged_remote_branch() {
2789        let (_tmp, clone) = origin_and_clone_with_feat();
2790        git_ok(&clone, &["push", "origin", "feat"]);
2791
2792        delete_branches("1", "feat", &clone).await;
2793
2794        assert!(
2795            remote_has(&clone, "feat"),
2796            "unmerged remote branch must survive"
2797        );
2798        assert!(
2799            !local_has(&clone, "feat"),
2800            "local copy is on origin, so it can go"
2801        );
2802    }
2803
2804    #[tokio::test]
2805    async fn delete_branches_leaves_the_screenshots_branch_alone() {
2806        let (_tmp, clone) = origin_and_clone_with_feat();
2807        git_ok(&clone, &["branch", SCREENSHOTS_BRANCH, "feat"]);
2808        git_ok(&clone, &["push", "origin", SCREENSHOTS_BRANCH]);
2809        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2810        git_ok(&clone, &["push", "origin", "main"]);
2811
2812        delete_branches("1", SCREENSHOTS_BRANCH, &clone).await;
2813
2814        assert!(remote_has(&clone, SCREENSHOTS_BRANCH));
2815        assert!(local_has(&clone, SCREENSHOTS_BRANCH));
2816    }
2817
2818    #[tokio::test]
2819    async fn delete_branches_removes_a_branch_main_already_contains() {
2820        let (_tmp, clone) = origin_and_clone_with_feat();
2821        git_ok(&clone, &["push", "origin", "feat"]);
2822        git_ok(&clone, &["merge", "--ff-only", "feat"]);
2823        git_ok(&clone, &["push", "origin", "main"]);
2824
2825        delete_branches("1", "feat", &clone).await;
2826
2827        assert!(!remote_has(&clone, "feat"));
2828        assert!(!local_has(&clone, "feat"));
2829    }
2830
2831    fn add_worktree(clone: &Path, tmp: &Path, branch: &str) -> PathBuf {
2832        let wt = tmp.join("wt");
2833        git_ok(clone, &["worktree", "add", wt.to_str().unwrap(), branch]);
2834        wt
2835    }
2836
2837    async fn store_approved_task(
2838        store: &TaskStore,
2839        repo: &str,
2840        external_id: &str,
2841        title: &str,
2842        body: &str,
2843        worktree: &Path,
2844    ) -> i64 {
2845        let id = store
2846            .create(&NewTask {
2847                external_id: Some(external_id.to_string()),
2848                repo: repo.to_string(),
2849                origin: "github".to_string(),
2850                title: title.to_string(),
2851                body: body.to_string(),
2852                ..Default::default()
2853            })
2854            .await
2855            .unwrap();
2856        store.update_status(id, TaskStatus::Approved).await.unwrap();
2857        store
2858            .set_fields(
2859                id,
2860                &[
2861                    ("branch", serde_json::json!("feat")),
2862                    (
2863                        "worktree",
2864                        serde_json::json!(worktree.to_string_lossy().as_ref()),
2865                    ),
2866                    ("pr_number", serde_json::json!(7)),
2867                ],
2868            )
2869            .await
2870            .unwrap();
2871        id
2872    }
2873
2874    #[tokio::test]
2875    async fn cleanup_pushes_unpushed_commits_to_a_rescue_branch() {
2876        let (tmp, clone) = origin_and_clone_with_feat();
2877        let wt = add_worktree(&clone, tmp.path(), "feat");
2878
2879        let sha = git_ok(&clone, &["rev-parse", "--short", "feat"]);
2880        let sha = sha.trim();
2881        match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2882            Rescue::Pushed { branch, commits } => {
2883                assert_eq!(branch, "rescue/feat");
2884                assert!(rescue_comment(&branch, &commits).contains(sha));
2885            }
2886            other => panic!("expected a rescue push, got {other:?}"),
2887        }
2888
2889        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2890
2891        assert!(removed);
2892        assert!(remote_has(&clone, "rescue/feat"));
2893        assert!(!local_has(&clone, "feat"));
2894    }
2895
2896    #[tokio::test]
2897    async fn cleanup_commits_uncommitted_changes_before_the_rescue() {
2898        let (tmp, clone) = origin_and_clone_with_feat();
2899        git_ok(&clone, &["push", "origin", "feat"]);
2900        let wt = add_worktree(&clone, tmp.path(), "feat");
2901        std_fs::write(wt.join("dirty.txt"), "dirty").unwrap();
2902
2903        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2904
2905        assert!(removed);
2906        assert!(remote_has(&clone, "rescue/feat"));
2907    }
2908
2909    #[tokio::test]
2910    async fn cleanup_makes_no_rescue_branch_for_a_pushed_branch() {
2911        let (tmp, clone) = origin_and_clone_with_feat();
2912        git_ok(&clone, &["push", "origin", "feat"]);
2913        let wt = add_worktree(&clone, tmp.path(), "feat");
2914
2915        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2916
2917        assert!(removed);
2918        assert!(!remote_has(&clone, "rescue/feat"));
2919        assert!(!local_has(&clone, "feat"));
2920    }
2921
2922    #[tokio::test]
2923    async fn rescue_skips_a_branch_on_the_remote_without_a_tracking_ref() {
2924        let (_tmp, clone) = origin_and_clone_with_feat();
2925        git_ok(&clone, &["push", "origin", "feat"]);
2926        git_ok(&clone, &["update-ref", "-d", "refs/remotes/origin/feat"]);
2927
2928        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
2929
2930        assert_eq!(rescue, Rescue::Clean);
2931        assert!(!remote_has(&clone, "rescue/feat"));
2932    }
2933
2934    #[tokio::test]
2935    async fn merged_pr_cleanup_pushes_no_rescue_branch() {
2936        let (tmp, clone) = origin_and_clone_with_feat();
2937        let wt = add_worktree(&clone, tmp.path(), "feat");
2938
2939        let removed =
2940            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, true).await;
2941
2942        assert!(removed);
2943        assert!(!wt.exists());
2944        assert!(!remote_has(&clone, "rescue/feat"));
2945    }
2946
2947    #[tokio::test]
2948    async fn cleanup_skips_rescue_for_an_approved_merged_pr() {
2949        let (tmp, clone) = origin_and_clone_with_feat();
2950        let wt = add_worktree(&clone, tmp.path(), "feat");
2951        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2952        store_approved_task(&store, "owner/repo", "task-12", "CI-01 Set up CI", "", &wt).await;
2953
2954        cleanup_merged_pr_worktree("task-12", "owner/repo", &store)
2955            .await
2956            .unwrap();
2957
2958        let task = store
2959            .get(
2960                store
2961                    .resolve_task_id("owner/repo", "task-12")
2962                    .await
2963                    .unwrap()
2964                    .unwrap(),
2965            )
2966            .await
2967            .unwrap();
2968        assert_eq!(task.status, TaskStatus::Approved);
2969        assert!(task.worktree_cleaned);
2970        assert!(!wt.exists());
2971        assert!(!remote_has(&clone, "rescue/feat"));
2972    }
2973
2974    #[tokio::test]
2975    async fn cleanup_rescues_an_approved_task_when_the_pr_is_not_known_merged() {
2976        let (tmp, clone) = origin_and_clone_with_feat();
2977        let wt = add_worktree(&clone, tmp.path(), "feat");
2978        let store = Arc::new(TaskStore::open_memory().await.unwrap());
2979        store_approved_task(&store, "owner/repo", "task-12", "fix something", "", &wt).await;
2980
2981        cleanup_task_worktree("task-12", "owner/repo", &store)
2982            .await
2983            .unwrap();
2984
2985        assert!(remote_has(&clone, "rescue/feat"));
2986    }
2987
2988    #[tokio::test]
2989    async fn unmerged_cleanup_still_pushes_a_rescue_branch() {
2990        let (tmp, clone) = origin_and_clone_with_feat();
2991        let wt = add_worktree(&clone, tmp.path(), "feat");
2992
2993        let removed =
2994            remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, false).await;
2995
2996        assert!(removed);
2997        assert!(remote_has(&clone, "rescue/feat"));
2998    }
2999
3000    #[tokio::test]
3001    async fn rescue_ignores_a_remote_tip_missing_locally() {
3002        let (tmp, clone) = origin_and_clone_with_feat();
3003        git_ok(&clone, &["push", "origin", "feat"]);
3004        let other = tmp.path().join("other");
3005        git_ok(
3006            tmp.path(),
3007            &["clone", "-b", "feat", "origin.git", other.to_str().unwrap()],
3008        );
3009        std_fs::write(other.join("extra.txt"), "extra").unwrap();
3010        git_ok(&other, &["add", "."]);
3011        git_ok(&other, &["commit", "-m", "extra"]);
3012        git_ok(&other, &["push", "origin", "feat"]);
3013
3014        let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3015
3016        assert_ne!(rescue, Rescue::Failed);
3017    }
3018
3019    #[tokio::test]
3020    async fn cleanup_keeps_worktree_and_branch_when_the_rescue_push_fails() {
3021        let (tmp, clone) = origin_and_clone_with_feat();
3022        let wt = add_worktree(&clone, tmp.path(), "feat");
3023        git_ok(
3024            &clone,
3025            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3026        );
3027
3028        let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3029
3030        assert!(!removed);
3031        assert!(wt.exists());
3032        assert!(local_has(&clone, "feat"));
3033    }
3034
3035    #[tokio::test]
3036    async fn delete_branches_keeps_a_local_branch_with_unpushed_commits() {
3037        let (_tmp, clone) = origin_and_clone_with_feat();
3038
3039        delete_branches("1", "feat", &clone).await;
3040
3041        assert!(
3042            local_has(&clone, "feat"),
3043            "unpushed commits must not be deleted"
3044        );
3045    }
3046}