1use crate::backends::{ExternalBackend, ExternalId, ExternalTask, Status};
8use crate::cmd::CommandErrorContext;
9use crate::engine::ci_bootstrap;
10use crate::engine::tasks::{TaskManager, store_task_to_external};
11use crate::store;
12use crate::store::TaskStatus;
13use crate::store::TaskStore;
14use crate::store::store_log_activity;
15use dashmap::{DashMap, DashSet};
16use futures::future;
17use std::collections::HashMap;
18use std::collections::HashSet;
19use std::env;
20use std::fs as std_fs;
21use std::io::ErrorKind;
22use std::path::Path;
23use std::path::PathBuf;
24use std::process::Output;
25use std::sync::Arc;
26use std::sync::LazyLock;
27use std::time::SystemTime;
28use std::time::{Duration, Instant};
29use tokio::fs;
30use tokio::process::Command;
31use tokio::time::sleep;
32use tokio::time::timeout;
33
34use super::sync::ReviewTaskSnapshot;
35use crate::config;
36use crate::engine;
37use crate::engine::auto_merge;
38use crate::engine::runner::cow;
39use crate::engine::runner::git_ops;
40use crate::engine::runner::screenshots::SCREENSHOTS_BRANCH;
41use crate::engine::runner::worktree;
42use crate::github::http::GhHttp;
43use crate::home;
44
45const RECONCILIATION_LIST_TIMEOUT: Duration = Duration::from_secs(30);
50const RECONCILIATION_INTERVAL: Duration = Duration::from_mins(15);
53
54const RUN_OUTPUT_RETENTION_DAYS: i32 = 30;
55const RUN_PRUNE_INTERVAL_SECS: i64 = 24 * 60 * 60;
56const RUN_PRUNE_KV_KEY: &str = "task_runs_pruned_at";
57
58static CLEANUP_IN_FLIGHT: LazyLock<DashSet<String>> = LazyLock::new(DashSet::new);
61static RECONCILIATION_LAST_RUN: LazyLock<DashMap<String, Instant>> = LazyLock::new(DashMap::new);
63
64struct CleanupInFlightGuard {
65 repo: String,
66}
67
68impl CleanupInFlightGuard {
69 fn try_acquire(repo: &str) -> Option<Self> {
70 if CLEANUP_IN_FLIGHT.insert(repo.to_string()) {
71 Some(Self {
72 repo: repo.to_string(),
73 })
74 } else {
75 None
76 }
77 }
78}
79
80impl Drop for CleanupInFlightGuard {
81 fn drop(&mut self) {
82 CLEANUP_IN_FLIGHT.remove(&self.repo);
83 }
84}
85
86fn should_run_reconciliation(repo: &str) -> bool {
87 let now = Instant::now();
88 if let Some(mut last_run) = RECONCILIATION_LAST_RUN.get_mut(repo) {
89 if last_run.elapsed() < RECONCILIATION_INTERVAL {
90 return false;
91 }
92 *last_run = now;
93 true
94 } else {
95 RECONCILIATION_LAST_RUN.insert(repo.to_string(), now);
96 true
97 }
98}
99
100#[derive(Debug, Clone)]
102pub struct JanitorOptions {
103 pub ttl_hours: u64,
111
112 pub dry_run: bool,
116}
117
118impl Default for JanitorOptions {
119 fn default() -> Self {
120 Self {
121 ttl_hours: 24,
122 dry_run: false,
123 }
124 }
125}
126
127impl JanitorOptions {
128 pub fn from_config() -> Self {
130 let ttl_hours = config::get("workflow.worktree_janitor_ttl_hours")
131 .ok()
132 .and_then(|v| v.parse::<u64>().ok())
133 .unwrap_or(24);
134
135 let dry_run = config::get("workflow.worktree_janitor_dry_run").is_ok_and(|v| v == "true");
136
137 Self { ttl_hours, dry_run }
138 }
139}
140
141pub(crate) async fn cleanup_done_worktrees(
150 backend: &Arc<dyn ExternalBackend>,
151 repo: &str,
152 task_manager: &Arc<TaskManager>,
153 store: &Arc<TaskStore>,
154) -> anyhow::Result<()> {
155 let opts = JanitorOptions::from_config();
156 cleanup_done_worktrees_with_opts(backend, repo, task_manager, store, &opts).await
157}
158
159pub(crate) async fn prune_old_run_output(store: &TaskStore) -> anyhow::Result<()> {
161 let now = chrono::Utc::now().timestamp();
162 let last = store
163 .kv_get(RUN_PRUNE_KV_KEY)
164 .await?
165 .and_then(|v| v.parse::<i64>().ok())
166 .unwrap_or(0);
167 if now - last < RUN_PRUNE_INTERVAL_SECS {
168 return Ok(());
169 }
170 store.kv_set(RUN_PRUNE_KV_KEY, &now.to_string()).await?;
171
172 let pruned = store.prune_old_runs(RUN_OUTPUT_RETENTION_DAYS).await?;
173 if pruned > 0 {
174 tracing::info!(pruned, "cleared the output of old task runs");
175 }
176 Ok(())
177}
178
179async fn reconcile_closed_tasks(
180 task_ids: &mut Vec<String>,
181 all_tasks: &[ExternalTask],
182 task_manager: &TaskManager,
183 store: &Arc<TaskStore>,
184 repo: &str,
185) {
186 const MAX_RECONCILE_PER_CYCLE: usize = 10;
187
188 let done_set: HashSet<String> = task_ids.iter().cloned().collect();
189 let mut closed_not_in_store = Vec::new();
190 for task in all_tasks {
191 if task.state != "closed" || done_set.contains(&task.id.0) {
192 continue;
193 }
194 if ci_bootstrap::hold_after_merge(task, store, repo).await {
195 continue;
196 }
197 closed_not_in_store.push(task);
198 }
199
200 let mut already_labeled: Vec<String> = Vec::new();
203 let mut needs_reconcile: Vec<&ExternalTask> = Vec::new();
204 for task in &closed_not_in_store {
205 if task.labels.iter().any(|l| l == "status:done") {
206 already_labeled.push(task.id.0.clone());
207 } else {
208 needs_reconcile.push(task);
209 }
210 }
211
212 if !already_labeled.is_empty() {
214 tracing::debug!(
215 count = already_labeled.len(),
216 "closed issues already labeled status:done — adding to cleanup set"
217 );
218 task_ids.extend(already_labeled);
219 }
220
221 if needs_reconcile.len() > MAX_RECONCILE_PER_CYCLE {
224 tracing::info!(
225 total = needs_reconcile.len(),
226 batch = MAX_RECONCILE_PER_CYCLE,
227 "capping closed-issue reconciliation to avoid rate-limit exhaustion"
228 );
229 }
230 let mut futs = Vec::new();
231 for task in needs_reconcile.iter().take(MAX_RECONCILE_PER_CYCLE) {
232 let id = task.id.clone();
233 let task_id_str = task.id.0.clone();
234 let labels = task.labels.clone();
235 futs.push(async move {
236 tracing::info!(
237 task_id = task_id_str,
238 labels = ?labels,
239 "closed issue with stale status label — reconciling to done and cleaning up"
240 );
241 match task_manager.update_task_status(&id, Status::Done).await {
242 Ok(()) => Some(task_id_str),
243 Err(e) => {
244 tracing::warn!(
245 task_id = task_id_str,
246 err = %e,
247 "failed to reconcile closed issue status to done"
248 );
249 None
250 }
251 }
252 });
253 }
254 let results = future::join_all(futs).await;
255 task_ids.extend(results.into_iter().flatten());
256}
257
258pub(crate) async fn cleanup_done_worktrees_with_opts(
264 backend: &Arc<dyn ExternalBackend>,
265 repo: &str,
266 task_manager: &Arc<TaskManager>,
267 store: &Arc<TaskStore>,
268 opts: &JanitorOptions,
269) -> anyhow::Result<()> {
270 let Some(_guard) = CleanupInFlightGuard::try_acquire(repo) else {
271 tracing::debug!(
272 repo = %repo,
273 "cleanup already in progress for repo — skipping overlapping run"
274 );
275 return Ok(());
276 };
277
278 let done_tasks = task_manager.list_all_by_status(Status::Done).await?;
280 tracing::debug!(count = done_tasks.len(), "checking done tasks for cleanup");
281
282 let mut task_ids: Vec<String> = done_tasks.iter().map(|t| t.id.0.clone()).collect();
284
285 if should_run_reconciliation(repo) {
297 match timeout(
298 RECONCILIATION_LIST_TIMEOUT,
299 backend.list_reconciliation_candidates(),
300 )
301 .await
302 {
303 Ok(Ok(candidates)) if !candidates.is_empty() => {
304 tracing::debug!(
305 count = candidates.len(),
306 "reconciling closed issues from candidates"
307 );
308 reconcile_closed_tasks(&mut task_ids, &candidates, task_manager, store, repo).await;
309 }
310 Ok(Ok(_)) => {
311 tracing::debug!("no reconciliation candidates found");
312 }
313 Ok(Err(e)) => {
314 tracing::warn!(err = %e, "failed to list reconciliation candidates");
315 }
316 Err(_) => {
317 tracing::warn!(
318 timeout_secs = RECONCILIATION_LIST_TIMEOUT.as_secs(),
319 retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
320 "timed out listing reconciliation candidates"
321 );
322 }
323 }
324 } else {
325 tracing::debug!(
326 retry_after_secs = RECONCILIATION_INTERVAL.as_secs(),
327 "skipping closed-issue reconciliation until interval elapses"
328 );
329 }
330
331 if let Ok(internal_done) = task_manager.list_internal_by_status(TaskStatus::Done).await {
333 task_ids.extend(internal_done.iter().map(|t| t.id.0.clone()));
334 }
335
336 tracing::debug!(
337 count = task_ids.len(),
338 "checking all terminal tasks for cleanup"
339 );
340
341 if let Err(e) = auto_merge::sweep_ci_timer_keys(store, repo, &task_ids).await {
342 tracing::warn!(err = %e, "failed to sweep CI timer keys");
343 }
344
345 let already_cleaned = store.cleaned_external_ids(repo).await.unwrap_or_default();
350
351 let mut cleaned_any = false;
352 for task_id in &task_ids {
353 if already_cleaned.contains(task_id) {
355 continue;
356 }
357
358 match cleanup_task_worktree_with_opts(task_id, repo, store, opts).await {
359 Ok(true) => cleaned_any = true,
360 Ok(false) => {
361 }
364 Err(e) => {
365 tracing::warn!(task_id, err = %e, "worktree cleanup failed for task");
366 }
367 }
368 }
369
370 let worktrees_base = match home::worktrees_dir() {
383 Ok(base) => Some(base),
384 Err(e) => {
385 tracing::debug!(err = %e, "worktrees_dir() unavailable — skipping orphan scan");
386 None
387 }
388 };
389
390 if let Some(base) = worktrees_base {
391 let project = repo
392 .rsplit('/')
393 .next()
394 .unwrap_or(repo)
395 .trim_end_matches(".git");
396 let repo_worktrees = base.join(project);
397
398 let mut combined: HashSet<String> = HashSet::new();
400 if let Ok(tasks) = store.list_all(repo).await {
401 for t in tasks {
402 if !t.worktree.is_empty() {
403 combined.insert(t.worktree);
404 }
405 }
406 }
407
408 if let Ok(project_dir) = resolve_project_dir_for_repo(repo)
410 && let Ok(active_wts) = worktree::list_project_worktrees(&project_dir).await
411 {
412 for wt in active_wts {
413 combined.insert(wt.to_string_lossy().to_string());
414 }
415 }
416
417 if let Ok(mut entries) = fs::read_dir(&repo_worktrees).await {
419 while let Ok(Some(entry)) = entries.next_entry().await {
420 let Ok(entry_meta) = entry.metadata().await else {
421 continue;
422 };
423 if !entry_meta.is_dir() {
424 continue;
425 }
426 let wt_path = entry.path().to_string_lossy().to_string();
427 if combined.contains(&wt_path) {
428 continue;
429 }
430 tracing::info!(worktree = %wt_path, "found orphaned worktree — attempting cleanup");
431 match resolve_repo_root_for_orphaned_worktree(&entry.path()).await {
432 Ok(repo_root) => {
433 let removed = remove_worktree_and_branch(
434 "orphan-scan",
435 &entry.path(),
436 None,
437 Path::new(&repo_root),
438 false,
439 )
440 .await;
441 if removed {
442 tracing::info!(worktree = %wt_path, "orphaned worktree removed");
443 } else {
444 tracing::warn!(worktree = %wt_path, "orphaned worktree could not be removed");
445 }
446 }
447 Err(e) => {
448 tracing::warn!(worktree = %wt_path, err = %e, "could not resolve repo root for orphaned worktree");
449 }
450 }
451 }
452 }
453 }
454
455 if cleaned_any
457 && !opts.dry_run
458 && let Ok(repo_root) = resolve_repo_root(repo).await
459 {
460 const GIT_TIMEOUT: Duration = Duration::from_secs(60);
461
462 let prune_result = timeout(
464 GIT_TIMEOUT,
465 Command::new("git")
466 .args(["-C", &repo_root, "worktree", "prune"])
467 .output_with_context(),
468 )
469 .await;
470 match prune_result {
471 Ok(Ok(output)) if output.status.success() => {
472 tracing::debug!(%repo, "pruned stale worktree metadata");
473 }
474 Ok(Ok(output)) => {
475 let stderr = String::from_utf8_lossy(&output.stderr);
476 tracing::warn!(%repo, err = %stderr, "git worktree prune failed");
477 }
478 Ok(Err(e)) => {
479 tracing::warn!(%repo, err = %e, "git worktree prune failed");
480 }
481 Err(_) => {
482 tracing::warn!(%repo, "git worktree prune timed out after 60s");
483 }
484 }
485
486 let fetch_result = timeout(
491 GIT_TIMEOUT,
492 git_ops::git()
493 .await
494 .args(["-C", &repo_root, "fetch", "--prune", "origin"])
495 .output_with_context(),
496 )
497 .await;
498 match fetch_result {
499 Ok(Ok(output)) if output.status.success() => {
500 tracing::info!(%repo, "fetched origin after worktree cleanup");
501 }
502 Ok(Ok(output)) => {
503 let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
504 tracing::debug!(%repo, err = %stderr, "git fetch skipped after cleanup");
505 }
506 Ok(Err(e)) => {
507 tracing::debug!(%repo, err = %e, "git fetch failed after cleanup");
508 }
509 Err(_) => {
510 tracing::debug!(%repo, "git fetch timed out after 60s");
511 }
512 }
513 }
514
515 Ok(())
516}
517
518pub(crate) async fn cleanup_task_worktree(
528 task_id: &str,
529 repo: &str,
530 store: &Arc<TaskStore>,
531) -> anyhow::Result<()> {
532 cleanup_task_worktree_impl(task_id, repo, store, false).await
533}
534
535pub(crate) async fn cleanup_merged_pr_worktree(
537 task_id: &str,
538 repo: &str,
539 store: &Arc<TaskStore>,
540) -> anyhow::Result<()> {
541 cleanup_task_worktree_impl(task_id, repo, store, true).await
542}
543
544async fn cleanup_task_worktree_impl(
545 task_id: &str,
546 repo: &str,
547 store: &Arc<TaskStore>,
548 known_pr_merged: bool,
549) -> anyhow::Result<()> {
550 let opts = JanitorOptions {
551 ttl_hours: 0,
552 ..Default::default()
553 };
554 let cleaned =
555 cleanup_task_worktree_with_opts_impl(task_id, repo, store, &opts, known_pr_merged).await?;
556
557 if cleaned && let Ok(repo_root) = resolve_repo_root(repo).await {
562 const GIT_TIMEOUT: Duration = Duration::from_secs(60);
563 let fetch_result = timeout(
564 GIT_TIMEOUT,
565 git_ops::git()
566 .await
567 .args(["-C", &repo_root, "fetch", "--prune", "origin"])
568 .output_with_context(),
569 )
570 .await;
571 match fetch_result {
572 Ok(Ok(output)) if output.status.success() => {
573 tracing::debug!(%repo, "fetched origin after task cleanup");
574 }
575 Ok(Ok(output)) => {
576 let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
577 tracing::debug!(%repo, err = %stderr, "git fetch skipped after task cleanup");
578 }
579 Ok(Err(e)) => {
580 tracing::debug!(%repo, err = %e, "git fetch failed after task cleanup");
581 }
582 Err(_) => {
583 tracing::debug!(%repo, "git fetch timed out after 60s");
584 }
585 }
586 }
587
588 Ok(())
589}
590
591pub(crate) async fn cleanup_task_worktree_with_opts(
598 task_id: &str,
599 repo: &str,
600 store: &Arc<TaskStore>,
601 opts: &JanitorOptions,
602) -> anyhow::Result<bool> {
603 cleanup_task_worktree_with_opts_impl(task_id, repo, store, opts, false).await
604}
605
606async fn cleanup_task_worktree_with_opts_impl(
607 task_id: &str,
608 repo: &str,
609 store: &Arc<TaskStore>,
610 opts: &JanitorOptions,
611 known_pr_merged: bool,
612) -> anyhow::Result<bool> {
613 let (worktree, branch, keep_remote_branch, merged, external_id, pr_merged) =
614 store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
615 .await
616 .map_or((None, None, false, false, None, false), |t| {
617 let keep = t.status == TaskStatus::Blocked && t.pr_number.is_some();
620 let merged = t.status == TaskStatus::Done || known_pr_merged;
621 (
622 Some(t.worktree),
623 Some(t.branch),
624 keep,
625 merged,
626 t.external_id,
627 known_pr_merged || (t.status == TaskStatus::Done && t.pr_number.is_some()),
629 )
630 });
631
632 let worktree_path = worktree.as_ref().map(PathBuf::from);
633
634 let worktrees_base = home::worktrees_dir().unwrap_or_else(|_| PathBuf::from(".orch/worktrees"));
636
637 let worktree_to_remove = if let Some(ref wt) = worktree_path {
643 if match fs::try_exists(wt).await {
644 Ok(true) => true,
645 Ok(false) => false,
646 Err(e) => {
647 tracing::warn!(task_id, worktree = %wt.display(), err = %e, "try_exists failed on stored worktree path — trying branch-based fallback");
648 false
649 }
650 } {
651 Some(wt.clone())
652 } else {
653 if let Some(ref b) = branch {
655 let project = repo
656 .rsplit('/')
657 .next()
658 .unwrap_or(repo)
659 .trim_end_matches(".git");
660 let candidate = worktrees_base.join(project).join(b);
661 if match fs::try_exists(&candidate).await {
662 Ok(true) => true,
663 Ok(false) => false,
664 Err(e) => {
665 tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
666 false
667 }
668 } {
669 Some(candidate)
670 } else {
671 None
672 }
673 } else {
674 None
675 }
676 }
677 } else if let Some(ref b) = branch {
678 let project = repo
680 .rsplit('/')
681 .next()
682 .unwrap_or(repo)
683 .trim_end_matches(".git");
684 let candidate = worktrees_base.join(project).join(b);
685 if match fs::try_exists(&candidate).await {
686 Ok(true) => true,
687 Ok(false) => false,
688 Err(e) => {
689 tracing::warn!(task_id, err = %e, "try_exists failed — assuming candidate does not exist");
690 false
691 }
692 } {
693 Some(candidate)
694 } else {
695 None
696 }
697 } else {
698 None
699 };
700
701 let mut did_clean = false;
702
703 let branch_nonempty = branch.as_ref().filter(|b| !b.is_empty());
709 if worktree_to_remove.is_none() && branch_nonempty.is_none() {
710 if worktree_path
713 .as_ref()
714 .is_some_and(|p| !p.as_os_str().is_empty())
715 {
716 tracing::debug!(
717 task_id,
718 worktree = ?worktree_path,
719 "stored worktree path no longer exists on disk — marking cleaned"
720 );
721 if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
722 && let Err(e) = store.mark_cleaned(store_id).await
723 {
724 tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned in store — will retry");
725 }
726 }
727 return Ok(false);
728 }
729
730 if let Some(wt) = worktree_to_remove {
731 if let Some(age_hours) = worktree_age_hours(&wt).await
733 && age_hours < opts.ttl_hours
734 {
735 tracing::debug!(
736 task_id,
737 worktree = %wt.display(),
738 age_hours,
739 ttl_hours = opts.ttl_hours,
740 "worktree too young for cleanup, skipping"
741 );
742 return Ok(false);
743 }
744
745 if is_worktree_in_active_session(&wt).await {
747 tracing::warn!(
748 task_id,
749 worktree = %wt.display(),
750 "worktree is referenced by an active tmux session — skipping cleanup"
751 );
752 return Ok(false);
753 }
754
755 let gitdir_valid = worktree::validate_worktree_gitdir(&wt).await;
759
760 if opts.dry_run {
761 tracing::info!(
762 task_id,
763 worktree = %wt.display(),
764 branch = ?branch,
765 gitdir_valid,
766 "[dry-run] would remove worktree and branch"
767 );
768 } else if !gitdir_valid {
769 tracing::warn!(
770 task_id,
771 worktree = %wt.display(),
772 "worktree has broken git metadata — force-removing directory without git operations"
773 );
774 match fs::remove_dir_all(&wt).await {
775 Ok(()) => {
776 did_clean = true;
777 }
778 Err(e) => {
779 tracing::warn!(
780 task_id,
781 worktree = %wt.display(),
782 err = %e,
783 "failed to force-remove broken worktree directory"
784 );
785 }
786 }
787 } else {
788 tracing::info!(task_id, worktree = %wt.display(), "removing worktree");
789 let repo_root = match resolve_repo_root_from_worktree(&wt).await {
794 Ok(root) => root,
795 Err(_) => resolve_repo_root(repo).await?,
796 };
797 if let Some(br) = branch.as_deref().filter(|b| !b.is_empty())
798 && !pr_merged
799 && !rescue_and_notify(
800 task_id,
801 repo,
802 external_id.as_deref(),
803 Some(&wt),
804 br,
805 Path::new(&repo_root),
806 )
807 .await
808 {
809 return Ok(false);
810 }
811 if merged {
812 cow::warm_from_merged(
813 Path::new(&repo_root),
814 &worktree::project_name(Path::new(&repo_root)),
815 &wt,
816 )
817 .await;
818 }
819 let removed = remove_worktree_and_branch_unchecked(
820 task_id,
821 &wt,
822 branch.as_deref(),
823 Path::new(&repo_root),
824 keep_remote_branch,
825 )
826 .await;
827 if removed {
828 did_clean = true;
829 } else {
830 tracing::warn!(
831 task_id,
832 worktree = %wt.display(),
833 "worktree directory still exists after removal attempt — not marking as cleaned"
834 );
835 }
836 }
837 } else if let Some(ref br) = branch {
838 if !opts.dry_run && !keep_remote_branch {
841 tracing::debug!(task_id, branch = %br, "no worktree on disk, cleaning up branch only");
842 let repo_root = resolve_repo_root(repo).await?;
844 if !pr_merged
845 && !rescue_and_notify(
846 task_id,
847 repo,
848 external_id.as_deref(),
849 None,
850 br,
851 Path::new(&repo_root),
852 )
853 .await
854 {
855 return Ok(false);
856 }
857 delete_branches(task_id, br, Path::new(&repo_root)).await;
858 did_clean = true;
859 }
860 }
861
862 if did_clean {
863 if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
867 && let Err(e) = store.mark_cleaned(store_id).await
868 {
869 tracing::warn!(task_id, err = %e, "failed to mark worktree cleaned — will retry");
870 return Ok(did_clean);
872 }
873 store_log_activity(
874 &Some(Arc::clone(store)),
875 repo,
876 task_id,
877 "branch_delete",
878 None,
879 None,
880 None,
881 None,
882 Some(&serde_json::json!({
883 "worktree": worktree_path.map(|p| p.display().to_string()),
884 "branch": branch,
885 "keep_remote_branch": keep_remote_branch,
886 })),
887 )
888 .await;
889 }
890
891 Ok(did_clean)
892}
893
894#[derive(Debug, PartialEq, Eq)]
896pub(crate) enum Rescue {
897 Clean,
899 Pushed {
901 branch: String,
902 commits: Vec<String>,
903 },
904 Failed,
906}
907
908async fn git_in(dir: &Path, args: &[&str]) -> Option<Output> {
909 git_ops::git()
910 .await
911 .arg("-C")
912 .arg(dir)
913 .args(args)
914 .output_with_context()
915 .await
916 .ok()
917 .map(git_ops::sanitize_git_output)
918}
919
920pub(crate) async fn rescue_unpushed_work(
926 task_id: &str,
927 wt: Option<&Path>,
928 br: &str,
929 repo_root: &Path,
930) -> Rescue {
931 if let Some(wt) = wt
932 && fs::try_exists(wt.join(".git")).await.unwrap_or(false)
933 {
934 let Some(status) = git_in(wt, &["status", "--porcelain"]).await else {
935 tracing::warn!(task_id, branch = %br, "cannot read worktree status, keeping worktree");
936 return Rescue::Failed;
937 };
938 if !status.status.success() {
939 tracing::warn!(task_id, branch = %br, "git status failed, keeping worktree");
940 return Rescue::Failed;
941 }
942 if !status.stdout.is_empty() {
943 let committed = git_in(wt, &["add", "-A"])
944 .await
945 .is_some_and(|o| o.status.success())
946 && git_in(
947 wt,
948 &[
949 "-c",
950 "user.name=orch",
951 "-c",
952 "user.email=orch@localhost",
953 "commit",
954 "-m",
955 "rescue: uncommitted changes saved by orch cleanup",
956 ],
957 )
958 .await
959 .is_some_and(|o| o.status.success());
960 if !committed {
961 tracing::warn!(task_id, branch = %br, "cannot commit uncommitted changes, keeping worktree");
962 return Rescue::Failed;
963 }
964 }
965 }
966
967 let local_ref = format!("refs/heads/{br}");
968 let exists = git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
969 .await
970 .is_some_and(|o| o.status.success());
971 if !exists {
972 return Rescue::Clean;
973 }
974 let default_ref = format!(
976 "refs/heads/{}",
977 worktree::detect_default_branch(repo_root).await
978 );
979 let remote_tip = git_in(repo_root, &["ls-remote", "origin", &local_ref])
981 .await
982 .filter(|o| o.status.success())
983 .and_then(|o| {
984 String::from_utf8_lossy(&o.stdout)
985 .split_whitespace()
986 .next()
987 .map(str::to_string)
988 });
989 let remote_tip = match remote_tip {
991 Some(tip) => {
992 let known = git_in(repo_root, &["cat-file", "-e", &format!("{tip}^{{commit}}")])
993 .await
994 .is_some_and(|o| o.status.success());
995 known.then_some(tip)
996 }
997 None => None,
998 };
999 let mut args = vec!["rev-list", "--oneline", &local_ref, "--not", "--remotes"];
1000 if let Some(tip) = remote_tip.as_deref() {
1001 args.push(tip);
1002 }
1003 let default_exists = default_ref != local_ref
1004 && git_in(
1005 repo_root,
1006 &["rev-parse", "--verify", "--quiet", &default_ref],
1007 )
1008 .await
1009 .is_some_and(|o| o.status.success());
1010 if default_exists {
1011 args.push(&default_ref);
1012 }
1013 let Some(list) = git_in(repo_root, &args)
1014 .await
1015 .filter(|o| o.status.success())
1016 else {
1017 tracing::warn!(task_id, branch = %br, "cannot list unpushed commits, keeping worktree and branch");
1018 return Rescue::Failed;
1019 };
1020 let commits: Vec<String> = String::from_utf8_lossy(&list.stdout)
1021 .lines()
1022 .map(str::to_string)
1023 .collect();
1024 if commits.is_empty() {
1025 return Rescue::Clean;
1026 }
1027
1028 let rescue = format!("rescue/{br}");
1029 let refspec = format!("{local_ref}:refs/heads/{rescue}");
1030 let pushed = git_in(
1031 repo_root,
1032 &["-c", "lfs.locksverify=false", "push", "origin", &refspec],
1033 )
1034 .await;
1035 if pushed.is_some_and(|o| o.status.success()) {
1036 let tracking = format!("refs/remotes/origin/{rescue}");
1039 if let Some(tip) = git_in(repo_root, &["rev-parse", &local_ref])
1040 .await
1041 .filter(|o| o.status.success())
1042 {
1043 let sha = String::from_utf8_lossy(&tip.stdout).trim().to_string();
1044 if !sha.is_empty() {
1045 let _ = git_in(repo_root, &["update-ref", &tracking, &sha]).await;
1046 }
1047 }
1048 tracing::warn!(task_id, branch = %br, rescue = %rescue, commits = commits.len(), "pushed unpushed commits to rescue branch");
1049 Rescue::Pushed {
1050 branch: rescue,
1051 commits,
1052 }
1053 } else {
1054 tracing::warn!(task_id, branch = %br, rescue = %rescue, "rescue push failed, keeping worktree and branch");
1055 Rescue::Failed
1056 }
1057}
1058
1059fn rescue_comment(rescue_branch: &str, commits: &[String]) -> String {
1060 let list = commits
1061 .iter()
1062 .map(|c| format!("- `{c}`"))
1063 .collect::<Vec<_>>()
1064 .join("\n");
1065 format!(
1066 "Cleanup found commits on this task's branch that were never pushed. They are saved on `{rescue_branch}`:\n\n{list}"
1067 )
1068}
1069
1070async fn rescue_and_notify(
1072 task_id: &str,
1073 repo: &str,
1074 external_id: Option<&str>,
1075 wt: Option<&Path>,
1076 br: &str,
1077 repo_root: &Path,
1078) -> bool {
1079 match rescue_unpushed_work(task_id, wt, br, repo_root).await {
1080 Rescue::Clean => true,
1081 Rescue::Failed => false,
1082 Rescue::Pushed { branch, commits } => {
1083 if let Some(number) = external_id.filter(|n| n.parse::<u64>().is_ok()) {
1084 match GhHttp::new() {
1085 Ok(gh) => {
1086 if let Err(e) = gh
1087 .add_comment(repo, number, &rescue_comment(&branch, &commits))
1088 .await
1089 {
1090 tracing::warn!(task_id, err = %e, "failed to comment on rescue branch");
1091 }
1092 }
1093 Err(e) => {
1094 tracing::warn!(task_id, err = %e, "cannot comment on rescue branch");
1095 }
1096 }
1097 }
1098 true
1099 }
1100 }
1101}
1102
1103pub(crate) async fn remove_worktree_and_branch(
1110 task_id: &str,
1111 wt: &Path,
1112 branch: Option<&str>,
1113 repo_root: &Path,
1114 keep_remote_branch: bool,
1115) -> bool {
1116 remove_worktree_and_branch_for(task_id, wt, branch, repo_root, keep_remote_branch, false).await
1117}
1118
1119pub(crate) async fn remove_worktree_and_branch_for(
1122 task_id: &str,
1123 wt: &Path,
1124 branch: Option<&str>,
1125 repo_root: &Path,
1126 keep_remote_branch: bool,
1127 pr_merged: bool,
1128) -> bool {
1129 if !pr_merged
1130 && let Some(br) = branch.filter(|b| !b.is_empty())
1131 && rescue_unpushed_work(task_id, Some(wt), br, repo_root).await == Rescue::Failed
1132 {
1133 return false;
1134 }
1135
1136 remove_worktree_and_branch_unchecked(task_id, wt, branch, repo_root, keep_remote_branch).await
1137}
1138
1139async fn remove_worktree_and_branch_unchecked(
1141 task_id: &str,
1142 wt: &Path,
1143 branch: Option<&str>,
1144 repo_root: &Path,
1145 keep_remote_branch: bool,
1146) -> bool {
1147 let wt_str = wt.to_string_lossy().to_string();
1148 let repo_root_str = repo_root.to_string_lossy();
1149
1150 let remove_result = Command::new("git")
1153 .args([
1154 "-C",
1155 repo_root_str.as_ref(),
1156 "worktree",
1157 "remove",
1158 &wt_str,
1159 "--force",
1160 ])
1161 .output_with_context()
1162 .await;
1163
1164 match remove_result {
1165 Ok(output) if output.status.success() => {
1166 const MAX_RETRIES: usize = 3;
1167 const RETRY_DELAY_MS: u64 = 2000;
1168 tracing::info!(task_id, "worktree removed (git metadata)");
1169 let mut removed = false;
1174 for attempt in 0..MAX_RETRIES {
1175 match fs::remove_dir_all(wt).await {
1176 Ok(()) => {
1177 removed = true;
1178 break;
1179 }
1180 Err(e) if e.kind() == ErrorKind::NotFound => {
1181 removed = true;
1183 break;
1184 }
1185 Err(e) if e.kind() == ErrorKind::DirectoryNotEmpty => {
1186 if attempt < MAX_RETRIES - 1 {
1187 tracing::debug!(
1188 task_id,
1189 path = %wt.display(),
1190 attempt = attempt + 1,
1191 "worktree directory not empty — retrying after delay"
1192 );
1193 sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
1194 } else {
1195 tracing::warn!(
1196 task_id,
1197 path = %wt.display(),
1198 err = %e,
1199 "failed to remove worktree directory after {} attempts (Directory not empty)",
1200 MAX_RETRIES
1201 );
1202 }
1203 }
1204 Err(e) => {
1205 tracing::warn!(task_id, path = %wt.display(), err = %e, "failed to remove worktree directory");
1206 break;
1207 }
1208 }
1209 }
1210 if removed {
1211 tracing::info!(task_id, path = %wt.display(), "worktree physical directory removed");
1212 } else {
1213 tracing::warn!(
1214 task_id,
1215 path = %wt.display(),
1216 "worktree directory still present after git worktree remove"
1217 );
1218 }
1219 }
1220 Ok(output) => {
1221 let stderr = String::from_utf8_lossy(&output.stderr);
1222 if stderr.contains("is not a working tree") {
1228 tracing::debug!(
1229 task_id,
1230 path = %wt.display(),
1231 "path is not a registered worktree (stale metadata) — removing directory and pruning"
1232 );
1233 if let Err(e) = fs::remove_dir_all(wt).await {
1234 if match fs::try_exists(wt).await {
1236 Ok(true) => true,
1237 Ok(false) => false,
1238 Err(e) => {
1239 tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree does not exist");
1240 false
1241 }
1242 } {
1243 tracing::warn!(
1244 task_id,
1245 path = %wt.display(),
1246 err = %e,
1247 "failed to remove stale worktree directory"
1248 );
1249 }
1250 } else {
1251 tracing::debug!(task_id, path = %wt.display(), "stale worktree directory removed");
1252 }
1253 if let Err(e) = Command::new("git")
1256 .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1257 .output_with_context()
1258 .await
1259 {
1260 tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1261 }
1262 } else {
1263 tracing::warn!(task_id, err = %stderr, "failed to remove worktree");
1264 }
1265 }
1266 Err(e) => {
1267 tracing::warn!(task_id, err = %e, "failed to remove worktree");
1268 }
1269 }
1270
1271 if let Err(e) = Command::new("git")
1273 .args(["-C", repo_root_str.as_ref(), "worktree", "prune"])
1274 .output_with_context()
1275 .await
1276 {
1277 tracing::warn!(task_id, err = %e, "git worktree prune failed — stale metadata may persist");
1278 }
1279
1280 if let Some(br) = branch {
1284 if keep_remote_branch {
1285 delete_local_branch(task_id, br, repo_root).await;
1286 } else {
1287 delete_branches(task_id, br, repo_root).await;
1288 }
1289 }
1290
1291 match fs::try_exists(wt).await {
1293 Ok(false) => true, Ok(true) => false, Err(e) => {
1296 tracing::warn!(task_id, err = %e, "try_exists failed — assuming worktree still present");
1297 false }
1299 }
1300}
1301
1302pub(crate) async fn local_branch_exists(repo_root: &Path, br: &str) -> bool {
1304 let local_ref = format!("refs/heads/{br}");
1305 git_in(repo_root, &["rev-parse", "--verify", "--quiet", &local_ref])
1306 .await
1307 .is_some_and(|o| o.status.success())
1308}
1309
1310async fn commits_outside(repo_root: &Path, rev: &str, others: &[&str]) -> Option<u64> {
1312 let mut args = vec![
1313 "rev-list".to_string(),
1314 "--count".to_string(),
1315 rev.to_string(),
1316 ];
1317 let mut exclude = Vec::new();
1319 for other in others {
1320 let exists = Command::new("git")
1321 .args([
1322 "-C",
1323 &repo_root.to_string_lossy(),
1324 "rev-parse",
1325 "--verify",
1326 "--quiet",
1327 other,
1328 ])
1329 .output_with_context()
1330 .await
1331 .is_ok_and(|o| o.status.success());
1332 if exists {
1333 exclude.push(other.to_string());
1334 }
1335 }
1336 if !exclude.is_empty() {
1337 args.push("--not".to_string());
1338 args.extend(exclude);
1339 }
1340 let output = Command::new("git")
1341 .arg("-C")
1342 .arg(repo_root)
1343 .args(&args)
1344 .output_with_context()
1345 .await
1346 .ok()?;
1347 if !output.status.success() {
1348 return None;
1349 }
1350 String::from_utf8_lossy(&output.stdout).trim().parse().ok()
1351}
1352
1353async fn delete_local_branch(task_id: &str, br: &str, repo_root: &Path) {
1355 let default_branch = worktree::detect_default_branch(repo_root).await;
1356 let remote_ref = format!("refs/remotes/origin/{br}");
1357 let default_ref = format!("refs/remotes/origin/{default_branch}");
1358 let local_ref = format!("refs/heads/{br}");
1359 let local_default_ref = format!("refs/heads/{default_branch}");
1360 let rescue_ref = format!("refs/remotes/origin/rescue/{br}");
1361 match commits_outside(
1362 repo_root,
1363 &local_ref,
1364 &[&remote_ref, &default_ref, &local_default_ref, &rescue_ref],
1365 )
1366 .await
1367 {
1368 Some(0) => {}
1369 Some(n) => {
1370 tracing::warn!(task_id, branch = %br, unpushed = n, "local branch holds commits origin does not have, keeping it");
1371 return;
1372 }
1373 None => {
1374 tracing::debug!(task_id, branch = %br, "local branch missing or unreadable, nothing to delete");
1375 return;
1376 }
1377 }
1378 let output = Command::new("git")
1379 .args(["-C", &repo_root.to_string_lossy(), "branch", "-D", br])
1380 .output_with_context()
1381 .await;
1382 match output {
1383 Ok(o) if o.status.success() => {
1384 tracing::info!(task_id, branch = %br, "local branch deleted");
1385 }
1386 Ok(o) => {
1387 let stderr = String::from_utf8_lossy(&o.stderr);
1388 tracing::debug!(task_id, err = %stderr, "local branch delete skipped (may not exist)");
1389 }
1390 Err(e) => {
1391 tracing::warn!(task_id, err = %e, "failed to delete local branch");
1392 }
1393 }
1394}
1395
1396pub(crate) async fn delete_branches(task_id: &str, br: &str, repo_root: &Path) {
1404 if br == SCREENSHOTS_BRANCH {
1405 return;
1406 }
1407 let default_branch = worktree::detect_default_branch(repo_root).await;
1408 let _ = git_ops::git()
1409 .await
1410 .args([
1411 "-C",
1412 &repo_root.to_string_lossy(),
1413 "fetch",
1414 "origin",
1415 &default_branch,
1416 br,
1417 ])
1418 .output_with_context()
1419 .await;
1420
1421 delete_local_branch(task_id, br, repo_root).await;
1422
1423 let remote_ref = format!("refs/remotes/origin/{br}");
1424 let default_ref = format!("refs/remotes/origin/{default_branch}");
1425 match commits_outside(repo_root, &remote_ref, &[&default_ref]).await {
1426 Some(0) => {}
1427 Some(n) => {
1428 tracing::info!(task_id, branch = %br, unmerged = n, "remote branch has commits the default branch lacks, keeping it");
1429 return;
1430 }
1431 None => {
1432 tracing::debug!(task_id, branch = %br, "remote branch missing or unreadable, nothing to delete");
1433 return;
1434 }
1435 }
1436
1437 let remote_delete = git_ops::git()
1438 .await
1439 .args([
1440 "-C",
1441 &repo_root.to_string_lossy(),
1442 "-c",
1443 "lfs.locksverify=false",
1444 "push",
1445 "origin",
1446 "--delete",
1447 br,
1448 ])
1449 .output_with_context()
1450 .await;
1451
1452 match remote_delete {
1453 Ok(output) if output.status.success() => {
1454 tracing::info!(task_id, branch = %br, "remote branch deleted");
1455 git_ops::clear_push_lease(repo_root, br).await;
1456 }
1457 Ok(output) => {
1458 let stderr = git_ops::sanitize_push_error(&String::from_utf8_lossy(&output.stderr));
1459 tracing::debug!(task_id, err = %stderr, "remote branch delete skipped");
1460 }
1461 Err(e) => {
1462 tracing::warn!(task_id, err = %e, "failed to delete remote branch");
1463 }
1464 }
1465}
1466
1467async fn is_worktree_in_active_session(worktree: &Path) -> bool {
1473 let output = Command::new("tmux")
1474 .args(["list-panes", "-a", "-F", "#{pane_current_path}"])
1475 .output()
1476 .await;
1477
1478 match output {
1479 Ok(o) if o.status.success() => {
1480 let worktree_str = worktree.to_string_lossy();
1481 String::from_utf8_lossy(&o.stdout).lines().any(|line| {
1482 let pane_path = line.trim();
1483 pane_path == worktree_str.as_ref()
1485 || pane_path.starts_with(&format!("{worktree_str}/"))
1486 })
1487 }
1488 _ => false,
1490 }
1491}
1492
1493async fn worktree_age_hours(worktree: &Path) -> Option<u64> {
1497 let metadata = fs::metadata(worktree).await.ok()?;
1499 let modified = metadata.modified().ok()?;
1500 let age = SystemTime::now().duration_since(modified).ok()?;
1501 Some(age.as_secs() / 3600)
1502}
1503
1504pub(crate) async fn cleanup_orphaned_worktree(task_id: &str, wt: &Path) -> bool {
1510 let repo_root = match resolve_repo_root_for_orphaned_worktree(wt).await {
1511 Ok(root) => root,
1512 Err(e) => {
1513 tracing::warn!(
1514 worktree = %wt.display(),
1515 error = %e,
1516 "cannot resolve repo root for orphaned worktree"
1517 );
1518 return false;
1519 }
1520 };
1521 remove_worktree_and_branch(task_id, wt, None, Path::new(&repo_root), false).await
1522}
1523
1524pub(crate) async fn resolve_repo_root_for_orphaned_worktree(wt: &Path) -> anyhow::Result<String> {
1531 let git_file = wt.join(".git");
1532 if fs::try_exists(&git_file).await.unwrap_or(false)
1533 && let Ok(content) = fs::read_to_string(&git_file).await
1534 {
1535 for line in content.lines() {
1536 if let Some(gitdir) = line.strip_prefix("gitdir: ") {
1537 let gitdir_path = PathBuf::from(gitdir);
1541 if let Some(repo_root) = gitdir_path.parent().and_then(|p| p.parent()) {
1542 return Ok(repo_root.display().to_string());
1543 }
1544 }
1545 }
1546 }
1547
1548 let worktrees_dir = home::worktrees_dir()?;
1551 if let Ok(relative) = wt.strip_prefix(&worktrees_dir)
1552 && let Some(project_name) = relative.components().next()
1553 {
1554 let project_name = project_name.as_os_str().to_string_lossy();
1555
1556 if let Ok(paths) = config::get_project_paths() {
1557 for path_str in &paths {
1558 let path = Path::new(path_str);
1559 if let Some(name) = path.file_name()
1560 && name.to_string_lossy() == project_name
1561 {
1562 return Ok(path_str.clone());
1563 }
1564 }
1565 }
1566
1567 let parts: Vec<&str> = project_name.split('/').collect();
1569 let bare = if parts.len() == 2 {
1570 home::projects_dir()
1571 .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1572 .unwrap_or_default()
1573 } else {
1574 return Err(anyhow::anyhow!(
1575 "cannot determine owner for single-segment project name '{project_name}' — \
1576 bare repo location is ambiguous; register the project with 'owner/repo' format"
1577 ));
1578 };
1579 if fs::try_exists(&bare).await.unwrap_or(false) {
1580 return Ok(bare.display().to_string());
1581 }
1582 }
1583
1584 anyhow::bail!(
1585 "cannot resolve repo root for orphaned worktree at {} — tried .git parsing and project lookup",
1586 wt.display()
1587 )
1588}
1589
1590pub async fn resolve_repo_root_from_worktree(wt: &Path) -> anyhow::Result<String> {
1597 let git_file = wt.join(".git");
1599 if !fs::try_exists(&git_file).await.unwrap_or(false) {
1600 anyhow::bail!(".git file not found in worktree at {}", wt.display());
1601 }
1602
1603 let content = fs::read_to_string(&git_file)
1604 .await
1605 .map_err(|e| anyhow::anyhow!("failed to read .git file: {e}"))?;
1606
1607 for line in content.lines() {
1609 if let Some(gitdir) = line.strip_prefix("gitdir:") {
1610 let gitdir = gitdir.trim();
1611 let gitdir_path = Path::new(gitdir);
1615 if let Some(repo_root) = gitdir_path
1616 .parent() .and_then(|p| p.parent()) .and_then(|p| p.parent())
1619 {
1621 return Ok(repo_root.display().to_string());
1622 }
1623 }
1624 }
1625
1626 anyhow::bail!(
1627 "cannot resolve repo root from worktree .git file at {}",
1628 wt.display()
1629 )
1630}
1631
1632pub(crate) async fn resolve_repo_root(repo: &str) -> anyhow::Result<String> {
1637 let paths = config::get_project_paths().unwrap_or_default();
1639 for path_str in &paths {
1640 let path = Path::new(path_str);
1641 let orch_yml = path.join(".orch.yml");
1642 let legacy = path.join(".orchestrator.yml");
1643 let config_file = if fs::try_exists(&orch_yml).await.unwrap_or(false) {
1644 orch_yml
1645 } else if fs::try_exists(&legacy).await.unwrap_or(false) {
1646 legacy
1647 } else {
1648 continue;
1649 };
1650 if let Ok(content) = fs::read_to_string(&config_file).await
1651 && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1652 && let Some(r) = doc
1653 .get("gh")
1654 .and_then(|gh| gh.get("repo"))
1655 .and_then(|r| r.as_str())
1656 && r == repo
1657 {
1658 return Ok(path_str.clone());
1659 }
1660 }
1661
1662 let parts: Vec<&str> = repo.split('/').collect();
1664 let bare = if parts.len() == 2 {
1665 home::projects_dir()
1666 .map(|d| d.join(parts[0]).join(format!("{}.git", parts[1])))
1667 .unwrap_or_default()
1668 } else {
1669 PathBuf::new()
1670 };
1671 if fs::try_exists(&bare).await.unwrap_or(false) {
1672 return Ok(bare.display().to_string());
1673 }
1674
1675 anyhow::bail!(
1676 "cannot find local path for project {repo} — \
1677 checked {} registered project(s) and bare clone at {}",
1678 paths.len(),
1679 bare.display()
1680 )
1681}
1682
1683fn resolve_project_dir_for_repo(repo: &str) -> anyhow::Result<PathBuf> {
1688 if let Ok(dir) = env::var("PROJECT_DIR")
1690 && !dir.is_empty()
1691 {
1692 return Ok(PathBuf::from(dir));
1693 }
1694
1695 if let Ok(paths) = config::get_project_paths() {
1697 for path_str in &paths {
1698 let path = Path::new(path_str);
1699 let orch_yml = path.join(".orch.yml");
1700 let legacy = path.join(".orchestrator.yml");
1701 let config_file = if Path::new(&orch_yml).exists() {
1702 orch_yml
1703 } else if Path::new(&legacy).exists() {
1704 legacy
1705 } else {
1706 continue;
1707 };
1708 if let Ok(content) = std_fs::read_to_string(&config_file)
1709 && let Ok(doc) = serde_norway::from_str::<serde_norway::Value>(&content)
1710 && let Some(r) = doc
1711 .get("gh")
1712 .and_then(|gh| gh.get("repo"))
1713 .and_then(|r| r.as_str())
1714 && r == repo
1715 {
1716 return Ok(path_str.clone().into());
1717 }
1718 }
1719 }
1720
1721 anyhow::bail!("project dir not found for repo: {repo}")
1722}
1723
1724pub(crate) async fn check_merged_prs(
1729 backend: &Arc<dyn ExternalBackend>,
1730 repo: &str,
1731 store: &Arc<TaskStore>,
1732 task_manager: &Arc<TaskManager>,
1733 in_review_tasks: &[ReviewTaskSnapshot],
1734 needs_review_tasks: &[ReviewTaskSnapshot],
1735) -> anyhow::Result<()> {
1736 let mut branch_map: HashMap<String, String> = HashMap::new();
1737 let all_review_tasks: Vec<&ReviewTaskSnapshot> = in_review_tasks
1738 .iter()
1739 .chain(needs_review_tasks.iter())
1740 .collect();
1741
1742 for task in &all_review_tasks {
1743 if !task.stored.branch.is_empty() {
1744 branch_map.insert(task.external.id.0.clone(), task.stored.branch.clone());
1745 }
1746 }
1747
1748 tracing::debug!(
1749 count = all_review_tasks.len(),
1750 "checking review tasks for merged PRs"
1751 );
1752
1753 let mut task_branches: Vec<(String, String)> = Vec::new();
1757 for task in all_review_tasks {
1758 let task_id = &task.external.id.0;
1759 let branch = if let Some(b) = branch_map.get(task_id) {
1760 b.clone()
1761 } else {
1762 match store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
1763 .await
1764 .map(|t| t.branch)
1765 {
1766 Some(b) if !b.is_empty() => b,
1767 _ => {
1768 tracing::debug!(task_id, "no branch info, skipping PR check");
1769 continue;
1770 }
1771 }
1772 };
1773 task_branches.push((task_id.clone(), branch));
1774 }
1775
1776 if task_branches.is_empty() {
1777 return Ok(());
1778 }
1779
1780 let branches: Vec<String> = task_branches.iter().map(|(_, b)| b.clone()).collect();
1782 let merged_map = match backend.batch_is_pr_merged(&branches).await {
1783 Ok(map) => map,
1784 Err(e) => {
1785 tracing::warn!(
1786 err = %e,
1787 branch_count = branches.len(),
1788 "batch PR merge check failed, retrying affected branches individually"
1789 );
1790 fallback_is_pr_merged_by_branch(backend, &branches).await
1791 }
1792 };
1793
1794 for (task_id, branch) in task_branches {
1795 let is_merged = merged_map.get(&branch).copied().unwrap_or(false);
1796 if !is_merged {
1797 continue;
1798 }
1799
1800 tracing::info!(task_id, branch = %branch, "PR merged, marking task complete");
1801
1802 let stored = store::opt_store_get_task(&Some(Arc::clone(store)), repo, &task_id).await;
1803 if let Some(ref stored) = stored {
1804 let external = store_task_to_external(stored);
1805 if ci_bootstrap::hold_after_merge(&external, store, repo).await {
1806 tracing::info!(
1807 task_id,
1808 branch = %branch,
1809 "CI bootstrap PR merged, holding done until the ruleset exists"
1810 );
1811 if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1812 tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1813 }
1814 continue;
1815 }
1816 }
1817
1818 let id = ExternalId(task_id.clone());
1819 if let Err(e) = task_manager.update_task_status(&id, Status::Done).await {
1820 tracing::warn!(task_id, err = %e, "failed to update status to done");
1821 continue;
1822 }
1823
1824 if let Err(e) = cleanup_merged_pr_worktree(&task_id, repo, store).await {
1825 tracing::warn!(task_id, err = %e, "worktree cleanup after merge failed");
1826 }
1827
1828 let comment = format!("PR merged, marking task complete{}", engine::orch_footer());
1829 if let Err(e) = backend.post_comment(&id, &comment).await {
1830 tracing::warn!(task_id, err = %e, "failed to post comment");
1831 }
1832 }
1833
1834 Ok(())
1835}
1836
1837async fn fallback_is_pr_merged_by_branch(
1838 backend: &Arc<dyn ExternalBackend>,
1839 branches: &[String],
1840) -> HashMap<String, bool> {
1841 let futs: Vec<_> = branches
1842 .iter()
1843 .map(|branch| {
1844 let backend = Arc::clone(backend);
1845 let branch = branch.clone();
1846 async move {
1847 match backend.is_pr_merged(&branch).await {
1848 Ok(is_merged) => Some((branch, is_merged)),
1849 Err(err) => {
1850 tracing::warn!(
1851 branch,
1852 err = %err,
1853 "fallback PR merge check failed for branch"
1854 );
1855 None
1856 }
1857 }
1858 }
1859 })
1860 .collect();
1861
1862 future::join_all(futs).await.into_iter().flatten().collect()
1863}
1864
1865#[cfg(test)]
1866mod tests {
1867 use super::*;
1868 use crate::engine::tasks;
1869 use crate::store::{NewTask, TaskStore};
1870 use async_trait::async_trait;
1871 use std::collections::HashMap;
1872 use std::process;
1873 use std::sync::Mutex;
1874 use std::sync::atomic::{AtomicUsize, Ordering};
1875
1876 #[tokio::test]
1879 async fn worktree_age_hours_returns_none_for_missing_dir() {
1880 let tmp = tempfile::tempdir().unwrap();
1881 let missing = tmp.path().join("does-not-exist");
1882 assert!(worktree_age_hours(&missing).await.is_none());
1883 }
1884
1885 #[tokio::test]
1886 async fn worktree_age_hours_returns_some_for_existing_dir() {
1887 let tmp = tempfile::tempdir().unwrap();
1888 let age = worktree_age_hours(tmp.path()).await;
1890 assert!(age.is_some());
1891 assert_eq!(age.unwrap(), 0, "freshly created dir should have age 0h");
1892 }
1893
1894 #[test]
1897 fn janitor_options_default_values() {
1898 let opts = JanitorOptions::default();
1899 assert_eq!(opts.ttl_hours, 24);
1900 assert!(!opts.dry_run);
1901 }
1902
1903 fn setup_test_repo() -> Option<(tempfile::TempDir, PathBuf)> {
1907 let tmp = tempfile::tempdir().ok()?;
1908
1909 let repo_dir = tmp.path().join("repo.git");
1911 std_fs::create_dir_all(&repo_dir).ok()?;
1912
1913 let init = process::Command::new("git")
1914 .args([
1915 "init",
1916 "--initial-branch=main",
1917 repo_dir
1918 .to_str()
1919 .expect("test repo path contains non-UTF-8 characters"),
1920 ])
1921 .output()
1922 .ok()?;
1923 if !init.status.success() {
1924 let _ = process::Command::new("git")
1926 .args([
1927 "init",
1928 repo_dir
1929 .to_str()
1930 .expect("test repo path contains non-UTF-8 characters"),
1931 ])
1932 .output()
1933 .ok()?;
1934 }
1935
1936 let readme = repo_dir.join("README.md");
1938 std_fs::write(&readme, "# test").ok()?;
1939 process::Command::new("git")
1940 .args(["-C", repo_dir.to_str()?, "add", "."])
1941 .output()
1942 .ok()?;
1943 process::Command::new("git")
1944 .args([
1945 "-C",
1946 repo_dir.to_str()?,
1947 "-c",
1948 "user.email=test@test.com",
1949 "-c",
1950 "user.name=Test",
1951 "-c",
1952 "commit.gpgsign=false",
1953 "commit",
1954 "--allow-empty",
1955 "-m",
1956 "init",
1957 ])
1958 .output()
1959 .ok()?;
1960
1961 let wt_dir = tmp
1963 .path()
1964 .join("worktrees")
1965 .join("repo.git")
1966 .join("gh-issue-42-test");
1967 std_fs::create_dir_all(wt_dir.parent()?).ok()?;
1968 let wt_out = process::Command::new("git")
1969 .args([
1970 "-C",
1971 repo_dir.to_str()?,
1972 "worktree",
1973 "add",
1974 "-b",
1975 "gh-issue-42-test",
1976 wt_dir.to_str()?,
1977 ])
1978 .output()
1979 .ok()?;
1980
1981 if !wt_out.status.success() {
1982 return None;
1983 }
1984
1985 Some((tmp, wt_dir))
1986 }
1987
1988 #[tokio::test]
1989 async fn janitor_skips_young_worktree() {
1990 let Some((tmp, wt_dir)) = setup_test_repo() else {
1991 eprintln!("skipping test: git not available");
1992 return;
1993 };
1994
1995 assert!(wt_dir.exists(), "worktree should exist before janitor runs");
1997
1998 let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2000 assert!(
2001 age < 24,
2002 "freshly created worktree should be younger than 24h (age={age}h)"
2003 );
2004
2005 let opts = JanitorOptions {
2007 ttl_hours: 24,
2008 dry_run: true,
2009 };
2010 assert!(age < opts.ttl_hours, "janitor should skip young worktree");
2012
2013 drop(tmp); }
2015
2016 #[tokio::test]
2017 async fn janitor_eligible_when_ttl_zero() {
2018 let Some((tmp, wt_dir)) = setup_test_repo() else {
2019 eprintln!("skipping test: git not available");
2020 return;
2021 };
2022
2023 let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2025 let opts = JanitorOptions {
2026 ttl_hours: 0,
2027 dry_run: true,
2028 };
2029 assert!(
2030 age >= opts.ttl_hours,
2031 "worktree should be eligible when ttl=0 (age={age}h)"
2032 );
2033
2034 drop(tmp);
2035 }
2036
2037 #[tokio::test]
2038 async fn dry_run_does_not_remove_worktree() {
2039 let Some((tmp, wt_dir)) = setup_test_repo() else {
2041 eprintln!("skipping test: git not available");
2042 return;
2043 };
2044
2045 assert!(wt_dir.exists(), "worktree must exist before test");
2046
2047 let opts = JanitorOptions {
2050 ttl_hours: 0,
2051 dry_run: true,
2052 };
2053
2054 let age = worktree_age_hours(&wt_dir).await.unwrap_or(u64::MAX);
2059 let in_use = false; let skip = age < opts.ttl_hours || in_use;
2061
2062 if !skip && opts.dry_run {
2063 tracing::debug!("[dry-run] would remove {}", wt_dir.display());
2065 } else if !skip {
2066 }
2068
2069 assert!(
2071 wt_dir.exists(),
2072 "dry-run must not remove the worktree directory"
2073 );
2074
2075 drop(tmp);
2076 }
2077
2078 #[test]
2079 fn fallback_branch_path_construction() {
2080 let tmp = tempfile::tempdir().unwrap();
2083 let wt_dir = tmp.path().join("myrepo").join("gh-issue-7-fix");
2084 std_fs::create_dir_all(&wt_dir).unwrap();
2085
2086 let branch = "gh-issue-7-fix";
2088 let repo = "owner/myrepo";
2089 let worktrees_base = tmp.path();
2090
2091 let project = repo
2092 .rsplit('/')
2093 .next()
2094 .unwrap_or(repo)
2095 .trim_end_matches(".git");
2096 let candidate = worktrees_base.join(project).join(branch);
2097
2098 assert_eq!(candidate, wt_dir);
2099 assert!(
2100 candidate.exists(),
2101 "fallback path should point to existing dir"
2102 );
2103 }
2104
2105 #[tokio::test]
2108 async fn cleanup_returns_false_when_nothing_to_clean() {
2109 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2110
2111 let id = store
2113 .create(&NewTask {
2114 external_id: Some("999".to_string()),
2115 repo: "owner/repo".to_string(),
2116 origin: "github".to_string(),
2117 title: "Already cleaned".to_string(),
2118 ..Default::default()
2119 })
2120 .await
2121 .unwrap();
2122 store.update_status(id, TaskStatus::Done).await.unwrap();
2123
2124 let opts = JanitorOptions {
2125 ttl_hours: 0,
2126 dry_run: false,
2127 };
2128
2129 let result = cleanup_task_worktree_with_opts("999", "owner/repo", &store, &opts).await;
2130 if let Ok(cleaned) = result {
2134 assert!(
2135 !cleaned,
2136 "cleanup should return false when there is no worktree or branch to clean"
2137 );
2138 } else {
2139 }
2142 }
2143
2144 #[tokio::test]
2148 async fn cleanup_marks_cleaned_when_stored_path_is_gone() {
2149 let tmp = tempfile::tempdir().unwrap();
2150 let stale_wt = tmp.path().join("stale-worktree");
2152 assert!(!stale_wt.exists(), "stale path must not exist");
2153
2154 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2155
2156 let id = store
2157 .create(&NewTask {
2158 external_id: Some("1005".to_string()),
2159 repo: "owner/repo".to_string(),
2160 origin: "github".to_string(),
2161 title: "Stale worktree task".to_string(),
2162 ..Default::default()
2163 })
2164 .await
2165 .unwrap();
2166 store.update_status(id, TaskStatus::Done).await.unwrap();
2167 store
2169 .set_fields(
2170 id,
2171 &[("worktree", stale_wt.to_string_lossy().as_ref().into())],
2172 )
2173 .await
2174 .unwrap();
2175
2176 let before = store.get(id).await.unwrap();
2178 assert!(!before.worktree_cleaned, "task should not be cleaned yet");
2179 assert!(
2180 !before.worktree.is_empty(),
2181 "task must have a worktree path set"
2182 );
2183
2184 let opts = JanitorOptions {
2185 ttl_hours: 0,
2186 dry_run: false,
2187 };
2188
2189 let result = cleanup_task_worktree_with_opts("1005", "owner/repo", &store, &opts).await;
2190
2191 match result {
2194 Ok(_) => {}
2195 Err(e) => panic!("cleanup should not error for stale path: {e}"),
2196 }
2197
2198 let after = store.get(id).await.unwrap();
2200 assert!(
2201 after.worktree_cleaned,
2202 "task must be marked cleaned when stored worktree path is already gone"
2203 );
2204 }
2205
2206 #[tokio::test]
2209 async fn remove_worktree_returns_true_on_success() {
2210 let Some((tmp, wt_dir)) = setup_test_repo() else {
2211 eprintln!("skipping test: git not available");
2212 return;
2213 };
2214
2215 let repo_dir = tmp.path().join("repo.git");
2216 assert!(wt_dir.exists(), "worktree must exist before removal");
2217
2218 let removed =
2219 remove_worktree_and_branch("42", &wt_dir, Some("gh-issue-42-test"), &repo_dir, false)
2220 .await;
2221
2222 assert!(
2223 removed,
2224 "remove_worktree_and_branch should return true when directory is gone"
2225 );
2226 assert!(!wt_dir.exists(), "worktree directory should be removed");
2227 }
2228
2229 #[tokio::test]
2232 async fn remove_worktree_success_still_deletes_local_branch() {
2233 let Some((tmp, wt_dir)) = setup_test_repo() else {
2234 eprintln!("skipping test: git not available");
2235 return;
2236 };
2237
2238 let repo_dir = tmp.path().join("repo.git");
2239 let repo_dir_str = repo_dir
2240 .to_str()
2241 .expect("test repo path contains non-UTF-8 characters");
2242 let branch = "gh-issue-42-test";
2243
2244 let before = process::Command::new("git")
2245 .args([
2246 "-C",
2247 repo_dir_str,
2248 "show-ref",
2249 "--verify",
2250 "--quiet",
2251 &format!("refs/heads/{branch}"),
2252 ])
2253 .status()
2254 .unwrap();
2255 assert!(
2256 before.success(),
2257 "test setup must create local branch before cleanup"
2258 );
2259
2260 let removed =
2261 remove_worktree_and_branch("42", &wt_dir, Some(branch), &repo_dir, false).await;
2262 assert!(removed, "worktree should be removed");
2263
2264 let after = process::Command::new("git")
2265 .args([
2266 "-C",
2267 repo_dir_str,
2268 "show-ref",
2269 "--verify",
2270 "--quiet",
2271 &format!("refs/heads/{branch}"),
2272 ])
2273 .status()
2274 .unwrap();
2275 assert!(
2276 !after.success(),
2277 "local branch should be deleted after successful worktree cleanup"
2278 );
2279 }
2280
2281 #[tokio::test]
2285 async fn remove_worktree_returns_false_when_dir_persists() {
2286 let tmp = tempfile::tempdir().unwrap();
2287 let fake_wt = tmp.path().join("not-a-worktree");
2290 std_fs::create_dir_all(&fake_wt).unwrap();
2291
2292 let removed = remove_worktree_and_branch("99", &fake_wt, None, tmp.path(), false).await;
2295
2296 assert!(
2299 !removed,
2300 "remove_worktree_and_branch should return false when directory still exists"
2301 );
2302 assert!(fake_wt.exists(), "directory should still be on disk");
2303 }
2304
2305 struct MergeCheckMockBackend {
2306 batch_result: Mutex<anyhow::Result<HashMap<String, bool>>>,
2307 single_results: HashMap<String, anyhow::Result<bool>>,
2308 posted_comments: Arc<Mutex<Vec<(String, String)>>>,
2309 single_checked_branches: Arc<Mutex<Vec<String>>>,
2310 }
2311
2312 impl MergeCheckMockBackend {
2313 fn with_batch_error(
2314 error: anyhow::Error,
2315 single_results: HashMap<String, anyhow::Result<bool>>,
2316 ) -> Self {
2317 Self {
2318 batch_result: Mutex::new(Err(error)),
2319 single_results,
2320 posted_comments: Arc::new(Mutex::new(Vec::new())),
2321 single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2322 }
2323 }
2324
2325 fn with_merged(branch: &str) -> Self {
2326 Self {
2327 batch_result: Mutex::new(Ok(HashMap::from([(branch.to_string(), true)]))),
2328 single_results: HashMap::new(),
2329 posted_comments: Arc::new(Mutex::new(Vec::new())),
2330 single_checked_branches: Arc::new(Mutex::new(Vec::new())),
2331 }
2332 }
2333 }
2334
2335 #[async_trait]
2336 impl ExternalBackend for MergeCheckMockBackend {
2337 fn name(&self) -> &'static str {
2338 "mock"
2339 }
2340
2341 async fn create_task(
2342 &self,
2343 _title: &str,
2344 _body: &str,
2345 _labels: &[String],
2346 ) -> anyhow::Result<ExternalId> {
2347 anyhow::bail!("not implemented")
2348 }
2349
2350 async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2351 anyhow::bail!("not implemented")
2352 }
2353
2354 async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2355 Ok(vec![])
2356 }
2357
2358 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2359 Ok(vec![])
2360 }
2361
2362 async fn post_comment(&self, id: &ExternalId, body: &str) -> anyhow::Result<()> {
2363 self.posted_comments
2364 .lock()
2365 .unwrap()
2366 .push((id.0.clone(), body.to_string()));
2367 Ok(())
2368 }
2369
2370 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2371 Ok(())
2372 }
2373
2374 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2375 Ok(())
2376 }
2377
2378 async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2379 Ok(vec![])
2380 }
2381
2382 async fn create_sub_task(
2383 &self,
2384 _parent: &ExternalId,
2385 _title: &str,
2386 _body: &str,
2387 _labels: &[String],
2388 ) -> anyhow::Result<ExternalId> {
2389 anyhow::bail!("not implemented")
2390 }
2391
2392 async fn health_check(&self) -> anyhow::Result<()> {
2393 Ok(())
2394 }
2395
2396 async fn is_pr_merged(&self, branch: &str) -> anyhow::Result<bool> {
2397 self.single_checked_branches
2398 .lock()
2399 .unwrap()
2400 .push(branch.to_string());
2401 match self.single_results.get(branch) {
2402 Some(Ok(is_merged)) => Ok(*is_merged),
2403 Some(Err(err)) => Err(anyhow::anyhow!(err.to_string())),
2404 None => Ok(false),
2405 }
2406 }
2407
2408 async fn batch_is_pr_merged(
2409 &self,
2410 _branches: &[String],
2411 ) -> anyhow::Result<HashMap<String, bool>> {
2412 match &*self.batch_result.lock().unwrap() {
2413 Ok(map) => Ok(map.clone()),
2414 Err(err) => Err(anyhow::anyhow!(err.to_string())),
2415 }
2416 }
2417 }
2418
2419 #[tokio::test]
2420 async fn check_merged_prs_falls_back_to_single_branch_checks_on_batch_error() {
2421 let single_results = HashMap::from([
2422 ("branch-1".to_string(), Ok(true)),
2423 ("branch-2".to_string(), Ok(false)),
2424 ]);
2425 let backend = Arc::new(MergeCheckMockBackend::with_batch_error(
2426 anyhow::anyhow!("missing /data/repository in GraphQL response: no error details"),
2427 single_results,
2428 ));
2429 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2430
2431 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2432 let task1 = store
2433 .create(&NewTask {
2434 external_id: Some("101".to_string()),
2435 repo: "owner/repo".to_string(),
2436 origin: "github".to_string(),
2437 title: "Task 101".to_string(),
2438 ..Default::default()
2439 })
2440 .await
2441 .unwrap();
2442 store
2443 .update_status(task1, TaskStatus::NeedsReview)
2444 .await
2445 .unwrap();
2446 store
2447 .set_fields(task1, &[("branch", serde_json::json!("branch-1"))])
2448 .await
2449 .unwrap();
2450 let wt_dir = tempfile::tempdir().unwrap();
2452 let wt_path = wt_dir.path().join("branch-1");
2453 std_fs::create_dir_all(&wt_path).unwrap();
2454 store
2455 .set_fields(
2456 task1,
2457 &[("worktree", serde_json::json!(wt_path.to_string_lossy()))],
2458 )
2459 .await
2460 .unwrap();
2461
2462 let task2 = store
2463 .create(&NewTask {
2464 external_id: Some("102".to_string()),
2465 repo: "owner/repo".to_string(),
2466 origin: "github".to_string(),
2467 title: "Task 102".to_string(),
2468 ..Default::default()
2469 })
2470 .await
2471 .unwrap();
2472 store
2473 .update_status(task2, TaskStatus::InReview)
2474 .await
2475 .unwrap();
2476 store
2477 .set_fields(task2, &[("branch", serde_json::json!("branch-2"))])
2478 .await
2479 .unwrap();
2480
2481 let task_manager = Arc::new(TaskManager::with_store(
2482 Arc::clone(&backend_dyn),
2483 Arc::clone(&store),
2484 "owner/repo".to_string(),
2485 ));
2486
2487 let in_review_tasks = store
2488 .list_by_status("owner/repo", TaskStatus::InReview)
2489 .await
2490 .unwrap()
2491 .into_iter()
2492 .map(|stored| ReviewTaskSnapshot {
2493 external: tasks::store_task_to_external(&stored),
2494 stored,
2495 })
2496 .collect::<Vec<_>>();
2497 let needs_review_tasks = store
2498 .list_by_status("owner/repo", TaskStatus::NeedsReview)
2499 .await
2500 .unwrap()
2501 .into_iter()
2502 .map(|stored| ReviewTaskSnapshot {
2503 external: tasks::store_task_to_external(&stored),
2504 stored,
2505 })
2506 .collect::<Vec<_>>();
2507
2508 check_merged_prs(
2509 &backend_dyn,
2510 "owner/repo",
2511 &store,
2512 &task_manager,
2513 &in_review_tasks,
2514 &needs_review_tasks,
2515 )
2516 .await
2517 .unwrap();
2518
2519 let checked = backend.single_checked_branches.lock().unwrap().clone();
2520 assert_eq!(
2521 checked,
2522 vec!["branch-2".to_string(), "branch-1".to_string()]
2523 );
2524
2525 let updated_task1 = store.get(task1).await.unwrap();
2526 let updated_task2 = store.get(task2).await.unwrap();
2527 assert_eq!(updated_task1.status, TaskStatus::Done);
2528 assert!(updated_task1.worktree_cleaned);
2529 assert!(!wt_path.exists());
2530 assert!(!updated_task2.worktree_cleaned);
2531 assert_eq!(updated_task2.status, TaskStatus::InReview);
2532
2533 let comments = backend.posted_comments.lock().unwrap().clone();
2534 assert_eq!(comments.len(), 1);
2535 assert_eq!(comments[0].0, "101");
2536 assert!(comments[0].1.contains("PR merged, marking task complete"));
2537 }
2538
2539 #[tokio::test]
2540 async fn check_merged_prs_skips_rescue_while_ci_bootstrap_holds() {
2541 let (tmp, clone) = origin_and_clone_with_feat();
2542 let wt = add_worktree(&clone, tmp.path(), "feat");
2543 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2544 let id = store_approved_task(
2546 &store,
2547 "owner/repo",
2548 "task-12",
2549 "CI-01 Set up CI and tests",
2550 "<!-- orch-plan: CI-01 -->",
2551 &wt,
2552 )
2553 .await;
2554 let stored = store.get(id).await.unwrap();
2555
2556 let backend = Arc::new(MergeCheckMockBackend::with_merged("feat"));
2557 let backend_dyn: Arc<dyn ExternalBackend> = Arc::<MergeCheckMockBackend>::clone(&backend);
2558 let task_manager = Arc::new(TaskManager::with_store(
2559 Arc::clone(&backend_dyn),
2560 Arc::clone(&store),
2561 "owner/repo".to_string(),
2562 ));
2563
2564 check_merged_prs(
2565 &backend_dyn,
2566 "owner/repo",
2567 &store,
2568 &task_manager,
2569 &[ReviewTaskSnapshot {
2570 external: tasks::store_task_to_external(&stored),
2571 stored,
2572 }],
2573 &[],
2574 )
2575 .await
2576 .unwrap();
2577
2578 let after = store.get(id).await.unwrap();
2579 assert_eq!(after.status, TaskStatus::Approved);
2580 assert!(after.worktree_cleaned);
2581 assert!(!wt.exists());
2582 assert!(!remote_has(&clone, "rescue/feat"));
2583 assert!(backend.posted_comments.lock().unwrap().is_empty());
2584 }
2585
2586 struct ReconciliationMockBackend {
2587 reconciliation_calls: Arc<AtomicUsize>,
2588 }
2589
2590 #[async_trait]
2591 impl ExternalBackend for ReconciliationMockBackend {
2592 fn name(&self) -> &'static str {
2593 "mock"
2594 }
2595
2596 async fn create_task(
2597 &self,
2598 _title: &str,
2599 _body: &str,
2600 _labels: &[String],
2601 ) -> anyhow::Result<ExternalId> {
2602 anyhow::bail!("not implemented")
2603 }
2604
2605 async fn get_task(&self, _id: &ExternalId) -> anyhow::Result<ExternalTask> {
2606 anyhow::bail!("not implemented")
2607 }
2608
2609 async fn list_by_status(&self, _status: Status) -> anyhow::Result<Vec<ExternalTask>> {
2610 Ok(vec![])
2611 }
2612
2613 async fn list_reconciliation_candidates(&self) -> anyhow::Result<Vec<ExternalTask>> {
2614 self.reconciliation_calls.fetch_add(1, Ordering::SeqCst);
2615 Ok(vec![])
2616 }
2617
2618 async fn list_routable(&self) -> anyhow::Result<Vec<ExternalTask>> {
2619 Ok(vec![])
2620 }
2621
2622 async fn post_comment(&self, _id: &ExternalId, _body: &str) -> anyhow::Result<()> {
2623 Ok(())
2624 }
2625
2626 async fn set_labels(&self, _id: &ExternalId, _labels: &[String]) -> anyhow::Result<()> {
2627 Ok(())
2628 }
2629
2630 async fn remove_label(&self, _id: &ExternalId, _label: &str) -> anyhow::Result<()> {
2631 Ok(())
2632 }
2633
2634 async fn get_sub_issues(&self, _id: &ExternalId) -> anyhow::Result<Vec<ExternalId>> {
2635 Ok(vec![])
2636 }
2637
2638 async fn create_sub_task(
2639 &self,
2640 _parent: &ExternalId,
2641 _title: &str,
2642 _body: &str,
2643 _labels: &[String],
2644 ) -> anyhow::Result<ExternalId> {
2645 anyhow::bail!("not implemented")
2646 }
2647
2648 async fn health_check(&self) -> anyhow::Result<()> {
2649 Ok(())
2650 }
2651 }
2652
2653 #[tokio::test]
2654 async fn cleanup_reconciliation_is_rate_limited_per_repo() {
2655 let repo = "owner/repo-cleanup-rate-limit";
2656 RECONCILIATION_LAST_RUN.remove(repo);
2657
2658 let backend = Arc::new(ReconciliationMockBackend {
2659 reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2660 });
2661 let backend_dyn: Arc<dyn ExternalBackend> =
2662 Arc::<ReconciliationMockBackend>::clone(&backend);
2663 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2664 let task_manager = Arc::new(TaskManager::with_store(
2665 Arc::clone(&backend_dyn),
2666 Arc::clone(&store),
2667 repo.to_string(),
2668 ));
2669 let opts = JanitorOptions {
2670 ttl_hours: 0,
2671 dry_run: true,
2672 };
2673
2674 cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2675 .await
2676 .unwrap();
2677 cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2678 .await
2679 .unwrap();
2680
2681 assert_eq!(
2682 backend.reconciliation_calls.load(Ordering::SeqCst),
2683 1,
2684 "cleanup should not re-list reconciliation candidates on every sync pass"
2685 );
2686
2687 RECONCILIATION_LAST_RUN.remove(repo);
2688 }
2689
2690 #[tokio::test]
2693 async fn cleanup_skips_blocked_internal_tasks_and_cleans_done_ones() {
2694 let repo = "owner/repo-cleanup-blocked";
2695 RECONCILIATION_LAST_RUN.remove(repo);
2696 let backend_dyn: Arc<dyn ExternalBackend> = Arc::new(ReconciliationMockBackend {
2697 reconciliation_calls: Arc::new(AtomicUsize::new(0)),
2698 });
2699 let store = Arc::new(TaskStore::open_memory().await.unwrap());
2700 let task_manager = Arc::new(TaskManager::with_store(
2701 Arc::clone(&backend_dyn),
2702 Arc::clone(&store),
2703 repo.to_string(),
2704 ));
2705
2706 let mut ids = Vec::new();
2707 for (title, status) in [("blocked", TaskStatus::Blocked), ("done", TaskStatus::Done)] {
2708 let id = store
2709 .create_internal(repo, title, "", "test", title, None)
2710 .await
2711 .unwrap();
2712 store
2713 .set_fields(
2714 id,
2715 &[
2716 (
2717 "worktree",
2718 serde_json::json!(format!("/nonexistent/{title}")),
2719 ),
2720 ("branch", serde_json::json!("")),
2721 ],
2722 )
2723 .await
2724 .unwrap();
2725 store.update_status(id, status).await.unwrap();
2726 ids.push(id);
2727 }
2728
2729 let opts = JanitorOptions {
2730 ttl_hours: 0,
2731 dry_run: false,
2732 };
2733 cleanup_done_worktrees_with_opts(&backend_dyn, repo, &task_manager, &store, &opts)
2734 .await
2735 .unwrap();
2736
2737 assert!(
2738 !store.get(ids[0]).await.unwrap().worktree_cleaned,
2739 "blocked task must not be considered for cleanup"
2740 );
2741 assert!(
2742 store.get(ids[1]).await.unwrap().worktree_cleaned,
2743 "done task is cleaned, which proves the pass ran"
2744 );
2745 RECONCILIATION_LAST_RUN.remove(repo);
2746 }
2747
2748 fn git_ok(dir: &Path, args: &[&str]) -> String {
2749 let out = process::Command::new("git")
2750 .args([
2751 "-c",
2752 "commit.gpgsign=false",
2753 "-c",
2754 "user.name=T",
2755 "-c",
2756 "user.email=t@t",
2757 ])
2758 .args(args)
2759 .current_dir(dir)
2760 .output()
2761 .unwrap();
2762 assert!(
2763 out.status.success(),
2764 "git {args:?}: {}",
2765 String::from_utf8_lossy(&out.stderr)
2766 );
2767 String::from_utf8_lossy(&out.stdout).to_string()
2768 }
2769
2770 fn origin_and_clone_with_feat() -> (tempfile::TempDir, PathBuf) {
2772 let tmp = tempfile::tempdir().unwrap();
2773 let bare = tmp.path().join("origin.git");
2774 std_fs::create_dir_all(&bare).unwrap();
2775 git_ok(&bare, &["init", "--bare", "-b", "main"]);
2776 git_ok(tmp.path(), &["clone", bare.to_str().unwrap(), "clone"]);
2777 let clone = tmp.path().join("clone");
2778 std_fs::write(clone.join("base.txt"), "base").unwrap();
2779 git_ok(&clone, &["add", "."]);
2780 git_ok(&clone, &["commit", "-m", "base"]);
2781 git_ok(&clone, &["push", "origin", "main"]);
2782 git_ok(&clone, &["checkout", "-b", "feat"]);
2783 std_fs::write(clone.join("work.txt"), "work").unwrap();
2784 git_ok(&clone, &["add", "."]);
2785 git_ok(&clone, &["commit", "-m", "work"]);
2786 git_ok(&clone, &["checkout", "main"]);
2787 (tmp, clone)
2788 }
2789
2790 fn remote_has(clone: &Path, branch: &str) -> bool {
2791 !git_ok(clone, &["ls-remote", "--heads", "origin", branch])
2792 .trim()
2793 .is_empty()
2794 }
2795
2796 fn local_has(clone: &Path, branch: &str) -> bool {
2797 process::Command::new("git")
2798 .args([
2799 "show-ref",
2800 "--verify",
2801 "--quiet",
2802 &format!("refs/heads/{branch}"),
2803 ])
2804 .current_dir(clone)
2805 .status()
2806 .unwrap()
2807 .success()
2808 }
2809
2810 #[tokio::test]
2813 async fn delete_branches_keeps_an_unmerged_remote_branch() {
2814 let (_tmp, clone) = origin_and_clone_with_feat();
2815 git_ok(&clone, &["push", "origin", "feat"]);
2816
2817 delete_branches("1", "feat", &clone).await;
2818
2819 assert!(
2820 remote_has(&clone, "feat"),
2821 "unmerged remote branch must survive"
2822 );
2823 assert!(
2824 !local_has(&clone, "feat"),
2825 "local copy is on origin, so it can go"
2826 );
2827 }
2828
2829 #[tokio::test]
2830 async fn delete_branches_leaves_the_screenshots_branch_alone() {
2831 let (_tmp, clone) = origin_and_clone_with_feat();
2832 git_ok(&clone, &["branch", SCREENSHOTS_BRANCH, "feat"]);
2833 git_ok(&clone, &["push", "origin", SCREENSHOTS_BRANCH]);
2834 git_ok(&clone, &["merge", "--ff-only", "feat"]);
2835 git_ok(&clone, &["push", "origin", "main"]);
2836
2837 delete_branches("1", SCREENSHOTS_BRANCH, &clone).await;
2838
2839 assert!(remote_has(&clone, SCREENSHOTS_BRANCH));
2840 assert!(local_has(&clone, SCREENSHOTS_BRANCH));
2841 }
2842
2843 #[tokio::test]
2844 async fn delete_branches_removes_a_branch_main_already_contains() {
2845 let (_tmp, clone) = origin_and_clone_with_feat();
2846 git_ok(&clone, &["push", "origin", "feat"]);
2847 git_ok(&clone, &["merge", "--ff-only", "feat"]);
2848 git_ok(&clone, &["push", "origin", "main"]);
2849
2850 delete_branches("1", "feat", &clone).await;
2851
2852 assert!(!remote_has(&clone, "feat"));
2853 assert!(!local_has(&clone, "feat"));
2854 }
2855
2856 fn add_worktree(clone: &Path, tmp: &Path, branch: &str) -> PathBuf {
2857 let wt = tmp.join("wt");
2858 git_ok(clone, &["worktree", "add", wt.to_str().unwrap(), branch]);
2859 wt
2860 }
2861
2862 async fn store_approved_task(
2863 store: &TaskStore,
2864 repo: &str,
2865 external_id: &str,
2866 title: &str,
2867 body: &str,
2868 worktree: &Path,
2869 ) -> i64 {
2870 let id = store
2871 .create(&NewTask {
2872 external_id: Some(external_id.to_string()),
2873 repo: repo.to_string(),
2874 origin: "github".to_string(),
2875 title: title.to_string(),
2876 body: body.to_string(),
2877 ..Default::default()
2878 })
2879 .await
2880 .unwrap();
2881 store.update_status(id, TaskStatus::Approved).await.unwrap();
2882 store
2883 .set_fields(
2884 id,
2885 &[
2886 ("branch", serde_json::json!("feat")),
2887 (
2888 "worktree",
2889 serde_json::json!(worktree.to_string_lossy().as_ref()),
2890 ),
2891 ("pr_number", serde_json::json!(7)),
2892 ],
2893 )
2894 .await
2895 .unwrap();
2896 id
2897 }
2898
2899 #[tokio::test]
2900 async fn cleanup_pushes_unpushed_commits_to_a_rescue_branch() {
2901 let (tmp, clone) = origin_and_clone_with_feat();
2902 let wt = add_worktree(&clone, tmp.path(), "feat");
2903
2904 let sha = git_ok(&clone, &["rev-parse", "--short", "feat"]);
2905 let sha = sha.trim();
2906 match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2907 Rescue::Pushed { branch, commits } => {
2908 assert_eq!(branch, "rescue/feat");
2909 assert!(rescue_comment(&branch, &commits).contains(sha));
2910 }
2911 other => panic!("expected a rescue push, got {other:?}"),
2912 }
2913
2914 let tip = git_ok(&clone, &["rev-parse", "feat"]);
2915 let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2916 assert_eq!(
2917 tracking.trim(),
2918 tip.trim(),
2919 "rescue push must point the tracking ref at the tip it sent"
2920 );
2921
2922 let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2923
2924 assert!(removed);
2925 assert!(remote_has(&clone, "rescue/feat"));
2926 assert!(!local_has(&clone, "feat"));
2927 }
2928
2929 #[tokio::test]
2930 async fn rescue_push_sets_tracking_ref_without_a_fetchspec() {
2931 let (tmp, clone) = origin_and_clone_with_feat();
2932 let wt = add_worktree(&clone, tmp.path(), "feat");
2933 git_ok(&clone, &["config", "--unset-all", "remote.origin.fetch"]);
2934
2935 match rescue_unpushed_work("1", Some(&wt), "feat", &clone).await {
2936 Rescue::Pushed { branch, .. } => assert_eq!(branch, "rescue/feat"),
2937 other => panic!("expected a rescue push, got {other:?}"),
2938 }
2939
2940 let tracking = git_ok(&clone, &["rev-parse", "refs/remotes/origin/rescue/feat"]);
2941 let tip = git_ok(&clone, &["rev-parse", "feat"]);
2942 assert_eq!(tracking.trim(), tip.trim());
2943 }
2944
2945 #[tokio::test]
2946 async fn cleanup_commits_uncommitted_changes_before_the_rescue() {
2947 let (tmp, clone) = origin_and_clone_with_feat();
2948 git_ok(&clone, &["push", "origin", "feat"]);
2949 let wt = add_worktree(&clone, tmp.path(), "feat");
2950 std_fs::write(wt.join("dirty.txt"), "dirty").unwrap();
2951
2952 let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2953
2954 assert!(removed);
2955 assert!(remote_has(&clone, "rescue/feat"));
2956 }
2957
2958 #[tokio::test]
2959 async fn cleanup_makes_no_rescue_branch_for_a_pushed_branch() {
2960 let (tmp, clone) = origin_and_clone_with_feat();
2961 git_ok(&clone, &["push", "origin", "feat"]);
2962 let wt = add_worktree(&clone, tmp.path(), "feat");
2963
2964 let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
2965
2966 assert!(removed);
2967 assert!(!remote_has(&clone, "rescue/feat"));
2968 assert!(!local_has(&clone, "feat"));
2969 }
2970
2971 #[tokio::test]
2972 async fn rescue_skips_a_branch_on_the_remote_without_a_tracking_ref() {
2973 let (_tmp, clone) = origin_and_clone_with_feat();
2974 git_ok(&clone, &["push", "origin", "feat"]);
2975 git_ok(&clone, &["update-ref", "-d", "refs/remotes/origin/feat"]);
2976
2977 let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
2978
2979 assert_eq!(rescue, Rescue::Clean);
2980 assert!(!remote_has(&clone, "rescue/feat"));
2981 }
2982
2983 #[tokio::test]
2984 async fn merged_pr_cleanup_pushes_no_rescue_branch() {
2985 let (tmp, clone) = origin_and_clone_with_feat();
2986 let wt = add_worktree(&clone, tmp.path(), "feat");
2987
2988 let removed =
2989 remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, true).await;
2990
2991 assert!(removed);
2992 assert!(!wt.exists());
2993 assert!(!remote_has(&clone, "rescue/feat"));
2994 }
2995
2996 #[tokio::test]
2997 async fn cleanup_skips_rescue_for_an_approved_merged_pr() {
2998 let (tmp, clone) = origin_and_clone_with_feat();
2999 let wt = add_worktree(&clone, tmp.path(), "feat");
3000 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3001 store_approved_task(&store, "owner/repo", "task-12", "CI-01 Set up CI", "", &wt).await;
3002
3003 cleanup_merged_pr_worktree("task-12", "owner/repo", &store)
3004 .await
3005 .unwrap();
3006
3007 let task = store
3008 .get(
3009 store
3010 .resolve_task_id("owner/repo", "task-12")
3011 .await
3012 .unwrap()
3013 .unwrap(),
3014 )
3015 .await
3016 .unwrap();
3017 assert_eq!(task.status, TaskStatus::Approved);
3018 assert!(task.worktree_cleaned);
3019 assert!(!wt.exists());
3020 assert!(!remote_has(&clone, "rescue/feat"));
3021 }
3022
3023 #[tokio::test]
3024 async fn cleanup_rescues_an_approved_task_when_the_pr_is_not_known_merged() {
3025 let (tmp, clone) = origin_and_clone_with_feat();
3026 let wt = add_worktree(&clone, tmp.path(), "feat");
3027 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3028 store_approved_task(&store, "owner/repo", "task-12", "fix something", "", &wt).await;
3029
3030 cleanup_task_worktree("task-12", "owner/repo", &store)
3031 .await
3032 .unwrap();
3033
3034 assert!(remote_has(&clone, "rescue/feat"));
3035 }
3036
3037 #[tokio::test]
3038 async fn unmerged_cleanup_still_pushes_a_rescue_branch() {
3039 let (tmp, clone) = origin_and_clone_with_feat();
3040 let wt = add_worktree(&clone, tmp.path(), "feat");
3041
3042 let removed =
3043 remove_worktree_and_branch_for("1", &wt, Some("feat"), &clone, false, false).await;
3044
3045 assert!(removed);
3046 assert!(remote_has(&clone, "rescue/feat"));
3047 }
3048
3049 #[tokio::test]
3050 async fn rescue_ignores_a_remote_tip_missing_locally() {
3051 let (tmp, clone) = origin_and_clone_with_feat();
3052 git_ok(&clone, &["push", "origin", "feat"]);
3053 let other = tmp.path().join("other");
3054 git_ok(
3055 tmp.path(),
3056 &["clone", "-b", "feat", "origin.git", other.to_str().unwrap()],
3057 );
3058 std_fs::write(other.join("extra.txt"), "extra").unwrap();
3059 git_ok(&other, &["add", "."]);
3060 git_ok(&other, &["commit", "-m", "extra"]);
3061 git_ok(&other, &["push", "origin", "feat"]);
3062
3063 let rescue = rescue_unpushed_work("1", None, "feat", &clone).await;
3064
3065 assert_ne!(rescue, Rescue::Failed);
3066 }
3067
3068 #[tokio::test]
3069 async fn cleanup_keeps_worktree_and_branch_when_the_rescue_push_fails() {
3070 let (tmp, clone) = origin_and_clone_with_feat();
3071 let wt = add_worktree(&clone, tmp.path(), "feat");
3072 git_ok(
3073 &clone,
3074 &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3075 );
3076
3077 let removed = remove_worktree_and_branch("1", &wt, Some("feat"), &clone, false).await;
3078
3079 assert!(!removed);
3080 assert!(wt.exists());
3081 assert!(local_has(&clone, "feat"));
3082 }
3083
3084 #[tokio::test]
3085 async fn delete_branches_keeps_a_local_branch_with_unpushed_commits() {
3086 let (_tmp, clone) = origin_and_clone_with_feat();
3087
3088 delete_branches("1", "feat", &clone).await;
3089
3090 assert!(
3091 local_has(&clone, "feat"),
3092 "unpushed commits must not be deleted"
3093 );
3094 }
3095}