use super::config::WorktreeCleanupConfig;
use super::error::{CleanupError, CleanupResult};
use super::monitor::{CleanupRecommendation, WorktreeResourceMonitor};
use std::collections::{HashMap, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::process::Command;
use tokio::sync::{Mutex, RwLock};
use tokio::task::JoinHandle;
use tokio::time::{sleep, timeout};
use tracing::{debug, error, info, warn};
#[derive(Debug, Clone)]
pub enum CleanupTask {
Immediate {
worktree_path: PathBuf,
job_id: String,
},
Scheduled {
worktree_path: PathBuf,
delay: Duration,
},
Batch { worktree_paths: Vec<PathBuf> },
}
pub struct CleanupGuard {
worktree_path: PathBuf,
coordinator: Arc<WorktreeCleanupCoordinator>,
}
impl CleanupGuard {
pub async fn schedule_cleanup(self, delay: Duration) -> CleanupResult<()> {
self.coordinator
.schedule_cleanup(CleanupTask::Scheduled {
worktree_path: self.worktree_path,
delay,
})
.await
}
pub async fn immediate_cleanup(self) -> CleanupResult<()> {
self.coordinator
.cleanup_worktree(&self.worktree_path, true)
.await?;
Ok(())
}
}
pub struct WorktreeCleanupCoordinator {
config: WorktreeCleanupConfig,
active_worktrees: Arc<RwLock<HashMap<String, Vec<WorktreeHandle>>>>,
cleanup_queue: Arc<Mutex<VecDeque<CleanupTask>>>,
cleanup_worker: Arc<Mutex<Option<JoinHandle<()>>>>,
resource_monitor: Arc<Mutex<WorktreeResourceMonitor>>,
worktree_base_path: PathBuf,
}
#[derive(Debug, Clone)]
struct WorktreeHandle {
path: PathBuf,
_created_at: Instant,
_job_id: String,
_agent_id: String,
}
impl WorktreeCleanupCoordinator {
pub fn new(config: WorktreeCleanupConfig, worktree_base_path: PathBuf) -> Self {
let resource_monitor = Arc::new(Mutex::new(WorktreeResourceMonitor::new(
config.disk_usage_threshold_mb,
config.max_worktrees_per_job,
config.max_total_worktrees,
)));
Self {
config,
active_worktrees: Arc::new(RwLock::new(HashMap::new())),
cleanup_queue: Arc::new(Mutex::new(VecDeque::new())),
cleanup_worker: Arc::new(Mutex::new(None)),
resource_monitor,
worktree_base_path,
}
}
pub async fn start(&self) {
let mut worker_guard = self.cleanup_worker.lock().await;
if worker_guard.is_some() {
return;
}
let queue = Arc::clone(&self.cleanup_queue);
let config = self.config.clone();
let coordinator = Arc::new(self.clone());
let handle = tokio::spawn(async move {
loop {
let task = {
let mut queue_guard = queue.lock().await;
queue_guard.pop_front()
};
if let Some(task) = task {
if let Err(e) = Self::process_cleanup_task(task, &coordinator).await {
error!("Cleanup task failed: {}", e);
}
} else {
sleep(Duration::from_secs(1)).await;
}
if config.enable_monitoring {
if let Err(e) = coordinator.periodic_cleanup_check().await {
warn!("Periodic cleanup check failed: {}", e);
}
}
}
});
*worker_guard = Some(handle);
}
pub async fn stop(&self) {
let mut worker_guard = self.cleanup_worker.lock().await;
if let Some(handle) = worker_guard.take() {
handle.abort();
let _ = handle.await;
}
}
pub async fn register_job(&self, job_id: &str) -> CleanupGuard {
let mut active = self.active_worktrees.write().await;
active.entry(job_id.to_string()).or_insert_with(Vec::new);
CleanupGuard {
worktree_path: self.worktree_base_path.clone(),
coordinator: Arc::new(self.clone()),
}
}
pub async fn register_worktree(
&self,
job_id: &str,
agent_id: &str,
worktree_path: PathBuf,
) -> CleanupGuard {
let handle = WorktreeHandle {
path: worktree_path.clone(),
_created_at: Instant::now(),
_job_id: job_id.to_string(),
_agent_id: agent_id.to_string(),
};
let mut active = self.active_worktrees.write().await;
active
.entry(job_id.to_string())
.or_insert_with(Vec::new)
.push(handle);
if self.config.enable_monitoring {
if let Err(e) = self.check_resource_limits().await {
warn!("Resource limit check failed: {}", e);
}
}
CleanupGuard {
worktree_path,
coordinator: Arc::new(self.clone()),
}
}
pub async fn schedule_cleanup(&self, task: CleanupTask) -> CleanupResult<()> {
if let CleanupTask::Scheduled {
worktree_path,
delay,
} = task
{
let queue = Arc::clone(&self.cleanup_queue);
let job_id = String::new();
tokio::spawn(async move {
sleep(delay).await;
let mut queue_guard = queue.lock().await;
queue_guard.push_back(CleanupTask::Immediate {
worktree_path,
job_id,
});
});
return Ok(());
}
let mut queue = self.cleanup_queue.lock().await;
queue.push_back(task);
Ok(())
}
pub async fn cleanup_job(&self, job_id: &str) -> CleanupResult<usize> {
let handles = {
let mut active = self.active_worktrees.write().await;
active.remove(job_id).unwrap_or_default()
};
let count = handles.len();
info!("Cleaning up {} worktrees for job {}", count, job_id);
for handle in handles {
if let Err(e) = self.cleanup_worktree(&handle.path, false).await {
warn!(
"Failed to cleanup worktree {}: {}",
handle.path.display(),
e
);
}
}
Ok(count)
}
pub async fn cleanup_orphaned_worktrees(&self, max_age: Duration) -> CleanupResult<usize> {
let candidates =
WorktreeResourceMonitor::get_cleanup_candidates(&self.worktree_base_path, max_age)
.await?;
let mut cleaned = 0;
for path in candidates {
let is_tracked = {
let active = self.active_worktrees.read().await;
active
.values()
.any(|handles| handles.iter().any(|h| h.path == path))
};
if !is_tracked {
info!("Cleaning orphaned worktree: {}", path.display());
if let Err(e) = self.cleanup_worktree(&path, true).await {
warn!("Failed to cleanup orphaned worktree: {}", e);
} else {
cleaned += 1;
}
}
}
Ok(cleaned)
}
pub async fn cleanup_worktree(&self, worktree_path: &Path, force: bool) -> CleanupResult<()> {
info!("Cleaning up worktree: {}", worktree_path.display());
if !worktree_path.exists() {
debug!(
"Worktree doesn't exist, skipping: {}",
worktree_path.display()
);
return Ok(());
}
if !force && self.is_worktree_active(worktree_path).await? {
return Err(CleanupError::WorktreeActive);
}
let start_time = Instant::now();
let result = timeout(
Duration::from_secs(self.config.cleanup_timeout_secs),
self.remove_git_worktree(worktree_path),
)
.await;
match result {
Ok(Ok(())) => {
self.untrack_worktree(worktree_path).await;
let mut monitor = self.resource_monitor.lock().await;
monitor.record_cleanup(true, start_time.elapsed());
info!(
"Successfully cleaned up worktree: {}",
worktree_path.display()
);
Ok(())
}
Ok(Err(e)) => {
let mut monitor = self.resource_monitor.lock().await;
monitor.record_cleanup(false, start_time.elapsed());
Err(e)
}
Err(_) => {
let mut monitor = self.resource_monitor.lock().await;
monitor.record_cleanup(false, start_time.elapsed());
Err(CleanupError::Timeout {
timeout: Duration::from_secs(self.config.cleanup_timeout_secs),
})
}
}
}
async fn remove_git_worktree(&self, worktree_path: &Path) -> CleanupResult<()> {
let worktree_name = worktree_path
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| CleanupError::GitError("Invalid worktree path".to_string()))?;
let output = Command::new("git")
.args(["worktree", "remove", worktree_name, "--force"])
.current_dir(
self.worktree_base_path
.parent()
.unwrap_or(&self.worktree_base_path),
)
.output()
.await
.map_err(|e| CleanupError::GitError(format!("Failed to run git command: {}", e)))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
warn!(
"Git worktree remove failed: {}, attempting manual removal",
stderr
);
tokio::fs::remove_dir_all(worktree_path)
.await
.map_err(|e| CleanupError::RemovalFailed {
path: worktree_path.to_path_buf(),
source: e,
})?;
let _ = Command::new("git")
.args(["worktree", "prune"])
.current_dir(
self.worktree_base_path
.parent()
.unwrap_or(&self.worktree_base_path),
)
.output()
.await;
}
Ok(())
}
async fn is_worktree_active(&self, worktree_path: &Path) -> CleanupResult<bool> {
let git_lock = worktree_path.join(".git/index.lock");
if git_lock.exists() {
return Ok(true);
}
Ok(false)
}
async fn untrack_worktree(&self, worktree_path: &Path) {
let mut active = self.active_worktrees.write().await;
for handles in active.values_mut() {
handles.retain(|h| h.path != worktree_path);
}
active.retain(|_, handles| !handles.is_empty());
}
async fn check_resource_limits(&self) -> CleanupResult<()> {
let mut monitor = self.resource_monitor.lock().await;
monitor
.scan_worktree_directory(&self.worktree_base_path)
.await?;
if let Err(e) = monitor.check_limits() {
warn!("Resource limit exceeded: {}", e);
match monitor.cleanup_recommendation() {
CleanupRecommendation::EmergencyCleanup { reason } => {
warn!("Emergency cleanup triggered: {}", reason);
let _ = self
.cleanup_orphaned_worktrees(Duration::from_secs(300))
.await;
}
CleanupRecommendation::CleanupOld { threshold } => {
let _ = self.cleanup_orphaned_worktrees(threshold).await;
}
CleanupRecommendation::CleanupFailed => {
}
CleanupRecommendation::None => {}
}
}
Ok(())
}
async fn periodic_cleanup_check(&self) -> CleanupResult<()> {
static LAST_CHECK: std::sync::OnceLock<Mutex<Instant>> = std::sync::OnceLock::new();
let last_check = LAST_CHECK.get_or_init(|| Mutex::new(Instant::now()));
let mut last = last_check.lock().await;
if last.elapsed() < Duration::from_secs(60) {
return Ok(());
}
*last = Instant::now();
self.check_resource_limits().await?;
let cleaned = self
.cleanup_orphaned_worktrees(Duration::from_secs(3600))
.await?;
if cleaned > 0 {
info!("Periodic cleanup removed {} orphaned worktrees", cleaned);
}
Ok(())
}
async fn process_cleanup_task(
task: CleanupTask,
coordinator: &Arc<WorktreeCleanupCoordinator>,
) -> CleanupResult<()> {
match task {
CleanupTask::Immediate { worktree_path, .. } => {
coordinator.cleanup_worktree(&worktree_path, false).await?;
}
CleanupTask::Scheduled { .. } => {
}
CleanupTask::Batch { worktree_paths } => {
for path in worktree_paths {
if let Err(e) = coordinator.cleanup_worktree(&path, false).await {
warn!("Failed to cleanup worktree in batch: {}", e);
}
}
}
}
Ok(())
}
}
impl Clone for WorktreeCleanupCoordinator {
fn clone(&self) -> Self {
Self {
config: self.config.clone(),
active_worktrees: Arc::clone(&self.active_worktrees),
cleanup_queue: Arc::clone(&self.cleanup_queue),
cleanup_worker: Arc::clone(&self.cleanup_worker),
resource_monitor: Arc::clone(&self.resource_monitor),
worktree_base_path: self.worktree_base_path.clone(),
}
}
}