Skip to main content

track_core/
remote_agent.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::env;
3use std::fs;
4use std::io::Write;
5use std::path::PathBuf;
6use std::process::{Command, Stdio};
7use std::sync::{Condvar, Mutex, OnceLock};
8
9use serde::de::DeserializeOwned;
10use serde::{Deserialize, Serialize};
11use serde_json::json;
12use time::Duration;
13
14use crate::backend_config::RemoteAgentConfigService;
15use crate::dispatch_repository::DispatchRepository;
16use crate::errors::{ErrorCode, TrackError};
17use crate::paths::{collapse_home_path, path_to_string};
18use crate::project_repository::{ProjectMetadata, ProjectRepository};
19use crate::review_dispatch_repository::ReviewDispatchRepository;
20use crate::review_repository::ReviewRepository;
21use crate::task_description::{append_follow_up_request, parse_task_description};
22use crate::task_id::build_unique_task_id;
23use crate::task_repository::FileTaskRepository;
24use crate::time_utils::{format_iso_8601_millis, now_utc, parse_iso_8601_seconds};
25use crate::types::{
26    CreateReviewInput, DispatchStatus, RemoteAgentDispatchOutcome, RemoteAgentPreferredTool,
27    RemoteAgentReviewOutcome, RemoteCleanupSummary, RemoteResetSummary, ReviewRecord,
28    ReviewRunRecord, Status, TaskDispatchRecord, TaskUpdateInput,
29};
30
31const REMOTE_STATUS_FILE_NAME: &str = "status.txt";
32const REMOTE_RESULT_FILE_NAME: &str = "result.json";
33const REMOTE_STDERR_FILE_NAME: &str = "stderr.log";
34const REMOTE_FINISHED_AT_FILE_NAME: &str = "finished-at.txt";
35const REMOTE_PROMPT_FILE_NAME: &str = "prompt.md";
36const REMOTE_SCHEMA_FILE_NAME: &str = "result-schema.json";
37const REMOTE_LAUNCHER_PID_FILE_NAME: &str = "launcher.pid";
38// We keep the historical sidecar filename for the child agent PID so users can
39// still cancel or clean up runs that were launched before Claude support
40// landed. The file now stores whichever remote agent process is active.
41const REMOTE_CODEX_PID_FILE_NAME: &str = "codex.pid";
42// Repository bootstrap can legitimately take a while on first clone or after a
43// large fetch, so we keep the stale-preparing threshold generous. The API now
44// also refreshes the summary at each preparation phase so normal progress keeps
45// pushing this timeout forward instead of relying on one initial timestamp.
46const PREPARING_STALE_AFTER: Duration = Duration::minutes(30);
47
48const REVIEW_WORKTREE_DIRECTORY_NAME: &str = "review-worktrees";
49const REVIEW_RUN_DIRECTORY_NAME: &str = "review-runs";
50
51#[derive(Debug, Default)]
52struct TaskDispatchStartGate {
53    active_task_ids: Mutex<BTreeSet<String>>,
54    wake_waiters: Condvar,
55}
56
57#[derive(Debug)]
58struct TaskDispatchStartGuard {
59    task_id: String,
60}
61
62impl TaskDispatchStartGuard {
63    fn acquire(task_id: &str) -> Self {
64        let gate = task_dispatch_start_gate();
65        let mut active_task_ids = gate
66            .active_task_ids
67            .lock()
68            .expect("dispatch start gate should not be poisoned");
69
70        while active_task_ids.contains(task_id) {
71            active_task_ids = gate
72                .wake_waiters
73                .wait(active_task_ids)
74                .expect("dispatch start gate should not be poisoned");
75        }
76
77        active_task_ids.insert(task_id.to_owned());
78
79        Self {
80            task_id: task_id.to_owned(),
81        }
82    }
83}
84
85impl Drop for TaskDispatchStartGuard {
86    fn drop(&mut self) {
87        let gate = task_dispatch_start_gate();
88        let mut active_task_ids = gate
89            .active_task_ids
90            .lock()
91            .expect("dispatch start gate should not be poisoned");
92        active_task_ids.remove(&self.task_id);
93        gate.wake_waiters.notify_all();
94    }
95}
96
97fn task_dispatch_start_gate() -> &'static TaskDispatchStartGate {
98    static GATE: OnceLock<TaskDispatchStartGate> = OnceLock::new();
99
100    // Dispatch start requests are handled by one long-lived API process in the
101    // deployed shape, so a process-local gate is enough to close the race
102    // between "no active dispatch exists" and "persist a new preparing record".
103    // This keeps the fix lightweight and avoids inventing filesystem locks for
104    // a code path that only needs in-process serialization.
105    GATE.get_or_init(TaskDispatchStartGate::default)
106}
107
108#[derive(Debug, Default)]
109struct ReviewDispatchStartGate {
110    active_review_ids: Mutex<BTreeSet<String>>,
111    wake_waiters: Condvar,
112}
113
114#[derive(Debug)]
115struct ReviewDispatchStartGuard {
116    review_id: String,
117}
118
119impl ReviewDispatchStartGuard {
120    fn acquire(review_id: &str) -> Self {
121        let gate = review_dispatch_start_gate();
122        let mut active_review_ids = gate
123            .active_review_ids
124            .lock()
125            .expect("review dispatch start gate should not be poisoned");
126
127        while active_review_ids.contains(review_id) {
128            active_review_ids = gate
129                .wake_waiters
130                .wait(active_review_ids)
131                .expect("review dispatch start gate should not be poisoned");
132        }
133
134        active_review_ids.insert(review_id.to_owned());
135
136        Self {
137            review_id: review_id.to_owned(),
138        }
139    }
140}
141
142impl Drop for ReviewDispatchStartGuard {
143    fn drop(&mut self) {
144        let gate = review_dispatch_start_gate();
145        let mut active_review_ids = gate
146            .active_review_ids
147            .lock()
148            .expect("review dispatch start gate should not be poisoned");
149        active_review_ids.remove(&self.review_id);
150        gate.wake_waiters.notify_all();
151    }
152}
153
154fn review_dispatch_start_gate() -> &'static ReviewDispatchStartGate {
155    static GATE: OnceLock<ReviewDispatchStartGate> = OnceLock::new();
156
157    // Reviews are now follow-up capable, so the same "check for active work,
158    // then persist a preparing record" race that tasks already guard against
159    // applies here too. Keeping a dedicated gate per review id preserves the
160    // review domain boundary without forcing the task flow to share review-only
161    // coordination state.
162    GATE.get_or_init(ReviewDispatchStartGate::default)
163}
164
165#[derive(Debug, Clone, Default, PartialEq, Eq)]
166struct RemoteDispatchSnapshot {
167    status: Option<String>,
168    result: Option<String>,
169    stderr: Option<String>,
170    finished_at: Option<String>,
171}
172
173#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
174struct RemoteArtifactCleanupCounts {
175    worktrees_removed: usize,
176    run_directories_removed: usize,
177}
178
179#[derive(Debug, Deserialize)]
180struct RemoteArtifactCleanupReport {
181    #[serde(rename = "worktreesRemoved")]
182    worktrees_removed: usize,
183    #[serde(rename = "runDirectoriesRemoved")]
184    run_directories_removed: usize,
185}
186
187#[derive(Debug, Deserialize)]
188struct RemoteWorkspaceResetReport {
189    #[serde(rename = "workspaceEntriesRemoved")]
190    workspace_entries_removed: usize,
191    #[serde(rename = "registryRemoved")]
192    registry_removed: bool,
193}
194
195#[derive(Debug, Clone, Default, PartialEq, Eq)]
196pub struct RemoteReviewFollowUpReconciliation {
197    pub queued_dispatches: Vec<TaskDispatchRecord>,
198    pub review_notifications_updated: usize,
199    pub failures: usize,
200    pub events: Vec<RemoteReviewFollowUpEvent>,
201}
202
203#[derive(Debug, Clone, PartialEq, Eq)]
204pub struct RemoteReviewFollowUpEvent {
205    pub outcome: String,
206    pub detail: String,
207    pub task_id: String,
208    pub dispatch_id: String,
209    pub dispatch_status: String,
210    pub remote_host: String,
211    pub branch_name: Option<String>,
212    pub pull_request_url: Option<String>,
213    pub reviewer: String,
214    pub pr_is_open: Option<bool>,
215    pub pr_head_oid: Option<String>,
216    pub latest_review_state: Option<String>,
217    pub latest_review_submitted_at: Option<String>,
218}
219
220#[derive(Debug, Clone, PartialEq, Eq)]
221struct GithubPullRequestReference {
222    owner: String,
223    repository: String,
224    number: u64,
225}
226
227#[derive(Debug, Clone, PartialEq, Eq)]
228struct GithubPullRequestMetadata {
229    pull_request_url: String,
230    pull_request_number: u64,
231    pull_request_title: String,
232    repository_full_name: String,
233    repo_url: String,
234    git_url: String,
235    base_branch: String,
236    head_oid: String,
237}
238
239#[derive(Debug, Clone, PartialEq, Eq)]
240struct GithubPullRequestReviewState {
241    is_open: bool,
242    head_oid: String,
243    latest_eligible_review: Option<GithubSubmittedReview>,
244}
245
246#[derive(Debug, Clone, PartialEq, Eq)]
247struct GithubSubmittedReview {
248    state: String,
249    submitted_at: time::OffsetDateTime,
250}
251
252#[derive(Debug, Deserialize)]
253struct GithubPullRequestApiResponse {
254    state: String,
255    title: String,
256    #[serde(rename = "merged_at")]
257    merged_at: Option<String>,
258    base: GithubPullRequestBaseApiResponse,
259    head: GithubPullRequestHeadApiResponse,
260}
261
262#[derive(Debug, Deserialize)]
263struct GithubPullRequestBaseApiResponse {
264    #[serde(rename = "ref")]
265    branch_ref: String,
266}
267
268#[derive(Debug, Deserialize)]
269struct GithubPullRequestHeadApiResponse {
270    sha: String,
271}
272
273#[derive(Debug, Deserialize)]
274struct GithubUserApiResponse {
275    login: String,
276}
277
278#[derive(Debug, Deserialize)]
279struct GithubReviewApiResponse {
280    state: String,
281    #[serde(rename = "submitted_at")]
282    submitted_at: Option<String>,
283    user: Option<GithubUserApiResponse>,
284}
285
286#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
287struct RemoteProjectRegistryFile {
288    version: u8,
289    projects: BTreeMap<String, RemoteProjectRegistryEntry>,
290}
291
292#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
293struct RemoteProjectRegistryEntry {
294    #[serde(rename = "checkoutPath")]
295    checkout_path: String,
296    #[serde(rename = "forkGitUrl")]
297    fork_git_url: String,
298    #[serde(rename = "repoUrl")]
299    repo_url: String,
300    #[serde(rename = "gitUrl")]
301    git_url: String,
302    #[serde(rename = "baseBranch")]
303    base_branch: String,
304    #[serde(rename = "updatedAt")]
305    updated_at: String,
306}
307
308impl Default for RemoteProjectRegistryFile {
309    fn default() -> Self {
310        Self {
311            version: 1,
312            projects: BTreeMap::new(),
313        }
314    }
315}
316
317pub struct RemoteDispatchService<'a> {
318    pub config_service: &'a RemoteAgentConfigService,
319    pub dispatch_repository: &'a DispatchRepository,
320    pub project_repository: &'a ProjectRepository,
321    pub task_repository: &'a FileTaskRepository,
322    pub review_repository: &'a ReviewRepository,
323    pub review_dispatch_repository: &'a ReviewDispatchRepository,
324}
325
326pub struct RemoteReviewService<'a> {
327    pub config_service: &'a RemoteAgentConfigService,
328    pub project_repository: &'a ProjectRepository,
329    pub review_repository: &'a ReviewRepository,
330    pub review_dispatch_repository: &'a ReviewDispatchRepository,
331}
332
333impl<'a> RemoteDispatchService<'a> {
334    fn review_service(&self) -> RemoteReviewService<'_> {
335        RemoteReviewService {
336            config_service: self.config_service,
337            project_repository: self.project_repository,
338            review_repository: self.review_repository,
339            review_dispatch_repository: self.review_dispatch_repository,
340        }
341    }
342
343    // =============================================================================
344    // Remote Dispatch Entry Points
345    // =============================================================================
346    //
347    // Dispatch orchestration is split into two phases so the API can persist a
348    // visible "preparing environment" state immediately. The browser receives
349    // that state right away, while the slower SSH/bootstrap work continues in
350    // the background and later transitions the record into `running` or a
351    // terminal outcome.
352    pub fn queue_dispatch(
353        &self,
354        task_id: &str,
355        preferred_tool: Option<RemoteAgentPreferredTool>,
356    ) -> Result<TaskDispatchRecord, TrackError> {
357        let (remote_agent, task, _project_metadata) = self.load_dispatch_prerequisites(task_id)?;
358        let _dispatch_start_guard = TaskDispatchStartGuard::acquire(task_id);
359        self.ensure_no_blocking_active_dispatch(task_id)?;
360        let preferred_tool = preferred_tool.unwrap_or(remote_agent.preferred_tool);
361
362        let mut dispatch_record =
363            self.dispatch_repository
364                .create_dispatch(&task, &remote_agent.host, preferred_tool)?;
365        dispatch_record.branch_name = Some(format!("track/{}", dispatch_record.dispatch_id));
366        dispatch_record.worktree_path = Some(format!(
367            "{}/{}/worktrees/{}",
368            remote_agent.workspace_root.trim_end_matches('/'),
369            task.project,
370            dispatch_record.dispatch_id
371        ));
372        dispatch_record.updated_at = now_utc();
373        self.dispatch_repository.save_dispatch(&dispatch_record)?;
374
375        Ok(dispatch_record)
376    }
377
378    // =============================================================================
379    // Follow-Up Dispatches
380    // =============================================================================
381    //
382    // A follow-up continues an earlier remote attempt instead of starting from
383    // scratch. We keep the previous branch/worktree/PR context, append the new
384    // user request into the task Markdown for auditability, and store the
385    // latest follow-up request directly on the dispatch record so prompt
386    // generation can highlight the newest ask explicitly.
387    pub fn queue_follow_up_dispatch(
388        &self,
389        task_id: &str,
390        follow_up_request: &str,
391    ) -> Result<TaskDispatchRecord, TrackError> {
392        let trimmed_follow_up_request = follow_up_request.trim();
393        if trimmed_follow_up_request.is_empty() {
394            return Err(TrackError::new(
395                ErrorCode::EmptyInput,
396                "Please provide a follow-up request for the remote agent.",
397            ));
398        }
399
400        let (remote_agent, _task, _project_metadata) = self.load_dispatch_prerequisites(task_id)?;
401        let _dispatch_start_guard = TaskDispatchStartGuard::acquire(task_id);
402        self.ensure_no_blocking_active_dispatch(task_id)?;
403
404        let dispatch_history = self.dispatch_repository.dispatches_for_task(task_id)?;
405        let previous_dispatch = select_follow_up_base_dispatch(&dispatch_history)
406            .ok_or_else(|| {
407                TrackError::new(
408                    ErrorCode::DispatchNotFound,
409                    format!(
410                        "Task {task_id} does not have a previous reusable remote dispatch to follow up on."
411                    ),
412                )
413            })?;
414        let branch_name = previous_dispatch.branch_name.clone().ok_or_else(|| {
415            TrackError::new(
416                ErrorCode::DispatchNotFound,
417                format!(
418                    "Task {task_id} does not have a reusable branch from the previous remote dispatch."
419                ),
420            )
421        })?;
422        let worktree_path = previous_dispatch.worktree_path.clone().ok_or_else(|| {
423            TrackError::new(
424                ErrorCode::DispatchNotFound,
425                format!(
426                    "Task {task_id} does not have a reusable worktree from the previous remote dispatch."
427                ),
428            )
429        })?;
430
431        let updated_task =
432            self.append_follow_up_request_to_task(task_id, trimmed_follow_up_request)?;
433        let mut dispatch_record = self.dispatch_repository.create_dispatch(
434            &updated_task,
435            &remote_agent.host,
436            previous_dispatch.preferred_tool,
437        )?;
438        dispatch_record.branch_name = Some(branch_name);
439        dispatch_record.worktree_path = Some(worktree_path);
440        dispatch_record.pull_request_url = latest_pull_request_for_branch(
441            &dispatch_history,
442            dispatch_record
443                .branch_name
444                .as_deref()
445                .expect("follow-up dispatches should always have a branch name"),
446        )
447        .or(previous_dispatch.pull_request_url.clone());
448        dispatch_record.follow_up_request = Some(trimmed_follow_up_request.to_owned());
449        dispatch_record.review_request_head_oid = previous_dispatch.review_request_head_oid.clone();
450        dispatch_record.review_request_user = previous_dispatch.review_request_user.clone();
451        dispatch_record.summary = Some(format!(
452            "Follow-up request: {}",
453            first_follow_up_line(trimmed_follow_up_request)
454        ));
455        dispatch_record.updated_at = now_utc();
456        self.dispatch_repository.save_dispatch(&dispatch_record)?;
457
458        Ok(dispatch_record)
459    }
460
461    pub fn launch_prepared_dispatch(
462        &self,
463        mut dispatch_record: TaskDispatchRecord,
464    ) -> Result<TaskDispatchRecord, TrackError> {
465        if let Some(existing_record) =
466            self.load_saved_dispatch(&dispatch_record.task_id, &dispatch_record.dispatch_id)?
467        {
468            if !existing_record.status.is_active() {
469                return Ok(existing_record);
470            }
471        }
472
473        let worktree_path = dispatch_record
474            .worktree_path
475            .clone()
476            .expect("queued dispatches should always store a worktree path");
477        let branch_name = dispatch_record
478            .branch_name
479            .clone()
480            .expect("queued dispatches should always store a branch name");
481        let remote_run_directory =
482            derive_remote_run_directory(&worktree_path, &dispatch_record.dispatch_id)?;
483
484        let launch_result = (|| -> Result<(), TrackError> {
485            if !self.save_preparing_phase(
486                &mut dispatch_record,
487                "Checking remote agent prerequisites.",
488            )? {
489                return Ok(());
490            }
491            let (remote_agent, task, project_metadata) =
492                self.load_dispatch_prerequisites(&dispatch_record.task_id)?;
493            let ssh_client = SshClient::new(&remote_agent)?;
494            if !self.save_preparing_phase(
495                &mut dispatch_record,
496                "Loading the remote project registry.",
497            )? {
498                return Ok(());
499            }
500            let remote_registry =
501                load_remote_registry(&ssh_client, &remote_agent.projects_registry_path)?;
502            if !self.save_preparing_phase(
503                &mut dispatch_record,
504                "Checking GitHub authentication on the remote machine.",
505            )? {
506                return Ok(());
507            }
508            let github_login = ssh_client.fetch_github_login()?;
509            let repository_name = parse_github_repository_name(&project_metadata.repo_url)?;
510            let checkout_path = remote_registry
511                .projects
512                .get(&task.project)
513                .map(|entry| entry.checkout_path.clone())
514                .unwrap_or_else(|| {
515                    format!(
516                        "{}/{}/{}",
517                        remote_agent.workspace_root.trim_end_matches('/'),
518                        task.project,
519                        task.project
520                    )
521                });
522
523            if !self.save_preparing_phase(
524                &mut dispatch_record,
525                "Ensuring the remote checkout is up to date.",
526            )? {
527                return Ok(());
528            }
529            let fork_git_url = ssh_client.ensure_checkout(
530                &project_metadata,
531                &repository_name,
532                &checkout_path,
533                &github_login,
534            )?;
535
536            let mut updated_registry = remote_registry;
537            updated_registry.projects.insert(
538                task.project.clone(),
539                RemoteProjectRegistryEntry {
540                    checkout_path: checkout_path.clone(),
541                    fork_git_url: fork_git_url.clone(),
542                    repo_url: project_metadata.repo_url.clone(),
543                    git_url: project_metadata.git_url.clone(),
544                    base_branch: project_metadata.base_branch.clone(),
545                    updated_at: format_iso_8601_millis(now_utc()),
546                },
547            );
548            write_remote_registry(
549                &ssh_client,
550                &remote_agent.projects_registry_path,
551                &updated_registry,
552            )?;
553
554            if !self.save_preparing_phase(&mut dispatch_record, "Preparing the task worktree.")? {
555                return Ok(());
556            }
557            if dispatch_record.follow_up_request.is_some() {
558                ssh_client.ensure_follow_up_worktree(
559                    &checkout_path,
560                    &branch_name,
561                    &worktree_path,
562                )?;
563            } else {
564                ssh_client.create_worktree(
565                    &checkout_path,
566                    &project_metadata.base_branch,
567                    &branch_name,
568                    &worktree_path,
569                )?;
570            }
571
572            let prompt = build_remote_dispatch_prompt(
573                &task.project,
574                &project_metadata,
575                &branch_name,
576                &worktree_path,
577                &task.description,
578                dispatch_record.pull_request_url.as_deref(),
579                dispatch_record.follow_up_request.as_deref(),
580            );
581            let schema = build_remote_dispatch_schema();
582            if !self.save_preparing_phase(
583                &mut dispatch_record,
584                "Uploading the agent prompt and schema.",
585            )? {
586                return Ok(());
587            }
588            ssh_client.upload_remote_file(
589                &format!("{remote_run_directory}/{REMOTE_PROMPT_FILE_NAME}"),
590                &prompt,
591            )?;
592            ssh_client.upload_remote_file(
593                &format!("{remote_run_directory}/{REMOTE_SCHEMA_FILE_NAME}"),
594                &schema,
595            )?;
596
597            // Cancellation can arrive while the API is still preparing the
598            // remote checkout. We re-read the persisted record right before the
599            // expensive remote-agent launch so a user-triggered cancel can stop
600            // the flow before it starts spending more tokens.
601            if !self
602                .dispatch_is_still_active(&dispatch_record.task_id, &dispatch_record.dispatch_id)?
603            {
604                return Ok(());
605            }
606
607            if !self.save_preparing_phase(&mut dispatch_record, "Launching the remote agent.")? {
608                return Ok(());
609            }
610            ssh_client.launch_remote_dispatch(
611                &remote_run_directory,
612                &worktree_path,
613                dispatch_record.preferred_tool,
614            )?;
615
616            Ok(())
617        })();
618
619        match launch_result {
620            Ok(()) => {
621                if let Some(existing_record) = self
622                    .load_saved_dispatch(&dispatch_record.task_id, &dispatch_record.dispatch_id)?
623                {
624                    if !existing_record.status.is_active() {
625                        let _ = self.cancel_remote_dispatch_if_possible(&existing_record);
626                        return Ok(existing_record);
627                    }
628                }
629
630                dispatch_record.status = DispatchStatus::Running;
631                dispatch_record.updated_at = now_utc();
632                dispatch_record.finished_at = None;
633                dispatch_record.summary =
634                    Some("The remote agent is working in the prepared environment.".to_owned());
635                dispatch_record.error_message = None;
636                self.dispatch_repository.save_dispatch(&dispatch_record)?;
637                Ok(dispatch_record)
638            }
639            Err(error) => {
640                dispatch_record.status = DispatchStatus::Failed;
641                dispatch_record.updated_at = now_utc();
642                dispatch_record.finished_at = Some(dispatch_record.updated_at);
643                dispatch_record.error_message = Some(error.to_string());
644                self.dispatch_repository.save_dispatch(&dispatch_record)?;
645                Err(error)
646            }
647        }
648    }
649
650    // =============================================================================
651    // Dispatch Cancellation And Discard
652    // =============================================================================
653    //
654    // Cancellation and discard solve two different user intents:
655    //
656    // 1. "stop spending resources on this run" -> cancel the latest active run
657    // 2. "forget the previous outcome and let me try again cleanly" -> discard
658    //    the saved dispatch history for the task
659    //
660    // We keep them separate so the UI can expose both actions without
661    // overloading one button with two meanings.
662    pub fn cancel_dispatch(&self, task_id: &str) -> Result<TaskDispatchRecord, TrackError> {
663        let mut latest_dispatch = self
664            .latest_dispatches_for_tasks(&[task_id.to_owned()])?
665            .into_iter()
666            .next()
667            .ok_or_else(|| {
668                TrackError::new(
669                    ErrorCode::DispatchNotFound,
670                    format!("Task {task_id} does not have a remote dispatch to cancel."),
671                )
672            })?;
673
674        if !latest_dispatch.status.is_active() {
675            return Err(TrackError::new(
676                ErrorCode::DispatchNotFound,
677                format!("Task {task_id} does not have an active remote dispatch to cancel."),
678            ));
679        }
680
681        self.cancel_remote_dispatch_if_possible(&latest_dispatch)?;
682
683        latest_dispatch.status = DispatchStatus::Canceled;
684        latest_dispatch.updated_at = now_utc();
685        latest_dispatch.finished_at = Some(latest_dispatch.updated_at);
686        latest_dispatch.summary = Some("Canceled from the web UI.".to_owned());
687        latest_dispatch.notes = None;
688        latest_dispatch.error_message = None;
689        self.dispatch_repository.save_dispatch(&latest_dispatch)?;
690
691        Ok(latest_dispatch)
692    }
693
694    pub fn discard_dispatch_history(&self, task_id: &str) -> Result<(), TrackError> {
695        let latest_dispatch = self
696            .dispatch_repository
697            .latest_dispatch_for_task(task_id)?
698            .ok_or_else(|| {
699                TrackError::new(
700                    ErrorCode::DispatchNotFound,
701                    format!("Task {task_id} does not have a remote dispatch to discard."),
702                )
703            })?;
704
705        if latest_dispatch.status.is_active() {
706            return Err(TrackError::new(
707                ErrorCode::RemoteDispatchFailed,
708                "Cancel the active remote dispatch before discarding its history.",
709            ));
710        }
711
712        // Discard intentionally clears the entire visible dispatch history for
713        // the task so the card goes back to an undecorated state. That matches
714        // the UI intent of "let me try this task again from a clean slate".
715        // TODO: This currently leaves remote worktrees and dispatch directories
716        // alone on purpose. Product policy only asked for remote cleanup on
717        // task close/delete, not on discard.
718        // TODO: If users later want audit history, replace this hard delete
719        // with an explicit archived-history concept instead of reviving older
720        // dispatch records automatically.
721        self.dispatch_repository
722            .delete_dispatch_history_for_task(task_id)
723    }
724
725    // =============================================================================
726    // Task Lifecycle Cleanup
727    // =============================================================================
728    //
729    // Remote agent work leaves behind two different kinds of state:
730    //
731    // 1. lightweight metadata we want to keep for run history and follow-ups
732    // 2. heavyweight worktrees that can accumulate large Rust build outputs
733    //
734    // Closing a task should therefore keep dispatch history but release the
735    // worktree space. Deleting a task goes further and removes both the local
736    // dispatch history and the remote run directories as well.
737    // We intentionally leave branches and the shared project checkout in
738    // place. The heavy cost is in per-task worktrees and their build outputs,
739    // while branches and the reusable checkout are comparatively cheap and
740    // valuable for follow-up work.
741    pub fn update_task(
742        &self,
743        task_id: &str,
744        input: TaskUpdateInput,
745    ) -> Result<crate::types::Task, TrackError> {
746        let validated_input = input.validate()?;
747
748        if validated_input.status == Some(crate::types::Status::Closed) {
749            let dispatch_history = self.dispatch_repository.dispatches_for_task(task_id)?;
750            if !dispatch_history.is_empty() {
751                let cleanup_result = self.cleanup_task_remote_artifacts(
752                    task_id,
753                    &dispatch_history,
754                    RemoteTaskCleanupMode::CloseTask,
755                );
756
757                // The tracker should stay usable even if the remote machine,
758                // SSH key, or remote config disappears. Closing the task is a
759                // local filesystem mutation first; remote cleanup is only a
760                // best-effort follow-up.
761                match cleanup_result {
762                    Ok(_) => self.finalize_active_dispatches_locally(
763                        &dispatch_history,
764                        DispatchStatus::Canceled,
765                        "Canceled because the task was closed.",
766                        None,
767                    )?,
768                    Err(error) => {
769                        eprintln!("Skipping remote cleanup while closing task {task_id}: {error}");
770                        self.finalize_active_dispatches_locally(
771                            &dispatch_history,
772                            DispatchStatus::Canceled,
773                            "Canceled because the task was closed locally. Remote cleanup was skipped.",
774                            Some(error.message()),
775                        )?;
776                    }
777                }
778            }
779        }
780
781        self.task_repository.update_task(task_id, validated_input)
782    }
783
784    pub fn delete_task(&self, task_id: &str) -> Result<(), TrackError> {
785        let dispatch_history = self.dispatch_repository.dispatches_for_task(task_id)?;
786        if !dispatch_history.is_empty() {
787            if let Err(error) = self.cleanup_task_remote_artifacts(
788                task_id,
789                &dispatch_history,
790                RemoteTaskCleanupMode::DeleteTask,
791            ) {
792                // Delete is the strongest local intent: once the user removes a
793                // task, stale remote artifacts must not veto that choice.
794                // TODO: We intentionally do not persist this warning locally
795                // because delete also removes the task's local dispatch history.
796                eprintln!("Skipping remote cleanup while deleting task {task_id}: {error}");
797            }
798
799            // We intentionally remove the local dispatch history before the
800            // task file itself. If the final file delete fails, the user still
801            // sees the task and can retry, rather than ending up with invisible
802            // orphaned runs in the UI.
803            self.dispatch_repository
804                .delete_dispatch_history_for_task(task_id)?;
805        }
806
807        self.task_repository.delete_task(task_id)
808    }
809
810    // =============================================================================
811    // Manual Remote Cleanup
812    // =============================================================================
813    //
814    // The lifecycle hooks on close/delete protect new work from leaking
815    // worktrees forever, but users may already have historical leftovers from
816    // before that policy existed. Manual cleanup replays the same rules across
817    // the whole tracker state:
818    //
819    // - open task: keep referenced worktrees and dispatch metadata
820    // - closed task: remove worktrees, keep dispatch metadata
821    // - persisted review: keep local review history, but only keep remote
822    //   artifacts while the review run is still active
823    // - missing task/review: remove remote artifacts and local dispatch history
824    //
825    // After reconciling every saved history, we also sweep the remote
826    // workspace for orphaned task/review worktrees and run directories that no
827    // longer have any local record at all. Review-only checkouts are also
828    // treated as disposable cache during this explicit maintenance path
829    // because reviews currently do not support rerun or resume from an old
830    // worktree.
831    // TODO: Revisit review checkout cleanup if manual reviews ever gain
832    // rerun/reopen behavior that should preserve those remote caches.
833    pub fn cleanup_unused_remote_artifacts(&self) -> Result<RemoteCleanupSummary, TrackError> {
834        let remote_agent = self.load_remote_agent_for_global_cleanup()?;
835        let ssh_client = SshClient::new(&remote_agent)?;
836        let task_ids_with_history = self.dispatch_repository.task_ids_with_history()?;
837        let review_ids_with_history = self.review_dispatch_repository.review_ids_with_history()?;
838        let tracked_project_names = self
839            .project_repository
840            .list_projects()?
841            .into_iter()
842            .map(|project| project.canonical_name)
843            .collect::<BTreeSet<_>>();
844
845        let mut summary = RemoteCleanupSummary::default();
846        let mut kept_worktree_paths = BTreeSet::new();
847        let mut kept_run_directories = BTreeSet::new();
848        let mut review_workspace_keys = BTreeSet::new();
849        let mut active_review_workspace_keys = BTreeSet::new();
850
851        for task_id in task_ids_with_history {
852            let dispatch_history = self.dispatch_repository.dispatches_for_task(&task_id)?;
853            if dispatch_history.is_empty() {
854                continue;
855            }
856
857            match self.task_repository.get_task(&task_id) {
858                Ok(task) if task.status == Status::Open => {
859                    kept_worktree_paths.extend(unique_worktree_paths(&dispatch_history));
860                    kept_run_directories
861                        .extend(unique_run_directories(&dispatch_history, &remote_agent));
862                }
863                Ok(task) if task.status == Status::Closed => {
864                    let cleanup_counts = self.cleanup_task_remote_artifacts(
865                        &task.id,
866                        &dispatch_history,
867                        RemoteTaskCleanupMode::CloseTask,
868                    )?;
869                    self.finalize_active_dispatches_locally(
870                        &dispatch_history,
871                        DispatchStatus::Canceled,
872                        "Canceled because the task was closed.",
873                        None,
874                    )?;
875                    kept_run_directories
876                        .extend(unique_run_directories(&dispatch_history, &remote_agent));
877                    summary.closed_tasks_cleaned += 1;
878                    summary.remote_worktrees_removed += cleanup_counts.worktrees_removed;
879                    summary.remote_run_directories_removed +=
880                        cleanup_counts.run_directories_removed;
881                }
882                Err(error) if error.code == ErrorCode::TaskNotFound => {
883                    let cleanup_counts = self.cleanup_task_remote_artifacts(
884                        &task_id,
885                        &dispatch_history,
886                        RemoteTaskCleanupMode::DeleteTask,
887                    )?;
888                    self.dispatch_repository
889                        .delete_dispatch_history_for_task(&task_id)?;
890                    summary.missing_tasks_cleaned += 1;
891                    summary.local_dispatch_histories_removed += 1;
892                    summary.remote_worktrees_removed += cleanup_counts.worktrees_removed;
893                    summary.remote_run_directories_removed +=
894                        cleanup_counts.run_directories_removed;
895                }
896                Err(error) => return Err(error),
897                Ok(_) => unreachable!("tasks should only be open or closed"),
898            }
899        }
900
901        for review_id in review_ids_with_history {
902            let dispatch_history = self
903                .review_dispatch_repository
904                .dispatches_for_review(&review_id)?;
905            if dispatch_history.is_empty() {
906                continue;
907            }
908
909            let workspace_key = dispatch_history[0].workspace_key.clone();
910            review_workspace_keys.insert(workspace_key.clone());
911
912            match self.review_repository.get_review(&review_id) {
913                Ok(_) => {
914                    let active_dispatch_history = dispatch_history
915                        .iter()
916                        .filter(|record| record.status.is_active())
917                        .cloned()
918                        .collect::<Vec<_>>();
919                    if !active_dispatch_history.is_empty() {
920                        kept_worktree_paths
921                            .extend(unique_review_worktree_paths(&active_dispatch_history));
922                        kept_run_directories.extend(unique_review_run_directories(
923                            &active_dispatch_history,
924                            &remote_agent,
925                        ));
926                        active_review_workspace_keys.insert(workspace_key);
927                    }
928                }
929                Err(error) if error.code == ErrorCode::TaskNotFound => {
930                    self.review_dispatch_repository
931                        .delete_dispatch_history_for_review(&review_id)?;
932                    summary.local_dispatch_histories_removed += 1;
933                }
934                Err(error) => return Err(error),
935            }
936        }
937
938        let orphan_cleanup_counts = ssh_client.cleanup_orphaned_remote_artifacts(
939            &remote_agent.workspace_root,
940            &kept_worktree_paths.into_iter().collect::<Vec<_>>(),
941            &kept_run_directories.into_iter().collect::<Vec<_>>(),
942        )?;
943        summary.remote_worktrees_removed += orphan_cleanup_counts.worktrees_removed;
944        summary.remote_run_directories_removed += orphan_cleanup_counts.run_directories_removed;
945
946        let reclaimable_review_workspace_keys = review_workspace_keys
947            .into_iter()
948            .filter(|workspace_key| {
949                !tracked_project_names.contains(workspace_key)
950                    && !active_review_workspace_keys.contains(workspace_key)
951            })
952            .collect::<Vec<_>>();
953        self.cleanup_reclaimable_review_workspaces(
954            &ssh_client,
955            &remote_agent,
956            &reclaimable_review_workspace_keys,
957        )?;
958
959        Ok(summary)
960    }
961
962    // =============================================================================
963    // Full Remote Workspace Reset
964    // =============================================================================
965    //
966    // Manual cleanup reconciles remote state against local truth. Reset is the
967    // explicit escape hatch for the harder case where the remote machine is no
968    // longer trustworthy and the user wants to start that environment from
969    // scratch.
970    //
971    // We intentionally keep local task files and local dispatch history.
972    // Those remain the durable tracker state. The remote workspace root and
973    // the remote projects registry are treated as rebuildable cache.
974    pub fn reset_remote_workspace(&self) -> Result<RemoteResetSummary, TrackError> {
975        let active_task_dispatches = self.list_dispatches(None)?;
976        let active_review_dispatches = self.review_service().list_dispatches(None)?;
977        let active_dispatches =
978            describe_remote_reset_blockers(&active_task_dispatches, &active_review_dispatches);
979        if !active_dispatches.is_empty() {
980            return Err(TrackError::new(
981                ErrorCode::RemoteDispatchFailed,
982                format!(
983                    "Stop active remote task runs and PR reviews before resetting the remote workspace: {}.",
984                    active_dispatches.join(", ")
985                ),
986            ));
987        }
988
989        let remote_agent = self.load_remote_agent_for_global_cleanup()?;
990        let ssh_client = SshClient::new(&remote_agent)?;
991        ssh_client.reset_workspace(
992            &remote_agent.workspace_root,
993            &remote_agent.projects_registry_path,
994        )
995    }
996
997    pub fn latest_dispatches_for_tasks(
998        &self,
999        task_ids: &[String],
1000    ) -> Result<Vec<TaskDispatchRecord>, TrackError> {
1001        let records = self
1002            .dispatch_repository
1003            .latest_dispatches_for_tasks(task_ids)?;
1004        self.refresh_active_dispatch_records(records)
1005    }
1006
1007    // =============================================================================
1008    // Global Dispatch History
1009    // =============================================================================
1010    //
1011    // The frontend's Runs page needs the same "what is the remote machine
1012    // saying right now?" view as the task-level dispatch badges. We therefore
1013    // route the global history listing through the same refresh path instead of
1014    // reading raw JSON records and leaving active runs stale until some other
1015    // endpoint happens to reconcile them.
1016    pub fn list_dispatches(
1017        &self,
1018        limit: Option<usize>,
1019    ) -> Result<Vec<TaskDispatchRecord>, TrackError> {
1020        let records = self.dispatch_repository.list_dispatches(limit)?;
1021        self.refresh_active_dispatch_records(records)
1022    }
1023
1024    // The task drawer needs authoritative history for the selected task even
1025    // when the global Runs page is intentionally truncated for UI cost. We
1026    // therefore expose a task-scoped history path that keeps older tasks from
1027    // losing their latest status or drawer history just because newer runs
1028    // pushed them past the global limit.
1029    pub fn dispatch_history_for_task(
1030        &self,
1031        task_id: &str,
1032    ) -> Result<Vec<TaskDispatchRecord>, TrackError> {
1033        let mut records = self.dispatch_repository.dispatches_for_task(task_id)?;
1034
1035        // At most one dispatch should be active per task. If the newest record
1036        // is still active, route it through the same remote reconciliation path
1037        // as the queue badges so the drawer sees current state instead of raw
1038        // persisted JSON.
1039        if records
1040            .first()
1041            .is_some_and(|record| record.status.is_active())
1042        {
1043            if let Some(refreshed_latest) = self
1044                .latest_dispatches_for_tasks(&[task_id.to_owned()])?
1045                .into_iter()
1046                .next()
1047            {
1048                if let Some(first_record) = records.first_mut() {
1049                    *first_record = refreshed_latest;
1050                }
1051            }
1052        }
1053
1054        Ok(records)
1055    }
1056
1057    // =============================================================================
1058    // Review Follow-Up Reconciliation
1059    // =============================================================================
1060    //
1061    // The review automation stays intentionally narrow for now:
1062    //
1063    // - mention the configured `mainUser` on the PR after a PR head changes
1064    // - queue one follow-up when that same user leaves actionable review feedback
1065    //
1066    // We treat review submissions as "newer than the bot run" when their
1067    // timestamp is after the dispatch `created_at`. That is conservative on
1068    // purpose: if review feedback lands while a bot run is already active, we
1069    // would rather schedule one extra follow-up than silently miss the human
1070    // feedback entirely.
1071    pub fn reconcile_review_follow_up(
1072        &self,
1073    ) -> Result<RemoteReviewFollowUpReconciliation, TrackError> {
1074        let remote_agent = match self.config_service.load_remote_agent_runtime_config() {
1075            Ok(config) => config,
1076            Err(error)
1077                if matches!(
1078                    error.code,
1079                    ErrorCode::ConfigNotFound
1080                        | ErrorCode::InvalidConfig
1081                        | ErrorCode::InvalidRemoteAgentConfig
1082                ) =>
1083            {
1084                return Ok(RemoteReviewFollowUpReconciliation::default());
1085            }
1086            Err(error) => return Err(error),
1087        };
1088        let Some(remote_agent) = remote_agent else {
1089            return Ok(RemoteReviewFollowUpReconciliation::default());
1090        };
1091        let Some(review_follow_up) = remote_agent.review_follow_up.clone() else {
1092            return Ok(RemoteReviewFollowUpReconciliation::default());
1093        };
1094        if !remote_agent.managed_key_path.exists() {
1095            return Ok(RemoteReviewFollowUpReconciliation::default());
1096        }
1097
1098        let task_ids = self.dispatch_repository.task_ids_with_history()?;
1099        if task_ids.is_empty() {
1100            return Ok(RemoteReviewFollowUpReconciliation::default());
1101        }
1102
1103        let latest_dispatches = self.latest_dispatches_for_tasks(&task_ids)?;
1104        let ssh_client = SshClient::new(&remote_agent)?;
1105        let mut reconciliation = RemoteReviewFollowUpReconciliation::default();
1106
1107        for dispatch_record in latest_dispatches {
1108            let Some(pull_request_url) = dispatch_record
1109                .pull_request_url
1110                .as_deref()
1111                .map(str::trim)
1112                .filter(|value| !value.is_empty())
1113            else {
1114                continue;
1115            };
1116
1117            match self.task_repository.get_task(&dispatch_record.task_id) {
1118                Ok(task) if task.status == Status::Open => task,
1119                Ok(_) => continue,
1120                Err(error) if error.code == ErrorCode::TaskNotFound => continue,
1121                Err(error) => return Err(error),
1122            };
1123
1124            let pull_request_state = ssh_client
1125                .fetch_pull_request_review_state(pull_request_url, &review_follow_up.main_user)
1126                .map_err(|error| {
1127                    contextualize_track_error(
1128                        error,
1129                        format!(
1130                            "Review follow-up could not inspect task {} PR {} for reviewer @{}",
1131                            dispatch_record.task_id, pull_request_url, review_follow_up.main_user
1132                        ),
1133                    )
1134                });
1135            let pull_request_state = match pull_request_state {
1136                Ok(pull_request_state) => pull_request_state,
1137                Err(error) => {
1138                    reconciliation.failures += 1;
1139                    reconciliation.events.push(review_follow_up_event(
1140                        "fetch_failed",
1141                        error.to_string(),
1142                        &dispatch_record,
1143                        &review_follow_up.main_user,
1144                        None,
1145                    ));
1146                    continue;
1147                }
1148            };
1149
1150            reconciliation.events.push(review_follow_up_event(
1151                "task_evaluated",
1152                "Fetched PR review state for automatic follow-up reconciliation.",
1153                &dispatch_record,
1154                &review_follow_up.main_user,
1155                Some(&pull_request_state),
1156            ));
1157            if !pull_request_state.is_open {
1158                reconciliation.events.push(review_follow_up_event(
1159                    "skip_closed_pr",
1160                    "Skipped automatic follow-up because the PR is not open anymore.",
1161                    &dispatch_record,
1162                    &review_follow_up.main_user,
1163                    Some(&pull_request_state),
1164                ));
1165                continue;
1166            }
1167
1168            if dispatch_record.status.is_active() {
1169                reconciliation.events.push(review_follow_up_event(
1170                    "skip_active_dispatch",
1171                    "Skipped automatic follow-up because the latest dispatch is still active.",
1172                    &dispatch_record,
1173                    &review_follow_up.main_user,
1174                    Some(&pull_request_state),
1175                ));
1176                continue;
1177            }
1178
1179            if let Some(latest_review) = pull_request_state.latest_eligible_review.as_ref() {
1180                if latest_review.submitted_at > dispatch_record.created_at {
1181                    let follow_up_request = build_review_follow_up_request(
1182                        pull_request_url,
1183                        &review_follow_up.main_user,
1184                        dispatch_record.created_at,
1185                    );
1186                    let queued_dispatch = self
1187                        .queue_follow_up_dispatch(&dispatch_record.task_id, &follow_up_request)?;
1188                    reconciliation.events.push(review_follow_up_event(
1189                        "queue_follow_up",
1190                        format!(
1191                            "Queued a follow-up dispatch because reviewer @{} submitted {} at {} after dispatch {} started.",
1192                            review_follow_up.main_user,
1193                            latest_review.state,
1194                            format_iso_8601_millis(latest_review.submitted_at),
1195                            dispatch_record.dispatch_id,
1196                        ),
1197                        &queued_dispatch,
1198                        &review_follow_up.main_user,
1199                        Some(&pull_request_state),
1200                    ));
1201                    reconciliation.queued_dispatches.push(queued_dispatch);
1202                    continue;
1203                }
1204            }
1205
1206            if pull_request_state.head_oid.is_empty() {
1207                reconciliation.events.push(review_follow_up_event(
1208                    "skip_missing_head_oid",
1209                    "Skipped PR reviewer notification because the PR head SHA is missing.",
1210                    &dispatch_record,
1211                    &review_follow_up.main_user,
1212                    Some(&pull_request_state),
1213                ));
1214                continue;
1215            }
1216
1217            let already_recorded_for_head = dispatch_record.review_request_head_oid.as_deref()
1218                == Some(pull_request_state.head_oid.as_str())
1219                && dispatch_record.review_request_user.as_deref()
1220                    == Some(review_follow_up.main_user.as_str());
1221            if already_recorded_for_head {
1222                reconciliation.events.push(review_follow_up_event(
1223                    "skip_notification_already_recorded",
1224                    "Skipped PR reviewer notification because this PR head already recorded the same reviewer.",
1225                    &dispatch_record,
1226                    &review_follow_up.main_user,
1227                    Some(&pull_request_state),
1228                ));
1229                continue;
1230            }
1231
1232            let notification_comment = build_review_follow_up_notification_comment(
1233                &review_follow_up.main_user,
1234                &pull_request_state.head_oid,
1235            );
1236            let notify_reviewer_result = ssh_client
1237                .post_pull_request_comment(pull_request_url, &notification_comment)
1238                .map_err(|error| {
1239                    contextualize_track_error(
1240                        error,
1241                        format!(
1242                            "Review follow-up could not notify reviewer @{} for task {} PR {}",
1243                            review_follow_up.main_user, dispatch_record.task_id, pull_request_url
1244                        ),
1245                    )
1246                });
1247            if let Err(error) = notify_reviewer_result {
1248                reconciliation.failures += 1;
1249                reconciliation.events.push(review_follow_up_event(
1250                    "notify_reviewer_failed",
1251                    error.to_string(),
1252                    &dispatch_record,
1253                    &review_follow_up.main_user,
1254                    Some(&pull_request_state),
1255                ));
1256                continue;
1257            }
1258            self.mark_review_notification_for_head(
1259                &dispatch_record,
1260                &pull_request_state.head_oid,
1261                &review_follow_up.main_user,
1262            )?;
1263            reconciliation.events.push(review_follow_up_event(
1264                "notify_reviewer_posted",
1265                "Posted a PR comment mentioning the configured main GitHub user for the current PR head.",
1266                &dispatch_record,
1267                &review_follow_up.main_user,
1268                Some(&pull_request_state),
1269            ));
1270            reconciliation.review_notifications_updated += 1;
1271        }
1272
1273        Ok(reconciliation)
1274    }
1275
1276    fn refresh_active_dispatch_records(
1277        &self,
1278        records: Vec<TaskDispatchRecord>,
1279    ) -> Result<Vec<TaskDispatchRecord>, TrackError> {
1280        let remote_agent = match self.config_service.load_remote_agent_runtime_config() {
1281            Ok(config) => config,
1282            Err(error)
1283                if matches!(
1284                    error.code,
1285                    ErrorCode::ConfigNotFound
1286                        | ErrorCode::InvalidConfig
1287                        | ErrorCode::InvalidRemoteAgentConfig
1288                ) =>
1289            {
1290                let error_message = error.to_string();
1291                return self.release_active_dispatches_after_reconciliation_loss(
1292                    records,
1293                    "Remote reconciliation is unavailable locally, so active runs were released.",
1294                    &error_message,
1295                );
1296            }
1297            Err(error) => return Err(error),
1298        };
1299
1300        let Some(remote_agent) = remote_agent else {
1301            return self.release_active_dispatches_after_reconciliation_loss(
1302                records,
1303                "Remote reconciliation is unavailable locally, so active runs were released.",
1304                "Remote agent configuration is missing locally.",
1305            );
1306        };
1307        if !remote_agent.managed_key_path.exists() {
1308            let error_message = format!(
1309                "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
1310                collapse_home_path(&remote_agent.managed_key_path)
1311            );
1312            return self.release_active_dispatches_after_reconciliation_loss(
1313                records,
1314                "Remote reconciliation is unavailable locally, so active runs were released.",
1315                &error_message,
1316            );
1317        }
1318
1319        let ssh_client = SshClient::new(&remote_agent)?;
1320        let snapshots_by_dispatch_id = match load_dispatch_snapshots_for_records(
1321            &ssh_client,
1322            &records,
1323        ) {
1324            Ok(snapshots) => snapshots,
1325            Err(error) => {
1326                let error_message = error.to_string();
1327                return self.release_active_dispatches_after_reconciliation_loss(
1328                        records,
1329                        "Remote reconciliation could not reach the remote machine, so active runs were released locally.",
1330                        &error_message,
1331                    );
1332            }
1333        };
1334        let mut refreshed_records = Vec::with_capacity(records.len());
1335        for record in records {
1336            if !record.status.is_active() {
1337                refreshed_records.push(record);
1338                continue;
1339            }
1340
1341            let Some(snapshot) = snapshots_by_dispatch_id.get(&record.dispatch_id) else {
1342                if let Some(updated) = mark_abandoned_preparing_dispatch(record.clone()) {
1343                    self.dispatch_repository.save_dispatch(&updated)?;
1344                    refreshed_records.push(updated);
1345                } else {
1346                    let updated = self.finalize_dispatch_locally(
1347                        &record,
1348                        DispatchStatus::Blocked,
1349                        "Remote reconciliation could not find this run anymore, so it was released locally.",
1350                        Some("Remote dispatch snapshot is missing."),
1351                    )?;
1352                    refreshed_records.push(updated);
1353                }
1354                continue;
1355            };
1356
1357            match refresh_dispatch_record_from_snapshot(record.clone(), snapshot) {
1358                Ok(updated) => {
1359                    if updated != record {
1360                        self.dispatch_repository.save_dispatch(&updated)?;
1361                    }
1362                    refreshed_records.push(updated);
1363                }
1364                Err(error) => {
1365                    if let Some(updated) =
1366                        mark_terminal_refresh_failure(record.clone(), snapshot, &error)
1367                    {
1368                        self.dispatch_repository.save_dispatch(&updated)?;
1369                        refreshed_records.push(updated);
1370                    } else {
1371                        let error_message = error.to_string();
1372                        let updated = self.finalize_dispatch_locally(
1373                            &record,
1374                            DispatchStatus::Blocked,
1375                            "Remote reconciliation could not confirm this run, so it was released locally.",
1376                            Some(&error_message),
1377                        )?;
1378                        refreshed_records.push(updated);
1379                    }
1380                }
1381            }
1382        }
1383
1384        Ok(refreshed_records)
1385    }
1386
1387    // =============================================================================
1388    // Local Recovery When Remote Control Disappears
1389    // =============================================================================
1390    //
1391    // Remote agent runs are helpful, but they are not the source of truth. The
1392    // source of truth is still the local tracker on disk, and users need to be
1393    // able to keep closing, deleting, and retrying tasks even after the remote
1394    // machine has been replaced or the SSH setup has gone stale. These helpers
1395    // therefore turn "we can no longer inspect or clean the remote side" into
1396    // explicit local terminal records instead of leaving active dispatches stuck
1397    // forever.
1398    fn ensure_no_blocking_active_dispatch(&self, task_id: &str) -> Result<(), TrackError> {
1399        if let Some(existing_dispatch) = self
1400            .latest_dispatches_for_tasks(&[task_id.to_owned()])?
1401            .into_iter()
1402            .next()
1403            .filter(|record| record.status.is_active())
1404        {
1405            return Err(TrackError::new(
1406                ErrorCode::RemoteDispatchFailed,
1407                format!(
1408                    "Task {task_id} already has an active remote dispatch ({})",
1409                    existing_dispatch.dispatch_id
1410                ),
1411            ));
1412        }
1413
1414        Ok(())
1415    }
1416
1417    fn release_active_dispatches_after_reconciliation_loss(
1418        &self,
1419        records: Vec<TaskDispatchRecord>,
1420        summary: &str,
1421        error_message: &str,
1422    ) -> Result<Vec<TaskDispatchRecord>, TrackError> {
1423        let mut refreshed_records = Vec::with_capacity(records.len());
1424        for record in records {
1425            if record.status.is_active() {
1426                refreshed_records.push(self.finalize_dispatch_locally(
1427                    &record,
1428                    DispatchStatus::Blocked,
1429                    summary,
1430                    Some(error_message),
1431                )?);
1432            } else {
1433                refreshed_records.push(record);
1434            }
1435        }
1436
1437        Ok(refreshed_records)
1438    }
1439
1440    fn dispatch_is_still_active(
1441        &self,
1442        task_id: &str,
1443        dispatch_id: &str,
1444    ) -> Result<bool, TrackError> {
1445        Ok(self
1446            .load_saved_dispatch(task_id, dispatch_id)?
1447            .map(|record| record.status.is_active())
1448            .unwrap_or(false))
1449    }
1450
1451    fn load_saved_dispatch(
1452        &self,
1453        task_id: &str,
1454        dispatch_id: &str,
1455    ) -> Result<Option<TaskDispatchRecord>, TrackError> {
1456        self.dispatch_repository.get_dispatch(task_id, dispatch_id)
1457    }
1458
1459    fn cancel_remote_dispatch_if_possible(
1460        &self,
1461        dispatch_record: &TaskDispatchRecord,
1462    ) -> Result<(), TrackError> {
1463        let remote_agent = self
1464            .config_service
1465            .load_remote_agent_runtime_config()?
1466            .ok_or_else(|| {
1467            TrackError::new(
1468                ErrorCode::RemoteAgentNotConfigured,
1469                "Remote dispatch is not configured yet. Re-run `track` and add a remote agent host plus SSH key.",
1470            )
1471        })?;
1472
1473        if !remote_agent.managed_key_path.exists() {
1474            return Err(TrackError::new(
1475                ErrorCode::RemoteAgentNotConfigured,
1476                format!(
1477                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
1478                    collapse_home_path(&remote_agent.managed_key_path)
1479                ),
1480            ));
1481        }
1482
1483        let Some(worktree_path) = dispatch_record.worktree_path.as_deref() else {
1484            return Ok(());
1485        };
1486        let remote_run_directory =
1487            derive_remote_run_directory(worktree_path, &dispatch_record.dispatch_id)?;
1488        let ssh_client = SshClient::new(&remote_agent)?;
1489        ssh_client.cancel_remote_dispatch(&remote_run_directory)
1490    }
1491
1492    fn save_preparing_phase(
1493        &self,
1494        dispatch_record: &mut TaskDispatchRecord,
1495        summary: &str,
1496    ) -> Result<bool, TrackError> {
1497        if let Some(saved_record) =
1498            self.load_saved_dispatch(&dispatch_record.task_id, &dispatch_record.dispatch_id)?
1499        {
1500            if !saved_record.status.is_active() {
1501                *dispatch_record = saved_record;
1502                return Ok(false);
1503            }
1504        }
1505
1506        dispatch_record.status = DispatchStatus::Preparing;
1507        dispatch_record.summary = Some(summary.to_owned());
1508        dispatch_record.updated_at = now_utc();
1509        dispatch_record.finished_at = None;
1510        dispatch_record.error_message = None;
1511        self.dispatch_repository.save_dispatch(dispatch_record)?;
1512
1513        Ok(true)
1514    }
1515
1516    fn append_follow_up_request_to_task(
1517        &self,
1518        task_id: &str,
1519        follow_up_request: &str,
1520    ) -> Result<crate::types::Task, TrackError> {
1521        let task = self.task_repository.get_task(task_id)?;
1522        let timestamp_label = format_iso_8601_millis(now_utc());
1523        let next_description =
1524            append_follow_up_request(&task.description, &timestamp_label, follow_up_request);
1525
1526        self.task_repository.update_task(
1527            task_id,
1528            TaskUpdateInput {
1529                description: Some(next_description),
1530                priority: None,
1531                status: None,
1532            },
1533        )
1534    }
1535
1536    fn mark_review_notification_for_head(
1537        &self,
1538        dispatch_record: &TaskDispatchRecord,
1539        head_oid: &str,
1540        review_user: &str,
1541    ) -> Result<(), TrackError> {
1542        let mut updated_record = dispatch_record.clone();
1543
1544        // These persisted field names started out as "review request" markers.
1545        // We intentionally keep them for backward compatibility with existing
1546        // dispatch JSON while reusing them as a "notified this reviewer about
1547        // this PR head already" checkpoint.
1548        updated_record.review_request_head_oid = Some(head_oid.to_owned());
1549        updated_record.review_request_user = Some(review_user.to_owned());
1550        self.dispatch_repository.save_dispatch(&updated_record)
1551    }
1552
1553    fn cleanup_task_remote_artifacts(
1554        &self,
1555        task_id: &str,
1556        dispatch_history: &[TaskDispatchRecord],
1557        cleanup_mode: RemoteTaskCleanupMode,
1558    ) -> Result<RemoteArtifactCleanupCounts, TrackError> {
1559        if dispatch_history.is_empty() {
1560            return Ok(RemoteArtifactCleanupCounts::default());
1561        }
1562
1563        let remote_agent = self.load_remote_agent_for_cleanup(task_id)?;
1564        let ssh_client = SshClient::new(&remote_agent)?;
1565        let checkout_path = self.resolve_project_checkout_path(
1566            &ssh_client,
1567            &remote_agent,
1568            &dispatch_history[0].project,
1569        )?;
1570        let worktree_paths = unique_worktree_paths(dispatch_history);
1571        let run_directories = unique_run_directories(dispatch_history, &remote_agent);
1572
1573        ssh_client.cleanup_task_artifacts(
1574            &checkout_path,
1575            &worktree_paths,
1576            &run_directories,
1577            cleanup_mode,
1578        )
1579    }
1580
1581    fn finalize_active_dispatches_locally(
1582        &self,
1583        dispatch_history: &[TaskDispatchRecord],
1584        status: DispatchStatus,
1585        summary: &str,
1586        error_message: Option<&str>,
1587    ) -> Result<(), TrackError> {
1588        for dispatch_record in dispatch_history {
1589            if !dispatch_record.status.is_active() {
1590                continue;
1591            }
1592
1593            self.finalize_dispatch_locally(dispatch_record, status, summary, error_message)?;
1594        }
1595
1596        Ok(())
1597    }
1598
1599    fn finalize_dispatch_locally(
1600        &self,
1601        dispatch_record: &TaskDispatchRecord,
1602        status: DispatchStatus,
1603        summary: &str,
1604        error_message: Option<&str>,
1605    ) -> Result<TaskDispatchRecord, TrackError> {
1606        let mut updated_record = dispatch_record.clone();
1607        let now = now_utc();
1608        updated_record.status = status;
1609        updated_record.updated_at = now;
1610        updated_record.finished_at = Some(now);
1611        updated_record.summary = Some(summary.to_owned());
1612        updated_record.notes = None;
1613        updated_record.error_message = error_message.map(ToOwned::to_owned);
1614        self.dispatch_repository.save_dispatch(&updated_record)?;
1615
1616        Ok(updated_record)
1617    }
1618
1619    fn load_remote_agent_for_cleanup(
1620        &self,
1621        task_id: &str,
1622    ) -> Result<crate::types::RemoteAgentRuntimeConfig, TrackError> {
1623        let remote_agent = self
1624            .config_service
1625            .load_remote_agent_runtime_config()?
1626            .ok_or_else(|| {
1627            TrackError::new(
1628                ErrorCode::RemoteAgentNotConfigured,
1629                format!(
1630                    "Task {task_id} has remote dispatch history, but remote-agent configuration is missing so cleanup cannot run."
1631                ),
1632            )
1633        })?;
1634
1635        if !remote_agent.managed_key_path.exists() {
1636            return Err(TrackError::new(
1637                ErrorCode::RemoteAgentNotConfigured,
1638                format!(
1639                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again before cleaning task {task_id}.",
1640                    collapse_home_path(&remote_agent.managed_key_path)
1641                ),
1642            ));
1643        }
1644
1645        Ok(remote_agent)
1646    }
1647
1648    fn load_remote_agent_for_global_cleanup(
1649        &self,
1650    ) -> Result<crate::types::RemoteAgentRuntimeConfig, TrackError> {
1651        let remote_agent = self
1652            .config_service
1653            .load_remote_agent_runtime_config()?
1654            .ok_or_else(|| {
1655                TrackError::new(
1656                    ErrorCode::RemoteAgentNotConfigured,
1657                    "Remote cleanup cannot run because remote-agent configuration is missing.",
1658                )
1659            })?;
1660
1661        if !remote_agent.managed_key_path.exists() {
1662            return Err(TrackError::new(
1663                ErrorCode::RemoteAgentNotConfigured,
1664                format!(
1665                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again before running cleanup.",
1666                    collapse_home_path(&remote_agent.managed_key_path)
1667                ),
1668            ));
1669        }
1670
1671        Ok(remote_agent)
1672    }
1673
1674    fn cleanup_reclaimable_review_workspaces(
1675        &self,
1676        ssh_client: &SshClient,
1677        remote_agent: &crate::types::RemoteAgentRuntimeConfig,
1678        workspace_keys: &[String],
1679    ) -> Result<(), TrackError> {
1680        if workspace_keys.is_empty() {
1681            return Ok(());
1682        }
1683
1684        let mut remote_registry =
1685            load_remote_registry(ssh_client, &remote_agent.projects_registry_path)?;
1686        let checkout_paths = workspace_keys
1687            .iter()
1688            .map(|workspace_key| {
1689                remote_registry
1690                    .projects
1691                    .get(workspace_key)
1692                    .map(|entry| entry.checkout_path.clone())
1693                    .unwrap_or_else(|| {
1694                        format!(
1695                            "{}/{}/{}",
1696                            remote_agent.workspace_root.trim_end_matches('/'),
1697                            workspace_key,
1698                            workspace_key
1699                        )
1700                    })
1701            })
1702            .collect::<Vec<_>>();
1703
1704        ssh_client.cleanup_review_workspace_caches(&checkout_paths)?;
1705
1706        let mut registry_changed = false;
1707        for workspace_key in workspace_keys {
1708            registry_changed |= remote_registry.projects.remove(workspace_key).is_some();
1709        }
1710
1711        if registry_changed {
1712            write_remote_registry(
1713                ssh_client,
1714                &remote_agent.projects_registry_path,
1715                &remote_registry,
1716            )?;
1717        }
1718
1719        Ok(())
1720    }
1721
1722    fn resolve_project_checkout_path(
1723        &self,
1724        ssh_client: &SshClient,
1725        remote_agent: &crate::types::RemoteAgentRuntimeConfig,
1726        project_name: &str,
1727    ) -> Result<String, TrackError> {
1728        let remote_registry =
1729            load_remote_registry(ssh_client, &remote_agent.projects_registry_path)?;
1730
1731        Ok(remote_registry
1732            .projects
1733            .get(project_name)
1734            .map(|entry| entry.checkout_path.clone())
1735            .unwrap_or_else(|| {
1736                format!(
1737                    "{}/{}/{}",
1738                    remote_agent.workspace_root.trim_end_matches('/'),
1739                    project_name,
1740                    project_name
1741                )
1742            }))
1743    }
1744
1745    fn load_dispatch_prerequisites(
1746        &self,
1747        task_id: &str,
1748    ) -> Result<
1749        (
1750            crate::types::RemoteAgentRuntimeConfig,
1751            crate::types::Task,
1752            ProjectMetadata,
1753        ),
1754        TrackError,
1755    > {
1756        let remote_agent = self
1757            .config_service
1758            .load_remote_agent_runtime_config()?
1759            .ok_or_else(|| {
1760            TrackError::new(
1761                ErrorCode::RemoteAgentNotConfigured,
1762                "Remote dispatch is not configured yet. Re-run `track` and add a remote agent host plus SSH key.",
1763            )
1764        })?;
1765
1766        if !remote_agent.managed_key_path.exists() {
1767            return Err(TrackError::new(
1768                ErrorCode::RemoteAgentNotConfigured,
1769                format!(
1770                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
1771                    collapse_home_path(&remote_agent.managed_key_path)
1772                ),
1773            ));
1774        }
1775
1776        if remote_agent
1777            .shell_prelude
1778            .as_deref()
1779            .map(str::trim)
1780            .unwrap_or_default()
1781            .is_empty()
1782        {
1783            return Err(TrackError::new(
1784                ErrorCode::InvalidRemoteAgentConfig,
1785                "Remote runner setup is missing. Open the web UI and add the shell instructions that prepare PATH and toolchains for the remote runner.",
1786            ));
1787        }
1788
1789        let task = self.task_repository.get_task(task_id)?;
1790        let project = self.project_repository.get_project_by_name(&task.project)?;
1791        validate_project_metadata_for_dispatch(&project.metadata)?;
1792
1793        Ok((remote_agent, task, project.metadata))
1794    }
1795}
1796
1797impl<'a> RemoteReviewService<'a> {
1798    // =============================================================================
1799    // Review Request Entry Points
1800    // =============================================================================
1801    //
1802    // Reviews are intentionally a separate domain from task dispatches. They
1803    // still reuse the same remote runner and SSH bootstrap, but they start
1804    // from a PR URL, persist their own local records, and ask the agent to
1805    // submit a GitHub review directly instead of creating or updating a PR
1806    // branch.
1807    pub fn create_review(
1808        &self,
1809        input: CreateReviewInput,
1810    ) -> Result<(ReviewRecord, ReviewRunRecord), TrackError> {
1811        let validated_input = input.validate()?;
1812        let (remote_agent, review_settings) = self.load_review_runtime_prerequisites()?;
1813        let ssh_client = SshClient::new(&remote_agent)?;
1814        let pull_request_metadata =
1815            ssh_client.fetch_pull_request_metadata(&validated_input.pull_request_url)?;
1816        let initial_target_head_oid = pull_request_metadata.head_oid.clone();
1817        let project_match = self
1818            .project_repository
1819            .list_projects()?
1820            .into_iter()
1821            .find(|project| project.metadata.repo_url.trim() == pull_request_metadata.repo_url);
1822        let project_metadata_override = project_match
1823            .as_ref()
1824            .map(|project| project.metadata.clone());
1825        let workspace_key = project_match
1826            .as_ref()
1827            .map(|project| project.canonical_name.clone())
1828            .unwrap_or_else(|| build_review_workspace_key(&pull_request_metadata));
1829        let review_timestamp = now_utc();
1830        let review_id = build_unique_task_id(
1831            review_timestamp,
1832            &format!(
1833                "review {} pr {}",
1834                pull_request_metadata.repository_full_name,
1835                pull_request_metadata.pull_request_number
1836            ),
1837            |candidate| self.review_repository.get_review(candidate).is_ok(),
1838        );
1839        let review = ReviewRecord {
1840            id: review_id,
1841            pull_request_url: pull_request_metadata.pull_request_url,
1842            pull_request_number: pull_request_metadata.pull_request_number,
1843            pull_request_title: pull_request_metadata.pull_request_title,
1844            repository_full_name: pull_request_metadata.repository_full_name,
1845            repo_url: project_metadata_override
1846                .as_ref()
1847                .map(|metadata| metadata.repo_url.clone())
1848                .unwrap_or(pull_request_metadata.repo_url),
1849            git_url: project_metadata_override
1850                .as_ref()
1851                .map(|metadata| metadata.git_url.clone())
1852                .unwrap_or(pull_request_metadata.git_url),
1853            base_branch: project_metadata_override
1854                .as_ref()
1855                .map(|metadata| metadata.base_branch.clone())
1856                .unwrap_or(pull_request_metadata.base_branch),
1857            workspace_key,
1858            preferred_tool: validated_input
1859                .preferred_tool
1860                .unwrap_or(remote_agent.preferred_tool),
1861            project: project_match.map(|project| project.canonical_name),
1862            main_user: review_settings.main_user,
1863            default_review_prompt: review_settings.default_review_prompt,
1864            extra_instructions: validated_input.extra_instructions,
1865            created_at: review_timestamp,
1866            updated_at: review_timestamp,
1867        };
1868
1869        self.review_repository.save_review(&review)?;
1870        match self.queue_review_dispatch(
1871            &review,
1872            &remote_agent,
1873            None,
1874            Some(initial_target_head_oid.as_str()),
1875        ) {
1876            Ok(dispatch) => Ok((review, dispatch)),
1877            Err(error) => {
1878                let _ = self.review_repository.delete_review(&review.id);
1879                Err(error)
1880            }
1881        }
1882    }
1883
1884    // =============================================================================
1885    // Follow-Up Review Runs
1886    // =============================================================================
1887    //
1888    // A re-review should feel like the PR equivalent of a task follow-up: the
1889    // saved review record remains the durable anchor, while each new run stores
1890    // the latest user ask plus the exact PR head it targeted. We deliberately
1891    // fetch fresh PR metadata here so each run records which commit the agent
1892    // reviewed instead of assuming the PR stayed on the same head as the
1893    // initial request.
1894    pub fn queue_follow_up_review_dispatch(
1895        &self,
1896        review_id: &str,
1897        follow_up_request: &str,
1898    ) -> Result<ReviewRunRecord, TrackError> {
1899        let trimmed_follow_up_request = follow_up_request.trim();
1900        if trimmed_follow_up_request.is_empty() {
1901            return Err(TrackError::new(
1902                ErrorCode::EmptyInput,
1903                "Please provide a re-review request for the remote agent.",
1904            ));
1905        }
1906
1907        let (remote_agent, mut review) = self.load_review_dispatch_prerequisites(review_id)?;
1908        let _dispatch_start_guard = ReviewDispatchStartGuard::acquire(review_id);
1909        self.ensure_no_blocking_active_review_dispatch(review_id)?;
1910
1911        let ssh_client = SshClient::new(&remote_agent)?;
1912        let pull_request_metadata =
1913            ssh_client.fetch_pull_request_metadata(&review.pull_request_url)?;
1914        let previous_updated_at = review.updated_at;
1915        review.updated_at = now_utc();
1916        self.review_repository.save_review(&review)?;
1917
1918        match self.queue_review_dispatch(
1919            &review,
1920            &remote_agent,
1921            Some(trimmed_follow_up_request),
1922            Some(pull_request_metadata.head_oid.as_str()),
1923        ) {
1924            Ok(dispatch) => Ok(dispatch),
1925            Err(error) => {
1926                review.updated_at = previous_updated_at;
1927                let _ = self.review_repository.save_review(&review);
1928                Err(error)
1929            }
1930        }
1931    }
1932
1933    pub fn launch_prepared_review(
1934        &self,
1935        mut dispatch_record: ReviewRunRecord,
1936    ) -> Result<ReviewRunRecord, TrackError> {
1937        if let Some(existing_record) = self
1938            .load_saved_review_dispatch(&dispatch_record.review_id, &dispatch_record.dispatch_id)?
1939        {
1940            if !existing_record.status.is_active() {
1941                return Ok(existing_record);
1942            }
1943        }
1944
1945        let worktree_path = dispatch_record
1946            .worktree_path
1947            .clone()
1948            .expect("queued review dispatches should store a worktree path");
1949        let branch_name = dispatch_record
1950            .branch_name
1951            .clone()
1952            .expect("queued review dispatches should store a branch name");
1953        let remote_run_directory =
1954            derive_review_run_directory(&worktree_path, &dispatch_record.dispatch_id)?;
1955
1956        let launch_result = (|| -> Result<(), TrackError> {
1957            if !self.save_review_preparing_phase(
1958                &mut dispatch_record,
1959                "Checking remote review prerequisites.",
1960            )? {
1961                return Ok(());
1962            }
1963            let (remote_agent, review) =
1964                self.load_review_dispatch_prerequisites(&dispatch_record.review_id)?;
1965            let ssh_client = SshClient::new(&remote_agent)?;
1966
1967            if !self.save_review_preparing_phase(
1968                &mut dispatch_record,
1969                "Loading the remote project registry.",
1970            )? {
1971                return Ok(());
1972            }
1973            let remote_registry =
1974                load_remote_registry(&ssh_client, &remote_agent.projects_registry_path)?;
1975
1976            if !self.save_review_preparing_phase(
1977                &mut dispatch_record,
1978                "Checking GitHub authentication on the remote machine.",
1979            )? {
1980                return Ok(());
1981            }
1982            let github_login = ssh_client.fetch_github_login()?;
1983            let repository_name = parse_github_repository_name(&review.repo_url)?;
1984            let checkout_path = remote_registry
1985                .projects
1986                .get(&review.workspace_key)
1987                .map(|entry| entry.checkout_path.clone())
1988                .unwrap_or_else(|| {
1989                    format!(
1990                        "{}/{}/{}",
1991                        remote_agent.workspace_root.trim_end_matches('/'),
1992                        review.workspace_key,
1993                        review.workspace_key
1994                    )
1995                });
1996
1997            if !self.save_review_preparing_phase(
1998                &mut dispatch_record,
1999                "Ensuring the remote checkout is up to date.",
2000            )? {
2001                return Ok(());
2002            }
2003            let fork_git_url = ssh_client.ensure_checkout(
2004                &ProjectMetadata {
2005                    repo_url: review.repo_url.clone(),
2006                    git_url: review.git_url.clone(),
2007                    base_branch: review.base_branch.clone(),
2008                    description: None,
2009                },
2010                &repository_name,
2011                &checkout_path,
2012                &github_login,
2013            )?;
2014
2015            let mut updated_registry = remote_registry;
2016            updated_registry.projects.insert(
2017                review.workspace_key.clone(),
2018                RemoteProjectRegistryEntry {
2019                    checkout_path: checkout_path.clone(),
2020                    fork_git_url,
2021                    repo_url: review.repo_url.clone(),
2022                    git_url: review.git_url.clone(),
2023                    base_branch: review.base_branch.clone(),
2024                    updated_at: format_iso_8601_millis(now_utc()),
2025                },
2026            );
2027            write_remote_registry(
2028                &ssh_client,
2029                &remote_agent.projects_registry_path,
2030                &updated_registry,
2031            )?;
2032
2033            if !self.save_review_preparing_phase(
2034                &mut dispatch_record,
2035                "Preparing the review worktree.",
2036            )? {
2037                return Ok(());
2038            }
2039            ssh_client.create_review_worktree(
2040                &checkout_path,
2041                review.pull_request_number,
2042                &branch_name,
2043                &worktree_path,
2044                dispatch_record.target_head_oid.as_deref(),
2045            )?;
2046
2047            let dispatch_history = self
2048                .review_dispatch_repository
2049                .dispatches_for_review(&review.id)?;
2050            let previous_submitted_review = select_previous_submitted_review_run(
2051                &dispatch_history,
2052                &dispatch_record.dispatch_id,
2053            );
2054            let prompt =
2055                build_remote_review_prompt(&review, &dispatch_record, previous_submitted_review);
2056            let schema = build_remote_review_schema();
2057            if !self.save_review_preparing_phase(
2058                &mut dispatch_record,
2059                "Uploading the review prompt and schema.",
2060            )? {
2061                return Ok(());
2062            }
2063            ssh_client.upload_remote_file(
2064                &format!("{remote_run_directory}/{REMOTE_PROMPT_FILE_NAME}"),
2065                &prompt,
2066            )?;
2067            ssh_client.upload_remote_file(
2068                &format!("{remote_run_directory}/{REMOTE_SCHEMA_FILE_NAME}"),
2069                &schema,
2070            )?;
2071
2072            if !self.dispatch_is_still_active(
2073                &dispatch_record.review_id,
2074                &dispatch_record.dispatch_id,
2075            )? {
2076                return Ok(());
2077            }
2078
2079            if !self.save_review_preparing_phase(
2080                &mut dispatch_record,
2081                "Launching the remote review agent.",
2082            )? {
2083                return Ok(());
2084            }
2085            ssh_client.launch_remote_dispatch(
2086                &remote_run_directory,
2087                &worktree_path,
2088                dispatch_record.preferred_tool,
2089            )?;
2090
2091            Ok(())
2092        })();
2093
2094        match launch_result {
2095            Ok(()) => {
2096                if let Some(existing_record) = self.load_saved_review_dispatch(
2097                    &dispatch_record.review_id,
2098                    &dispatch_record.dispatch_id,
2099                )? {
2100                    if !existing_record.status.is_active() {
2101                        let _ = self.cancel_remote_review_if_possible(&existing_record);
2102                        return Ok(existing_record);
2103                    }
2104                }
2105
2106                dispatch_record.status = DispatchStatus::Running;
2107                dispatch_record.updated_at = now_utc();
2108                dispatch_record.finished_at = None;
2109                dispatch_record.summary =
2110                    Some("The remote agent is reviewing the prepared pull request.".to_owned());
2111                dispatch_record.error_message = None;
2112                self.review_dispatch_repository
2113                    .save_dispatch(&dispatch_record)?;
2114                Ok(dispatch_record)
2115            }
2116            Err(error) => {
2117                dispatch_record.status = DispatchStatus::Failed;
2118                dispatch_record.updated_at = now_utc();
2119                dispatch_record.finished_at = Some(dispatch_record.updated_at);
2120                dispatch_record.error_message = Some(error.to_string());
2121                self.review_dispatch_repository
2122                    .save_dispatch(&dispatch_record)?;
2123                Err(error)
2124            }
2125        }
2126    }
2127
2128    pub fn latest_dispatches_for_reviews(
2129        &self,
2130        review_ids: &[String],
2131    ) -> Result<Vec<ReviewRunRecord>, TrackError> {
2132        let mut records = Vec::new();
2133        for review_id in review_ids {
2134            if let Some(record) = self
2135                .review_dispatch_repository
2136                .latest_dispatch_for_review(review_id)?
2137            {
2138                records.push(record);
2139            }
2140        }
2141
2142        self.refresh_active_review_dispatch_records(records)
2143    }
2144
2145    pub fn list_dispatches(
2146        &self,
2147        limit: Option<usize>,
2148    ) -> Result<Vec<ReviewRunRecord>, TrackError> {
2149        let records = self.review_dispatch_repository.list_dispatches(limit)?;
2150        self.refresh_active_review_dispatch_records(records)
2151    }
2152
2153    pub fn dispatch_history_for_review(
2154        &self,
2155        review_id: &str,
2156    ) -> Result<Vec<ReviewRunRecord>, TrackError> {
2157        let mut records = self
2158            .review_dispatch_repository
2159            .dispatches_for_review(review_id)?;
2160        if records
2161            .first()
2162            .is_some_and(|record| record.status.is_active())
2163        {
2164            if let Some(refreshed_latest) = self
2165                .latest_dispatches_for_reviews(&[review_id.to_owned()])?
2166                .into_iter()
2167                .next()
2168            {
2169                if let Some(first_record) = records.first_mut() {
2170                    *first_record = refreshed_latest;
2171                }
2172            }
2173        }
2174
2175        Ok(records)
2176    }
2177
2178    pub fn cancel_dispatch(&self, review_id: &str) -> Result<ReviewRunRecord, TrackError> {
2179        let mut latest_dispatch = self
2180            .latest_dispatches_for_reviews(&[review_id.to_owned()])?
2181            .into_iter()
2182            .next()
2183            .ok_or_else(|| {
2184                TrackError::new(
2185                    ErrorCode::DispatchNotFound,
2186                    format!("Review {review_id} does not have a remote run to cancel."),
2187                )
2188            })?;
2189
2190        if !latest_dispatch.status.is_active() {
2191            return Err(TrackError::new(
2192                ErrorCode::DispatchNotFound,
2193                format!("Review {review_id} does not have an active remote run to cancel."),
2194            ));
2195        }
2196
2197        self.cancel_remote_review_if_possible(&latest_dispatch)?;
2198
2199        latest_dispatch.status = DispatchStatus::Canceled;
2200        latest_dispatch.updated_at = now_utc();
2201        latest_dispatch.finished_at = Some(latest_dispatch.updated_at);
2202        latest_dispatch.summary = Some("Canceled from the web UI.".to_owned());
2203        latest_dispatch.notes = None;
2204        latest_dispatch.error_message = None;
2205        self.review_dispatch_repository
2206            .save_dispatch(&latest_dispatch)?;
2207
2208        Ok(latest_dispatch)
2209    }
2210
2211    fn ensure_no_blocking_active_review_dispatch(&self, review_id: &str) -> Result<(), TrackError> {
2212        if let Some(existing_dispatch) = self
2213            .latest_dispatches_for_reviews(&[review_id.to_owned()])?
2214            .into_iter()
2215            .next()
2216            .filter(|record| record.status.is_active())
2217        {
2218            return Err(TrackError::new(
2219                ErrorCode::RemoteDispatchFailed,
2220                format!(
2221                    "Review {review_id} already has an active remote run ({})",
2222                    existing_dispatch.dispatch_id
2223                ),
2224            ));
2225        }
2226
2227        Ok(())
2228    }
2229
2230    pub fn delete_review(&self, review_id: &str) -> Result<(), TrackError> {
2231        let review = self.review_repository.get_review(review_id)?;
2232        let dispatch_history = self
2233            .review_dispatch_repository
2234            .dispatches_for_review(review_id)?;
2235        if !dispatch_history.is_empty() {
2236            if let Err(error) = self.cleanup_review_remote_artifacts(&review, &dispatch_history) {
2237                eprintln!("Skipping remote cleanup while deleting review {review_id}: {error}");
2238            }
2239
2240            self.review_dispatch_repository
2241                .delete_dispatch_history_for_review(review_id)?;
2242        }
2243
2244        self.review_repository.delete_review(review_id)
2245    }
2246
2247    fn queue_review_dispatch(
2248        &self,
2249        review: &ReviewRecord,
2250        remote_agent: &crate::types::RemoteAgentRuntimeConfig,
2251        follow_up_request: Option<&str>,
2252        target_head_oid: Option<&str>,
2253    ) -> Result<ReviewRunRecord, TrackError> {
2254        let mut dispatch_record = self.review_dispatch_repository.create_dispatch(
2255            review,
2256            &remote_agent.host,
2257            review.preferred_tool,
2258        )?;
2259        dispatch_record.branch_name = Some(format!("track-review/{}", dispatch_record.dispatch_id));
2260        dispatch_record.worktree_path = Some(format!(
2261            "{}/{}/{}/{}",
2262            remote_agent.workspace_root.trim_end_matches('/'),
2263            review.workspace_key,
2264            REVIEW_WORKTREE_DIRECTORY_NAME,
2265            dispatch_record.dispatch_id
2266        ));
2267        dispatch_record.follow_up_request = follow_up_request.map(str::trim).map(ToOwned::to_owned);
2268        dispatch_record.target_head_oid = target_head_oid
2269            .map(str::trim)
2270            .filter(|value| !value.is_empty())
2271            .map(ToOwned::to_owned);
2272        if let Some(follow_up_request) = dispatch_record.follow_up_request.as_deref() {
2273            dispatch_record.summary = Some(format!(
2274                "Re-review request: {}",
2275                first_follow_up_line(follow_up_request)
2276            ));
2277        }
2278        dispatch_record.updated_at = now_utc();
2279        self.review_dispatch_repository
2280            .save_dispatch(&dispatch_record)?;
2281
2282        Ok(dispatch_record)
2283    }
2284
2285    fn refresh_active_review_dispatch_records(
2286        &self,
2287        records: Vec<ReviewRunRecord>,
2288    ) -> Result<Vec<ReviewRunRecord>, TrackError> {
2289        let remote_agent = match self.config_service.load_remote_agent_runtime_config() {
2290            Ok(config) => config,
2291            Err(error)
2292                if matches!(
2293                    error.code,
2294                    ErrorCode::ConfigNotFound
2295                        | ErrorCode::InvalidConfig
2296                        | ErrorCode::InvalidRemoteAgentConfig
2297                ) =>
2298            {
2299                let error_message = error.to_string();
2300                return self.release_active_review_dispatches_after_reconciliation_loss(
2301                    records,
2302                    "Remote reconciliation is unavailable locally, so active review runs were released.",
2303                    &error_message,
2304                );
2305            }
2306            Err(error) => return Err(error),
2307        };
2308
2309        let Some(remote_agent) = remote_agent else {
2310            return self.release_active_review_dispatches_after_reconciliation_loss(
2311                records,
2312                "Remote reconciliation is unavailable locally, so active review runs were released.",
2313                "Remote agent configuration is missing locally.",
2314            );
2315        };
2316        if !remote_agent.managed_key_path.exists() {
2317            let error_message = format!(
2318                "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
2319                collapse_home_path(&remote_agent.managed_key_path)
2320            );
2321            return self.release_active_review_dispatches_after_reconciliation_loss(
2322                records,
2323                "Remote reconciliation is unavailable locally, so active review runs were released.",
2324                &error_message,
2325            );
2326        }
2327
2328        let ssh_client = SshClient::new(&remote_agent)?;
2329        let snapshots_by_dispatch_id = load_review_snapshots_for_records(&ssh_client, &records)?;
2330        let mut refreshed_records = Vec::with_capacity(records.len());
2331        for record in records {
2332            if !record.status.is_active() {
2333                refreshed_records.push(record);
2334                continue;
2335            }
2336
2337            let Some(snapshot) = snapshots_by_dispatch_id.get(&record.dispatch_id) else {
2338                if let Some(updated) = mark_abandoned_preparing_review_dispatch(record.clone()) {
2339                    self.review_dispatch_repository.save_dispatch(&updated)?;
2340                    refreshed_records.push(updated);
2341                } else {
2342                    let updated = self.finalize_review_dispatch_locally(
2343                        &record,
2344                        DispatchStatus::Blocked,
2345                        "Remote reconciliation could not find this review run anymore, so it was released locally.",
2346                        Some("Remote review snapshot is missing."),
2347                    )?;
2348                    refreshed_records.push(updated);
2349                }
2350                continue;
2351            };
2352
2353            match self.refresh_review_dispatch_record_from_snapshot(record.clone(), snapshot) {
2354                Ok(updated) => {
2355                    if updated != record {
2356                        self.review_dispatch_repository.save_dispatch(&updated)?;
2357                    }
2358                    refreshed_records.push(updated);
2359                }
2360                Err(error) => {
2361                    if let Some(updated) =
2362                        mark_terminal_review_refresh_failure(record.clone(), snapshot, &error)
2363                    {
2364                        self.review_dispatch_repository.save_dispatch(&updated)?;
2365                        refreshed_records.push(updated);
2366                    } else {
2367                        let error_message = error.to_string();
2368                        let updated = self.finalize_review_dispatch_locally(
2369                            &record,
2370                            DispatchStatus::Blocked,
2371                            "Remote reconciliation could not confirm this review run, so it was released locally.",
2372                            Some(&error_message),
2373                        )?;
2374                        refreshed_records.push(updated);
2375                    }
2376                }
2377            }
2378        }
2379
2380        Ok(refreshed_records)
2381    }
2382
2383    fn refresh_review_dispatch_record_from_snapshot(
2384        &self,
2385        mut record: ReviewRunRecord,
2386        snapshot: &RemoteDispatchSnapshot,
2387    ) -> Result<ReviewRunRecord, TrackError> {
2388        let remote_status = snapshot.status.as_deref().unwrap_or_default().trim();
2389        if remote_status.is_empty() {
2390            if let Some(updated) = mark_abandoned_preparing_review_dispatch(record.clone()) {
2391                return Ok(updated);
2392            }
2393
2394            return Ok(record);
2395        }
2396
2397        if remote_status == "running" {
2398            if record.status == DispatchStatus::Preparing {
2399                record.status = DispatchStatus::Running;
2400                record.updated_at = now_utc();
2401                record.finished_at = None;
2402                record.error_message = None;
2403            }
2404            return Ok(record);
2405        }
2406
2407        if remote_status == "canceled" {
2408            record.status = DispatchStatus::Canceled;
2409            record.updated_at = now_utc();
2410            record.finished_at = Some(parse_remote_finished_at(
2411                snapshot.finished_at.as_deref(),
2412                now_utc(),
2413            ));
2414            record.summary = Some(
2415                record
2416                    .summary
2417                    .unwrap_or_else(|| "Canceled from the web UI.".to_owned()),
2418            );
2419            record.error_message = None;
2420            return Ok(record);
2421        }
2422
2423        let now = now_utc();
2424        record.updated_at = now;
2425        if remote_status == "completed" {
2426            let remote_result = snapshot.result.as_deref().ok_or_else(|| {
2427                TrackError::new(
2428                    ErrorCode::RemoteDispatchFailed,
2429                    "Remote review run completed without producing a structured result.",
2430                )
2431            })?;
2432            let outcome = parse_remote_agent_output::<RemoteAgentReviewOutcome>(
2433                remote_result,
2434                record.preferred_tool,
2435                "Remote review result",
2436            )?;
2437
2438            // The review agent now owns the GitHub side effect directly, just
2439            // like the task flow owns its own pushes and PR creation. We keep
2440            // the local record intentionally narrow here: enough structured
2441            // metadata for status/history, without mirroring GitHub discussion
2442            // threads back into local storage.
2443            // TODO: If the web UI needs first-class inline-comment history,
2444            // persist dedicated review metadata rather than trying to mirror
2445            // every GitHub thread inside this local run record.
2446            // TODO: If we need to reconcile "review submitted, final JSON not
2447            // written" crashes, capture the GitHub review handle in a sidecar
2448            // file during the remote run before the final structured result.
2449            record.status = outcome.status;
2450            record.summary = Some(outcome.summary);
2451            record.review_submitted = outcome.review_submitted;
2452            record.github_review_id = outcome.github_review_id;
2453            record.github_review_url = outcome.github_review_url;
2454            record.worktree_path = Some(outcome.worktree_path);
2455            record.notes = outcome.notes;
2456            record.error_message = None;
2457            record.finished_at = Some(parse_remote_finished_at(
2458                snapshot.finished_at.as_deref(),
2459                now,
2460            ));
2461
2462            return Ok(record);
2463        }
2464
2465        record.status = DispatchStatus::Failed;
2466        record.finished_at = Some(parse_remote_finished_at(
2467            snapshot.finished_at.as_deref(),
2468            now,
2469        ));
2470        record.error_message = snapshot
2471            .stderr
2472            .as_deref()
2473            .map(str::trim)
2474            .filter(|value| !value.is_empty())
2475            .map(|value| value.to_owned())
2476            .or_else(|| {
2477                Some("Remote review run failed before returning a structured result.".to_owned())
2478            });
2479        Ok(record)
2480    }
2481
2482    fn release_active_review_dispatches_after_reconciliation_loss(
2483        &self,
2484        records: Vec<ReviewRunRecord>,
2485        summary: &str,
2486        error_message: &str,
2487    ) -> Result<Vec<ReviewRunRecord>, TrackError> {
2488        let mut refreshed_records = Vec::with_capacity(records.len());
2489        for record in records {
2490            if record.status.is_active() {
2491                refreshed_records.push(self.finalize_review_dispatch_locally(
2492                    &record,
2493                    DispatchStatus::Blocked,
2494                    summary,
2495                    Some(error_message),
2496                )?);
2497            } else {
2498                refreshed_records.push(record);
2499            }
2500        }
2501
2502        Ok(refreshed_records)
2503    }
2504
2505    fn finalize_review_dispatch_locally(
2506        &self,
2507        dispatch_record: &ReviewRunRecord,
2508        status: DispatchStatus,
2509        summary: &str,
2510        error_message: Option<&str>,
2511    ) -> Result<ReviewRunRecord, TrackError> {
2512        let mut updated_record = dispatch_record.clone();
2513        let now = now_utc();
2514        updated_record.status = status;
2515        updated_record.updated_at = now;
2516        updated_record.finished_at = Some(now);
2517        updated_record.summary = Some(summary.to_owned());
2518        updated_record.notes = None;
2519        updated_record.error_message = error_message.map(ToOwned::to_owned);
2520        self.review_dispatch_repository
2521            .save_dispatch(&updated_record)?;
2522
2523        Ok(updated_record)
2524    }
2525
2526    fn load_saved_review_dispatch(
2527        &self,
2528        review_id: &str,
2529        dispatch_id: &str,
2530    ) -> Result<Option<ReviewRunRecord>, TrackError> {
2531        self.review_dispatch_repository
2532            .get_dispatch(review_id, dispatch_id)
2533    }
2534
2535    fn dispatch_is_still_active(
2536        &self,
2537        review_id: &str,
2538        dispatch_id: &str,
2539    ) -> Result<bool, TrackError> {
2540        Ok(self
2541            .load_saved_review_dispatch(review_id, dispatch_id)?
2542            .map(|record| record.status.is_active())
2543            .unwrap_or(false))
2544    }
2545
2546    fn save_review_preparing_phase(
2547        &self,
2548        dispatch_record: &mut ReviewRunRecord,
2549        summary: &str,
2550    ) -> Result<bool, TrackError> {
2551        if let Some(saved_record) = self
2552            .load_saved_review_dispatch(&dispatch_record.review_id, &dispatch_record.dispatch_id)?
2553        {
2554            if !saved_record.status.is_active() {
2555                *dispatch_record = saved_record;
2556                return Ok(false);
2557            }
2558        }
2559
2560        dispatch_record.status = DispatchStatus::Preparing;
2561        dispatch_record.summary = Some(summary.to_owned());
2562        dispatch_record.updated_at = now_utc();
2563        dispatch_record.finished_at = None;
2564        dispatch_record.error_message = None;
2565        self.review_dispatch_repository
2566            .save_dispatch(dispatch_record)?;
2567
2568        Ok(true)
2569    }
2570
2571    fn cancel_remote_review_if_possible(
2572        &self,
2573        dispatch_record: &ReviewRunRecord,
2574    ) -> Result<(), TrackError> {
2575        let remote_agent = self
2576            .config_service
2577            .load_remote_agent_runtime_config()?
2578            .ok_or_else(|| {
2579            TrackError::new(
2580                ErrorCode::RemoteAgentNotConfigured,
2581                "Remote dispatch is not configured yet. Re-run `track` and add a remote agent host plus SSH key.",
2582            )
2583        })?;
2584
2585        if !remote_agent.managed_key_path.exists() {
2586            return Err(TrackError::new(
2587                ErrorCode::RemoteAgentNotConfigured,
2588                format!(
2589                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
2590                    collapse_home_path(&remote_agent.managed_key_path)
2591                ),
2592            ));
2593        }
2594
2595        let Some(worktree_path) = dispatch_record.worktree_path.as_deref() else {
2596            return Ok(());
2597        };
2598        let remote_run_directory =
2599            derive_review_run_directory(worktree_path, &dispatch_record.dispatch_id)?;
2600        let ssh_client = SshClient::new(&remote_agent)?;
2601        ssh_client.cancel_remote_dispatch(&remote_run_directory)
2602    }
2603
2604    fn cleanup_review_remote_artifacts(
2605        &self,
2606        review: &ReviewRecord,
2607        dispatch_history: &[ReviewRunRecord],
2608    ) -> Result<(), TrackError> {
2609        if dispatch_history.is_empty() {
2610            return Ok(());
2611        }
2612
2613        let remote_agent = self.load_remote_agent_for_review_cleanup(&review.id)?;
2614        let ssh_client = SshClient::new(&remote_agent)?;
2615        let checkout_path =
2616            self.resolve_review_checkout_path(&ssh_client, &remote_agent, &review.workspace_key)?;
2617        let worktree_paths = unique_review_worktree_paths(dispatch_history);
2618        let run_directories = unique_review_run_directories(dispatch_history, &remote_agent);
2619        let branch_names = dispatch_history
2620            .iter()
2621            .filter_map(|record| record.branch_name.clone())
2622            .collect::<BTreeSet<_>>()
2623            .into_iter()
2624            .collect::<Vec<_>>();
2625
2626        ssh_client.cleanup_review_artifacts(
2627            &checkout_path,
2628            &branch_names,
2629            &worktree_paths,
2630            &run_directories,
2631        )
2632    }
2633
2634    fn load_remote_agent_for_review_cleanup(
2635        &self,
2636        review_id: &str,
2637    ) -> Result<crate::types::RemoteAgentRuntimeConfig, TrackError> {
2638        let remote_agent = self
2639            .config_service
2640            .load_remote_agent_runtime_config()?
2641            .ok_or_else(|| {
2642            TrackError::new(
2643                ErrorCode::RemoteAgentNotConfigured,
2644                format!(
2645                    "Review {review_id} has remote history, but remote-agent configuration is missing so cleanup cannot run."
2646                ),
2647            )
2648        })?;
2649
2650        if !remote_agent.managed_key_path.exists() {
2651            return Err(TrackError::new(
2652                ErrorCode::RemoteAgentNotConfigured,
2653                format!(
2654                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again before cleaning review {review_id}.",
2655                    collapse_home_path(&remote_agent.managed_key_path)
2656                ),
2657            ));
2658        }
2659
2660        Ok(remote_agent)
2661    }
2662
2663    fn resolve_review_checkout_path(
2664        &self,
2665        ssh_client: &SshClient,
2666        remote_agent: &crate::types::RemoteAgentRuntimeConfig,
2667        workspace_key: &str,
2668    ) -> Result<String, TrackError> {
2669        let remote_registry =
2670            load_remote_registry(ssh_client, &remote_agent.projects_registry_path)?;
2671
2672        Ok(remote_registry
2673            .projects
2674            .get(workspace_key)
2675            .map(|entry| entry.checkout_path.clone())
2676            .unwrap_or_else(|| {
2677                format!(
2678                    "{}/{}/{}",
2679                    remote_agent.workspace_root.trim_end_matches('/'),
2680                    workspace_key,
2681                    workspace_key
2682                )
2683            }))
2684    }
2685
2686    // =============================================================================
2687    // Review Runner Prerequisites
2688    // =============================================================================
2689    //
2690    // Saved reviews snapshot the review-specific knobs they need for future
2691    // re-reviews, namely the main GitHub user and default prompt. That means
2692    // later follow-up runs should only depend on the remote runner itself still
2693    // being usable, not on the mutable global review-follow-up block still
2694    // existing in the current config.
2695    fn load_review_runner_prerequisites(
2696        &self,
2697    ) -> Result<crate::types::RemoteAgentRuntimeConfig, TrackError> {
2698        let remote_agent = self
2699            .config_service
2700            .load_remote_agent_runtime_config()?
2701            .ok_or_else(|| {
2702            TrackError::new(
2703                ErrorCode::RemoteAgentNotConfigured,
2704                "Remote reviews are not configured yet. Re-run `track` and add a remote agent host plus SSH key.",
2705            )
2706        })?;
2707
2708        if !remote_agent.managed_key_path.exists() {
2709            return Err(TrackError::new(
2710                ErrorCode::RemoteAgentNotConfigured,
2711                format!(
2712                    "Managed SSH key not found at {}. Re-run `track` and import the remote-agent key again.",
2713                    collapse_home_path(&remote_agent.managed_key_path)
2714                ),
2715            ));
2716        }
2717
2718        if remote_agent
2719            .shell_prelude
2720            .as_deref()
2721            .map(str::trim)
2722            .unwrap_or_default()
2723            .is_empty()
2724        {
2725            return Err(TrackError::new(
2726                ErrorCode::InvalidRemoteAgentConfig,
2727                "Remote runner setup is missing. Open the web UI and add the shell instructions that prepare PATH and toolchains for the remote runner.",
2728            ));
2729        }
2730
2731        Ok(remote_agent)
2732    }
2733
2734    fn load_review_runtime_prerequisites(
2735        &self,
2736    ) -> Result<
2737        (
2738            crate::types::RemoteAgentRuntimeConfig,
2739            crate::types::RemoteAgentReviewFollowUpRuntimeConfig,
2740        ),
2741        TrackError,
2742    > {
2743        let remote_agent = self.load_review_runner_prerequisites()?;
2744        let review_settings = remote_agent.review_follow_up.clone().ok_or_else(|| {
2745            TrackError::new(
2746                ErrorCode::InvalidRemoteAgentConfig,
2747                "PR reviews require a configured main GitHub user in the remote runner settings.",
2748            )
2749        })?;
2750
2751        Ok((remote_agent, review_settings))
2752    }
2753
2754    fn load_review_dispatch_prerequisites(
2755        &self,
2756        review_id: &str,
2757    ) -> Result<(crate::types::RemoteAgentRuntimeConfig, ReviewRecord), TrackError> {
2758        let remote_agent = self.load_review_runner_prerequisites()?;
2759        let review = self.review_repository.get_review(review_id)?;
2760
2761        Ok((remote_agent, review))
2762    }
2763}
2764
2765#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2766enum RemoteTaskCleanupMode {
2767    CloseTask,
2768    DeleteTask,
2769}
2770
2771fn first_follow_up_line(follow_up_request: &str) -> String {
2772    follow_up_request
2773        .lines()
2774        .map(str::trim)
2775        .find(|line| !line.is_empty())
2776        .unwrap_or("Continue the previous remote task.")
2777        .to_owned()
2778}
2779
2780fn select_follow_up_base_dispatch(
2781    dispatch_history: &[TaskDispatchRecord],
2782) -> Option<TaskDispatchRecord> {
2783    dispatch_history
2784        .iter()
2785        .find(|record| {
2786            !record.status.is_active()
2787                && record.branch_name.is_some()
2788                && record.worktree_path.is_some()
2789        })
2790        .cloned()
2791}
2792
2793fn select_previous_submitted_review_run<'a>(
2794    dispatch_history: &'a [ReviewRunRecord],
2795    current_dispatch_id: &str,
2796) -> Option<&'a ReviewRunRecord> {
2797    dispatch_history.iter().find(|record| {
2798        record.dispatch_id != current_dispatch_id
2799            && record.review_submitted
2800            && (record.github_review_url.is_some() || record.github_review_id.is_some())
2801    })
2802}
2803
2804fn latest_pull_request_for_branch(
2805    dispatch_history: &[TaskDispatchRecord],
2806    branch_name: &str,
2807) -> Option<String> {
2808    dispatch_history
2809        .iter()
2810        .find(|record| {
2811            record.branch_name.as_deref() == Some(branch_name)
2812                && record
2813                    .pull_request_url
2814                    .as_deref()
2815                    .map(str::trim)
2816                    .filter(|value| !value.is_empty())
2817                    .is_some()
2818        })
2819        .and_then(|record| record.pull_request_url.clone())
2820}
2821
2822fn unique_worktree_paths(dispatch_history: &[TaskDispatchRecord]) -> Vec<String> {
2823    dispatch_history
2824        .iter()
2825        .filter_map(|record| record.worktree_path.as_deref())
2826        .map(str::trim)
2827        .filter(|value| !value.is_empty())
2828        .map(ToOwned::to_owned)
2829        .collect::<BTreeSet<_>>()
2830        .into_iter()
2831        .collect()
2832}
2833
2834fn unique_run_directories(
2835    dispatch_history: &[TaskDispatchRecord],
2836    remote_agent: &crate::types::RemoteAgentRuntimeConfig,
2837) -> Vec<String> {
2838    dispatch_history
2839        .iter()
2840        .filter_map(|record| derive_remote_run_directory_for_record(record, remote_agent))
2841        .collect::<BTreeSet<_>>()
2842        .into_iter()
2843        .collect()
2844}
2845
2846fn validate_project_metadata_for_dispatch(metadata: &ProjectMetadata) -> Result<(), TrackError> {
2847    if metadata.repo_url.trim().is_empty()
2848        || metadata.git_url.trim().is_empty()
2849        || metadata.base_branch.trim().is_empty()
2850    {
2851        return Err(TrackError::new(
2852            ErrorCode::InvalidProjectMetadata,
2853            "Project metadata must include repo URL, git URL, and base branch before dispatching a remote agent.",
2854        ));
2855    }
2856
2857    parse_github_repository_name(&metadata.repo_url)?;
2858    Ok(())
2859}
2860
2861// =============================================================================
2862// Dispatch Refresh
2863// =============================================================================
2864//
2865// Dispatches usually run for minutes rather than seconds, so the API optimizes
2866// for fewer SSH handshakes instead of ultra-fresh status. We batch all active
2867// dispatch lookups into one SSH round-trip per poll cycle so multiple running
2868// jobs do not multiply connection setup overhead.
2869fn load_dispatch_snapshots_for_records(
2870    ssh_client: &SshClient,
2871    records: &[TaskDispatchRecord],
2872) -> Result<BTreeMap<String, RemoteDispatchSnapshot>, TrackError> {
2873    let mut dispatch_ids = Vec::new();
2874    let mut run_directories = Vec::new();
2875
2876    for record in records {
2877        if !record.status.is_active() {
2878            continue;
2879        }
2880
2881        let Some(worktree_path) = record.worktree_path.as_deref() else {
2882            continue;
2883        };
2884        let Ok(run_directory) = derive_remote_run_directory(worktree_path, &record.dispatch_id)
2885        else {
2886            continue;
2887        };
2888
2889        dispatch_ids.push(record.dispatch_id.clone());
2890        run_directories.push(run_directory);
2891    }
2892
2893    if run_directories.is_empty() {
2894        return Ok(BTreeMap::new());
2895    }
2896
2897    let snapshots = ssh_client.read_dispatch_snapshots(&run_directories)?;
2898    Ok(dispatch_ids.into_iter().zip(snapshots).collect())
2899}
2900
2901fn derive_remote_run_directory(
2902    worktree_path: &str,
2903    dispatch_id: &str,
2904) -> Result<String, TrackError> {
2905    worktree_path
2906        .rsplit_once("/worktrees/")
2907        .map(|(prefix, _suffix)| format!("{prefix}/dispatches/{dispatch_id}"))
2908        .ok_or_else(|| {
2909            TrackError::new(
2910                ErrorCode::RemoteDispatchFailed,
2911                "Could not derive the remote run directory from the worktree path.",
2912            )
2913        })
2914}
2915
2916fn derive_remote_run_directory_for_record(
2917    record: &TaskDispatchRecord,
2918    remote_agent: &crate::types::RemoteAgentRuntimeConfig,
2919) -> Option<String> {
2920    if let Some(worktree_path) = record.worktree_path.as_deref() {
2921        if let Ok(run_directory) = derive_remote_run_directory(worktree_path, &record.dispatch_id) {
2922            return Some(run_directory);
2923        }
2924    }
2925
2926    if record.project.trim().is_empty() || remote_agent.workspace_root.trim().is_empty() {
2927        return None;
2928    }
2929
2930    Some(format!(
2931        "{}/{}/dispatches/{}",
2932        remote_agent.workspace_root.trim_end_matches('/'),
2933        record.project,
2934        record.dispatch_id
2935    ))
2936}
2937
2938fn load_review_snapshots_for_records(
2939    ssh_client: &SshClient,
2940    records: &[ReviewRunRecord],
2941) -> Result<BTreeMap<String, RemoteDispatchSnapshot>, TrackError> {
2942    let mut dispatch_ids = Vec::new();
2943    let mut run_directories = Vec::new();
2944
2945    for record in records {
2946        if !record.status.is_active() {
2947            continue;
2948        }
2949
2950        let Some(worktree_path) = record.worktree_path.as_deref() else {
2951            continue;
2952        };
2953        let Ok(run_directory) = derive_review_run_directory(worktree_path, &record.dispatch_id)
2954        else {
2955            continue;
2956        };
2957
2958        dispatch_ids.push(record.dispatch_id.clone());
2959        run_directories.push(run_directory);
2960    }
2961
2962    if run_directories.is_empty() {
2963        return Ok(BTreeMap::new());
2964    }
2965
2966    let snapshots = ssh_client.read_dispatch_snapshots(&run_directories)?;
2967    Ok(dispatch_ids.into_iter().zip(snapshots).collect())
2968}
2969
2970fn derive_review_run_directory(
2971    worktree_path: &str,
2972    dispatch_id: &str,
2973) -> Result<String, TrackError> {
2974    worktree_path
2975        .rsplit_once(&format!("/{REVIEW_WORKTREE_DIRECTORY_NAME}/"))
2976        .map(|(prefix, _suffix)| format!("{prefix}/{REVIEW_RUN_DIRECTORY_NAME}/{dispatch_id}"))
2977        .ok_or_else(|| {
2978            TrackError::new(
2979                ErrorCode::RemoteDispatchFailed,
2980                "Could not derive the remote review run directory from the worktree path.",
2981            )
2982        })
2983}
2984
2985fn derive_review_run_directory_for_record(
2986    record: &ReviewRunRecord,
2987    remote_agent: &crate::types::RemoteAgentRuntimeConfig,
2988) -> Option<String> {
2989    if let Some(worktree_path) = record.worktree_path.as_deref() {
2990        if let Ok(run_directory) = derive_review_run_directory(worktree_path, &record.dispatch_id) {
2991            return Some(run_directory);
2992        }
2993    }
2994
2995    if record.workspace_key.trim().is_empty() || remote_agent.workspace_root.trim().is_empty() {
2996        return None;
2997    }
2998
2999    Some(format!(
3000        "{}/{}/{}/{}",
3001        remote_agent.workspace_root.trim_end_matches('/'),
3002        record.workspace_key,
3003        REVIEW_RUN_DIRECTORY_NAME,
3004        record.dispatch_id
3005    ))
3006}
3007
3008fn build_create_review_worktree_script() -> String {
3009    format!(
3010        r#"
3011set -eu
3012{path_helpers}
3013CHECKOUT_PATH="$(expand_remote_path "$1")"
3014PULL_REQUEST_NUMBER="$2"
3015BRANCH_NAME="$3"
3016WORKTREE_PATH="$(expand_remote_path "$4")"
3017TARGET_HEAD_OID="${{5:-}}"
3018
3019mkdir -p "$(dirname "$WORKTREE_PATH")"
3020
3021worktree_is_registered() {{
3022  git -C "$CHECKOUT_PATH" worktree list --porcelain | grep -F "worktree $WORKTREE_PATH" >/dev/null 2>&1
3023}}
3024
3025if [ -e "$WORKTREE_PATH" ]; then
3026  if worktree_is_registered; then
3027    git -C "$CHECKOUT_PATH" worktree remove --force "$WORKTREE_PATH" >&2 || true
3028  else
3029    echo "Refusing to overwrite unexpected existing path at $WORKTREE_PATH while preparing a review worktree." >&2
3030    exit 1
3031  fi
3032fi
3033
3034git -C "$CHECKOUT_PATH" worktree prune >&2
3035git -C "$CHECKOUT_PATH" fetch upstream "pull/$PULL_REQUEST_NUMBER/head:$BRANCH_NAME" >&2
3036
3037# Review runs persist the exact PR head they were queued against. We still
3038# refresh the PR ref so the checkout has current GitHub context, but then we
3039# pin the local review branch back to the recorded commit when that object is
3040# available. If the commit is gone, we fail explicitly instead of silently
3041# reviewing a newer PR head than the user requested.
3042TARGET_REF="$BRANCH_NAME"
3043if [ -n "$TARGET_HEAD_OID" ]; then
3044  if ! git -C "$CHECKOUT_PATH" cat-file -e "$TARGET_HEAD_OID^{{commit}}" 2>/dev/null; then
3045    git -C "$CHECKOUT_PATH" fetch upstream "$TARGET_HEAD_OID" >&2 || true
3046  fi
3047
3048  if git -C "$CHECKOUT_PATH" cat-file -e "$TARGET_HEAD_OID^{{commit}}" 2>/dev/null; then
3049    TARGET_REF="$TARGET_HEAD_OID"
3050  else
3051    FETCHED_HEAD_OID="$(git -C "$CHECKOUT_PATH" rev-parse "$BRANCH_NAME^{{commit}}")"
3052    echo "Requested review commit $TARGET_HEAD_OID is not available locally. The fetched PR head is $FETCHED_HEAD_OID, so the review would drift to a newer commit." >&2
3053    exit 1
3054  fi
3055fi
3056
3057git -C "$CHECKOUT_PATH" branch -f "$BRANCH_NAME" "$TARGET_REF" >&2
3058git -C "$CHECKOUT_PATH" worktree add -B "$BRANCH_NAME" "$WORKTREE_PATH" "$TARGET_REF" >&2
3059"#,
3060        path_helpers = remote_path_helpers_shell(),
3061    )
3062}
3063
3064fn unique_review_worktree_paths(dispatch_history: &[ReviewRunRecord]) -> Vec<String> {
3065    dispatch_history
3066        .iter()
3067        .filter_map(|record| record.worktree_path.as_deref())
3068        .map(str::trim)
3069        .filter(|value| !value.is_empty())
3070        .map(ToOwned::to_owned)
3071        .collect::<BTreeSet<_>>()
3072        .into_iter()
3073        .collect()
3074}
3075
3076fn unique_review_run_directories(
3077    dispatch_history: &[ReviewRunRecord],
3078    remote_agent: &crate::types::RemoteAgentRuntimeConfig,
3079) -> Vec<String> {
3080    dispatch_history
3081        .iter()
3082        .filter_map(|record| derive_review_run_directory_for_record(record, remote_agent))
3083        .collect::<BTreeSet<_>>()
3084        .into_iter()
3085        .collect()
3086}
3087
3088fn describe_remote_reset_blockers(
3089    task_dispatches: &[TaskDispatchRecord],
3090    review_dispatches: &[ReviewRunRecord],
3091) -> Vec<String> {
3092    let mut blockers = task_dispatches
3093        .iter()
3094        .filter(|record| record.status.is_active())
3095        .map(|record| format!("task {} ({})", record.task_id, record.dispatch_id))
3096        .collect::<Vec<_>>();
3097    blockers.extend(
3098        review_dispatches
3099            .iter()
3100            .filter(|record| record.status.is_active())
3101            .map(|record| format!("review {} ({})", record.review_id, record.dispatch_id)),
3102    );
3103    blockers
3104}
3105
3106fn build_remote_dispatch_prompt(
3107    project_name: &str,
3108    metadata: &ProjectMetadata,
3109    branch_name: &str,
3110    worktree_path: &str,
3111    task_description: &str,
3112    pull_request_url: Option<&str>,
3113    follow_up_request: Option<&str>,
3114) -> String {
3115    let sections = parse_task_description(task_description);
3116    let mut prompt = String::new();
3117    prompt.push_str("# Remote task dispatch\n\n");
3118    prompt.push_str(
3119        "You are working in a fully autonomous mode on a prepared repository worktree.\n",
3120    );
3121    prompt.push_str("The repository checkout, fork, and worktree are already set up for you.\n");
3122    prompt.push_str("You have full filesystem access, internet access, and `gh` is available.\n");
3123    prompt.push_str("Make the decisions needed to complete the task responsibly.\n");
3124    prompt.push_str(
3125        "The desired outcome is a GitHub PR unless the task is blocked or cannot be solved.\n\n",
3126    );
3127    prompt.push_str("## Repository context\n\n");
3128    prompt.push_str(&format!("- Project: {project_name}\n"));
3129    prompt.push_str(&format!("- Repo URL: {}\n", metadata.repo_url));
3130    prompt.push_str(&format!("- Git URL: {}\n", metadata.git_url));
3131    prompt.push_str(&format!("- Base branch: {}\n", metadata.base_branch));
3132    prompt.push_str(&format!("- Prepared branch: {branch_name}\n"));
3133    prompt.push_str(&format!("- Working directory: {worktree_path}\n\n"));
3134
3135    if let Some(pull_request_url) = pull_request_url.filter(|value| !value.trim().is_empty()) {
3136        prompt.push_str("## Existing PR\n\n");
3137        prompt.push_str(&format!("- Pull request: {pull_request_url}\n"));
3138        prompt.push_str(
3139            "- Continue working on this existing PR with the same prepared branch and worktree.\n",
3140        );
3141        prompt.push_str(
3142            "- Do not open a second PR unless the current PR is unusable and you explain why.\n\n",
3143        );
3144    }
3145
3146    prompt.push_str("## Expectations\n\n");
3147    prompt.push_str("- Pull the task through to a GitHub PR when possible.\n");
3148    prompt.push_str("- Use the current worktree as the only place to make changes.\n");
3149    prompt.push_str("- Use conventional commits for both commit messages and the PR title, for example `feat: Add X`, `fix: Correct Y`, or `chore: Update Z`.\n");
3150    prompt.push_str("- If the follow-up mentions review comments or reviewer feedback, fetch that context with `gh` instead of guessing.\n");
3151    prompt.push_str("- If the follow-up names a reviewer, only act on that reviewer's feedback unless the request explicitly says otherwise.\n");
3152    prompt.push_str("- If the task is blocked, explain the blocker clearly in the final JSON.\n\n");
3153    prompt.push_str("## Task title\n\n");
3154    prompt.push_str(&sections.title);
3155    prompt.push_str("\n\n");
3156
3157    if let Some(summary_markdown) = sections.summary_markdown.as_deref() {
3158        prompt.push_str("## Summary\n\n");
3159        prompt.push_str(summary_markdown);
3160        prompt.push_str("\n\n");
3161    }
3162
3163    if let Some(original_note) = sections.original_note.as_deref() {
3164        prompt.push_str("## Original note\n\n");
3165        prompt.push_str(original_note);
3166        prompt.push_str("\n\n");
3167    }
3168
3169    if let Some(follow_up_request) = follow_up_request.filter(|value| !value.trim().is_empty()) {
3170        prompt.push_str("## Current follow-up request\n\n");
3171        prompt.push_str(follow_up_request.trim());
3172        prompt.push_str("\n\n");
3173    }
3174
3175    prompt.push_str("## Final response\n\n");
3176    prompt.push_str("Return JSON only. The response must match the provided schema exactly.\n");
3177
3178    prompt
3179}
3180
3181fn build_remote_review_prompt(
3182    review: &ReviewRecord,
3183    dispatch_record: &ReviewRunRecord,
3184    previous_submitted_review: Option<&ReviewRunRecord>,
3185) -> String {
3186    let branch_name = dispatch_record
3187        .branch_name
3188        .as_deref()
3189        .expect("queued review dispatches should always have a branch name");
3190    let worktree_path = dispatch_record
3191        .worktree_path
3192        .as_deref()
3193        .expect("queued review dispatches should always have a worktree path");
3194    let mut prompt = String::new();
3195    prompt.push_str("# Remote PR review\n\n");
3196    prompt.push_str(
3197        "You are reviewing an existing GitHub pull request from a prepared repository worktree.\n",
3198    );
3199    prompt.push_str("The repository checkout and review worktree are already prepared for you.\n");
3200    prompt.push_str("You have full filesystem access, internet access, and `gh` is available.\n");
3201    prompt.push_str("This run is for review only: do not push commits, open PRs, or request reviewers yourself.\n");
3202    prompt.push_str("You are responsible for submitting the GitHub review yourself before you return the final JSON.\n\n");
3203    prompt.push_str("## Pull request context\n\n");
3204    prompt.push_str(&format!("- Pull request: {}\n", review.pull_request_url));
3205    prompt.push_str(&format!("- Title: {}\n", review.pull_request_title));
3206    prompt.push_str(&format!("- Repository: {}\n", review.repository_full_name));
3207    prompt.push_str(&format!("- Repo URL: {}\n", review.repo_url));
3208    prompt.push_str(&format!("- Base branch: {}\n", review.base_branch));
3209    prompt.push_str(&format!("- Prepared branch: {branch_name}\n"));
3210    prompt.push_str(&format!("- Working directory: {worktree_path}\n"));
3211    if let Some(target_head_oid) = dispatch_record.target_head_oid.as_deref() {
3212        prompt.push_str(&format!("- Pinned review commit: {target_head_oid}\n"));
3213    }
3214    prompt.push_str("\n");
3215    prompt.push_str("## Review instructions\n\n");
3216    prompt.push_str("- Submit one GitHub review in COMMENT mode.\n");
3217    prompt.push_str(&format!(
3218        "- The first line of the top-level review body must be `@{} requested me to review this PR.`\n",
3219        review.main_user
3220    ));
3221    prompt.push_str("- Prefer inline review comments for concrete file/line findings so people can reply in GitHub threads.\n");
3222    prompt.push_str("- Use the top-level review body for the overall summary, major risks, and any no-findings conclusion.\n");
3223    prompt.push_str(
3224        "- Focus on bugs, regressions, risky behavior changes, missing tests, and edge cases.\n",
3225    );
3226    prompt.push_str("- Use the checked-out code and `gh` to inspect the PR diff and context instead of guessing.\n");
3227    prompt.push_str("- If a pinned review commit is listed above, the prepared worktree is intended to match that exact commit. If it does not, stop and explain the mismatch instead of reviewing a newer head silently.\n");
3228    prompt.push_str("- Keep the review concise but concrete.\n");
3229    prompt.push_str(
3230        "- If you do not find problems, say so explicitly in the top-level review body.\n",
3231    );
3232    prompt.push_str("- If you cannot complete the review responsibly, explain the blocker in the summary and do not claim the review was submitted.\n");
3233    prompt.push_str("- Capture the submitted GitHub review's durable handle from the `gh` response and return it as `githubReviewId` and `githubReviewUrl` when submission succeeds.\n");
3234    prompt.push_str("- Return `reviewSubmitted` as `true` only after GitHub confirms that the review submission succeeded.\n\n");
3235
3236    if let Some(follow_up_request) = dispatch_record.follow_up_request.as_deref() {
3237        prompt.push_str("## Current re-review request\n\n");
3238        prompt.push_str(follow_up_request.trim());
3239        prompt.push_str("\n\n");
3240    }
3241
3242    if let Some(previous_submitted_review) = previous_submitted_review {
3243        prompt.push_str("## Previous bot review context\n\n");
3244        if let Some(github_review_url) = previous_submitted_review.github_review_url.as_deref() {
3245            prompt.push_str(&format!(
3246                "- Previous submitted review: {github_review_url}\n"
3247            ));
3248        }
3249        if let Some(github_review_id) = previous_submitted_review.github_review_id.as_deref() {
3250            prompt.push_str(&format!(
3251                "- Previous submitted review id: {github_review_id}\n"
3252            ));
3253        }
3254        if let Some(target_head_oid) = previous_submitted_review.target_head_oid.as_deref() {
3255            prompt.push_str(&format!(
3256                "- Previous review pinned commit: {target_head_oid}\n"
3257            ));
3258        }
3259        prompt.push_str("\n");
3260        prompt.push_str("## Re-review guidance\n\n");
3261        prompt.push_str("- Inspect the current PR conversation on GitHub before deciding whether an older bot finding still matters.\n");
3262        prompt.push_str(&format!(
3263            "- For context: your previous comments are always non-blocking input at the discretion of the reviewee unless @{} explicitly commented that a finding is valid and should be fixed.\n",
3264            review.main_user
3265        ));
3266        prompt.push_str(&format!(
3267            "- Only treat an older bot finding as something you must actively verify and potentially elevate into a primary finding if @{} explicitly said it is valid and should be fixed.\n",
3268            review.main_user
3269        ));
3270        prompt.push_str(&format!(
3271            "- If @{} or the reviewee explicitly said an older bot finding is not important, disputed it, or chose not to address it, do not repeat it as a primary finding just because it appeared in a previous bot review.\n",
3272            review.main_user
3273        ));
3274        prompt.push_str("- You may mention unresolved prior bot comments as brief context in the top-level summary when helpful, but re-evaluate the current code on its own merits.\n\n");
3275    }
3276
3277    if let Some(default_review_prompt) = review.default_review_prompt.as_deref() {
3278        prompt.push_str("## Default review prompt\n\n");
3279        prompt.push_str(default_review_prompt);
3280        prompt.push_str("\n\n");
3281    }
3282
3283    if let Some(extra_instructions) = review.extra_instructions.as_deref() {
3284        prompt.push_str("## Extra instructions\n\n");
3285        prompt.push_str(extra_instructions);
3286        prompt.push_str("\n\n");
3287    }
3288
3289    prompt.push_str("## Final response\n\n");
3290    prompt.push_str("Return JSON only. The response must match the provided schema exactly.\n");
3291
3292    prompt
3293}
3294
3295fn build_remote_dispatch_schema() -> String {
3296    serde_json::to_string_pretty(&json!({
3297        "type": "object",
3298        "additionalProperties": false,
3299        // Codex's structured-output validator is stricter than generic JSON
3300        // Schema consumers here: every declared property must appear in the
3301        // top-level `required` array, and optionality is expressed with
3302        // `null` in the property's type instead of omitting the field.
3303        "required": [
3304            "status",
3305            "summary",
3306            "pullRequestUrl",
3307            "branchName",
3308            "worktreePath",
3309            "notes"
3310        ],
3311        "properties": {
3312            "status": {
3313                "type": "string",
3314                "enum": ["succeeded", "failed", "blocked"]
3315            },
3316            "summary": {
3317                "type": "string"
3318            },
3319            "pullRequestUrl": {
3320                "type": ["string", "null"]
3321            },
3322            "branchName": {
3323                "type": ["string", "null"]
3324            },
3325            "worktreePath": {
3326                "type": "string"
3327            },
3328            "notes": {
3329                "type": ["string", "null"]
3330            }
3331        }
3332    }))
3333    .expect("dispatch schema serialization should succeed")
3334}
3335
3336fn build_remote_review_schema() -> String {
3337    serde_json::to_string_pretty(&json!({
3338        "type": "object",
3339        "additionalProperties": false,
3340        "required": [
3341            "status",
3342            "summary",
3343            "reviewSubmitted",
3344            "githubReviewId",
3345            "githubReviewUrl",
3346            "worktreePath",
3347            "notes"
3348        ],
3349        "properties": {
3350            "status": {
3351                "type": "string",
3352                "enum": ["succeeded", "failed", "blocked"]
3353            },
3354            "summary": {
3355                "type": "string"
3356            },
3357            "reviewSubmitted": {
3358                "type": "boolean"
3359            },
3360            "githubReviewId": {
3361                "type": ["string", "null"]
3362            },
3363            "githubReviewUrl": {
3364                "type": ["string", "null"]
3365            },
3366            "worktreePath": {
3367                "type": "string"
3368            },
3369            "notes": {
3370                "type": ["string", "null"]
3371            }
3372        }
3373    }))
3374    .expect("review schema serialization should succeed")
3375}
3376
3377fn refresh_dispatch_record_from_snapshot(
3378    mut record: TaskDispatchRecord,
3379    snapshot: &RemoteDispatchSnapshot,
3380) -> Result<TaskDispatchRecord, TrackError> {
3381    let remote_status = snapshot.status.as_deref().unwrap_or_default();
3382    let remote_status = remote_status.trim();
3383    if remote_status.is_empty() {
3384        if let Some(updated) = mark_abandoned_preparing_dispatch(record.clone()) {
3385            return Ok(updated);
3386        }
3387
3388        return Ok(record);
3389    }
3390
3391    if remote_status == "running" {
3392        if record.status == DispatchStatus::Preparing {
3393            record.status = DispatchStatus::Running;
3394            record.updated_at = now_utc();
3395            record.finished_at = None;
3396            record.error_message = None;
3397        }
3398        return Ok(record);
3399    }
3400
3401    if remote_status == "canceled" {
3402        record.status = DispatchStatus::Canceled;
3403        record.updated_at = now_utc();
3404        record.finished_at = Some(parse_remote_finished_at(
3405            snapshot.finished_at.as_deref(),
3406            now_utc(),
3407        ));
3408        record.summary = Some(
3409            record
3410                .summary
3411                .unwrap_or_else(|| "Canceled from the web UI.".to_owned()),
3412        );
3413        record.error_message = None;
3414        return Ok(record);
3415    }
3416
3417    let now = now_utc();
3418    record.updated_at = now;
3419    if remote_status == "completed" {
3420        let remote_result = snapshot.result.as_deref().ok_or_else(|| {
3421            TrackError::new(
3422                ErrorCode::RemoteDispatchFailed,
3423                "Remote agent run completed without producing a structured result.",
3424            )
3425        })?;
3426        let outcome = parse_remote_agent_output::<RemoteAgentDispatchOutcome>(
3427            remote_result,
3428            record.preferred_tool,
3429            "Remote agent result",
3430        )?;
3431        record.status = outcome.status;
3432        record.summary = Some(outcome.summary);
3433        record.pull_request_url = outcome.pull_request_url;
3434        record.branch_name = outcome.branch_name.or(record.branch_name);
3435        record.worktree_path = Some(outcome.worktree_path);
3436        record.notes = outcome.notes;
3437        record.error_message = None;
3438        record.finished_at = Some(parse_remote_finished_at(
3439            snapshot.finished_at.as_deref(),
3440            now,
3441        ));
3442        return Ok(record);
3443    }
3444
3445    record.status = DispatchStatus::Failed;
3446    record.finished_at = Some(parse_remote_finished_at(
3447        snapshot.finished_at.as_deref(),
3448        now,
3449    ));
3450    record.error_message = snapshot
3451        .stderr
3452        .as_deref()
3453        .map(str::trim)
3454        .filter(|value| !value.is_empty())
3455        .map(|value| value.to_owned())
3456        .or_else(|| {
3457            Some("Remote agent run failed before returning a structured result.".to_owned())
3458        });
3459    Ok(record)
3460}
3461
3462fn mark_abandoned_preparing_dispatch(mut record: TaskDispatchRecord) -> Option<TaskDispatchRecord> {
3463    if record.status != DispatchStatus::Preparing {
3464        return None;
3465    }
3466
3467    let now = now_utc();
3468    if now - record.updated_at < PREPARING_STALE_AFTER {
3469        return None;
3470    }
3471
3472    record.status = DispatchStatus::Failed;
3473    record.updated_at = now;
3474    record.finished_at = Some(now);
3475    record.error_message =
3476        Some("Dispatch preparation stopped before the remote agent launched.".to_owned());
3477    Some(record)
3478}
3479
3480fn mark_abandoned_preparing_review_dispatch(
3481    mut record: ReviewRunRecord,
3482) -> Option<ReviewRunRecord> {
3483    if record.status != DispatchStatus::Preparing {
3484        return None;
3485    }
3486
3487    let now = now_utc();
3488    if now - record.updated_at < PREPARING_STALE_AFTER {
3489        return None;
3490    }
3491
3492    record.status = DispatchStatus::Failed;
3493    record.updated_at = now;
3494    record.finished_at = Some(now);
3495    record.error_message =
3496        Some("Review preparation stopped before the remote agent launched.".to_owned());
3497    Some(record)
3498}
3499
3500fn parse_remote_finished_at(
3501    value: Option<&str>,
3502    fallback: time::OffsetDateTime,
3503) -> time::OffsetDateTime {
3504    value
3505        .map(str::trim)
3506        .filter(|value| !value.is_empty())
3507        .and_then(|value| parse_iso_8601_seconds(value).ok())
3508        .unwrap_or(fallback)
3509}
3510
3511fn mark_terminal_refresh_failure(
3512    mut record: TaskDispatchRecord,
3513    snapshot: &RemoteDispatchSnapshot,
3514    error: &TrackError,
3515) -> Option<TaskDispatchRecord> {
3516    let remote_status = snapshot.status.as_deref().unwrap_or_default().trim();
3517    if remote_status != "completed" && remote_status != "launcher_failed" {
3518        return None;
3519    }
3520
3521    let now = now_utc();
3522    record.status = DispatchStatus::Failed;
3523    record.updated_at = now;
3524    record.finished_at = Some(parse_remote_finished_at(
3525        snapshot.finished_at.as_deref(),
3526        now,
3527    ));
3528    record.error_message = Some(error.to_string());
3529    Some(record)
3530}
3531
3532fn mark_terminal_review_refresh_failure(
3533    mut record: ReviewRunRecord,
3534    snapshot: &RemoteDispatchSnapshot,
3535    error: &TrackError,
3536) -> Option<ReviewRunRecord> {
3537    let remote_status = snapshot.status.as_deref().unwrap_or_default().trim();
3538    if remote_status != "completed" && remote_status != "launcher_failed" {
3539        return None;
3540    }
3541
3542    let now = now_utc();
3543    record.status = DispatchStatus::Failed;
3544    record.updated_at = now;
3545    record.finished_at = Some(parse_remote_finished_at(
3546        snapshot.finished_at.as_deref(),
3547        now,
3548    ));
3549    record.error_message = Some(error.to_string());
3550    Some(record)
3551}
3552
3553fn load_remote_registry(
3554    ssh_client: &SshClient,
3555    registry_path: &str,
3556) -> Result<RemoteProjectRegistryFile, TrackError> {
3557    let Some(raw_registry) = ssh_client.read_remote_file(registry_path)? else {
3558        return Ok(RemoteProjectRegistryFile::default());
3559    };
3560
3561    serde_json::from_str::<RemoteProjectRegistryFile>(&raw_registry).map_err(|error| {
3562        TrackError::new(
3563            ErrorCode::RemoteDispatchFailed,
3564            format!("Remote projects registry is not valid JSON: {error}"),
3565        )
3566    })
3567}
3568
3569fn write_remote_registry(
3570    ssh_client: &SshClient,
3571    registry_path: &str,
3572    registry: &RemoteProjectRegistryFile,
3573) -> Result<(), TrackError> {
3574    let serialized = serde_json::to_string_pretty(registry).map_err(|error| {
3575        TrackError::new(
3576            ErrorCode::DispatchWriteFailed,
3577            format!("Could not serialize the remote projects registry: {error}"),
3578        )
3579    })?;
3580    ssh_client.upload_remote_file(registry_path, &serialized)
3581}
3582
3583fn parse_github_repository_name(repo_url: &str) -> Result<String, TrackError> {
3584    let trimmed = repo_url.trim().trim_end_matches('/');
3585    let without_suffix = trimmed.trim_end_matches(".git");
3586    let Some(repository_name) = without_suffix.rsplit('/').next() else {
3587        return Err(TrackError::new(
3588            ErrorCode::RemoteDispatchFailed,
3589            format!("Repo URL {repo_url} does not look like a GitHub repository."),
3590        ));
3591    };
3592
3593    if !without_suffix.contains("github.com/") || repository_name.is_empty() {
3594        return Err(TrackError::new(
3595            ErrorCode::RemoteDispatchFailed,
3596            format!("Repo URL {repo_url} does not look like a GitHub repository."),
3597        ));
3598    }
3599
3600    Ok(repository_name.to_owned())
3601}
3602
3603fn parse_github_pull_request_reference(
3604    pull_request_url: &str,
3605) -> Result<GithubPullRequestReference, TrackError> {
3606    let trimmed = pull_request_url.trim().trim_end_matches('/');
3607    let without_scheme = trimmed.strip_prefix("https://github.com/").ok_or_else(|| {
3608        TrackError::new(
3609            ErrorCode::RemoteDispatchFailed,
3610            format!(
3611                "Pull request URL {pull_request_url} does not look like a GitHub pull request."
3612            ),
3613        )
3614    })?;
3615    let parts = without_scheme.split('/').collect::<Vec<_>>();
3616    if parts.len() != 4 || parts[2] != "pull" {
3617        return Err(TrackError::new(
3618            ErrorCode::RemoteDispatchFailed,
3619            format!(
3620                "Pull request URL {pull_request_url} does not look like a GitHub pull request."
3621            ),
3622        ));
3623    }
3624
3625    let number = parts[3].parse::<u64>().map_err(|_| {
3626        TrackError::new(
3627            ErrorCode::RemoteDispatchFailed,
3628            format!("Pull request URL {pull_request_url} does not contain a valid PR number."),
3629        )
3630    })?;
3631
3632    Ok(GithubPullRequestReference {
3633        owner: parts[0].to_owned(),
3634        repository: parts[1].to_owned(),
3635        number,
3636    })
3637}
3638
3639fn build_review_workspace_key(pull_request: &GithubPullRequestMetadata) -> String {
3640    let slug = slug::slugify(
3641        pull_request
3642            .repository_full_name
3643            .replace('/', "-")
3644            .trim()
3645            .to_owned(),
3646    );
3647
3648    if slug.is_empty() {
3649        "review-repo".to_owned()
3650    } else {
3651        slug
3652    }
3653}
3654
3655fn build_review_follow_up_request(
3656    pull_request_url: &str,
3657    main_user: &str,
3658    dispatch_started_at: time::OffsetDateTime,
3659) -> String {
3660    format!(
3661        "Respond to new review feedback from @{main_user} on the existing PR.\n\n\
3662Use `gh` to fetch submitted PR reviews and inline review comments from @{main_user} only.\n\
3663Only use reviews with state COMMENTED or CHANGES_REQUESTED that were submitted after {dispatch_started_at}.\n\
3664Ignore APPROVED reviews and all feedback from other users.\n\
3665Keep using the existing PR at {pull_request_url} unless you explain why that is impossible.",
3666        dispatch_started_at = format_iso_8601_millis(dispatch_started_at),
3667    )
3668}
3669
3670fn build_review_follow_up_notification_comment(main_user: &str, head_oid: &str) -> String {
3671    let short_head_oid = head_oid.get(..7).unwrap_or(head_oid);
3672
3673    format!(
3674        "@{main_user} new bot updates are ready on commit `{short_head_oid}`. \
3675Please leave a PR review (COMMENTED or CHANGES_REQUESTED) if you want the bot to follow up automatically."
3676    )
3677}
3678
3679fn github_pull_request_endpoint(reference: &GithubPullRequestReference) -> String {
3680    format!(
3681        "repos/{}/{}/pulls/{}",
3682        reference.owner, reference.repository, reference.number
3683    )
3684}
3685
3686fn github_pull_request_reviews_endpoint(reference: &GithubPullRequestReference) -> String {
3687    format!(
3688        "{}/reviews?per_page=100",
3689        github_pull_request_endpoint(reference)
3690    )
3691}
3692
3693fn github_pull_request_issue_comments_endpoint(reference: &GithubPullRequestReference) -> String {
3694    format!(
3695        "repos/{}/{}/issues/{}/comments",
3696        reference.owner, reference.repository, reference.number
3697    )
3698}
3699
3700fn contextualize_track_error(error: TrackError, context: impl Into<String>) -> TrackError {
3701    TrackError::new(
3702        error.code,
3703        format!("{}: {}", context.into(), error.message()),
3704    )
3705}
3706
3707fn review_follow_up_event(
3708    outcome: &str,
3709    detail: impl Into<String>,
3710    dispatch_record: &TaskDispatchRecord,
3711    reviewer: &str,
3712    pull_request_state: Option<&GithubPullRequestReviewState>,
3713) -> RemoteReviewFollowUpEvent {
3714    let latest_review_state = pull_request_state
3715        .and_then(|state| state.latest_eligible_review.as_ref())
3716        .map(|review| review.state.clone());
3717    let latest_review_submitted_at = pull_request_state
3718        .and_then(|state| state.latest_eligible_review.as_ref())
3719        .map(|review| format_iso_8601_millis(review.submitted_at));
3720
3721    RemoteReviewFollowUpEvent {
3722        outcome: outcome.to_owned(),
3723        detail: detail.into(),
3724        task_id: dispatch_record.task_id.clone(),
3725        dispatch_id: dispatch_record.dispatch_id.clone(),
3726        dispatch_status: dispatch_record.status.as_str().to_owned(),
3727        remote_host: dispatch_record.remote_host.clone(),
3728        branch_name: dispatch_record.branch_name.clone(),
3729        pull_request_url: dispatch_record.pull_request_url.clone(),
3730        reviewer: reviewer.to_owned(),
3731        pr_is_open: pull_request_state.map(|state| state.is_open),
3732        pr_head_oid: pull_request_state.map(|state| state.head_oid.clone()),
3733        latest_review_state,
3734        latest_review_submitted_at,
3735    }
3736}
3737
3738fn remote_path_helpers_shell() -> &'static str {
3739    r#"
3740expand_remote_path() {
3741  case "$1" in
3742    "~")
3743      printf '%s\n' "$HOME"
3744      ;;
3745    "~/"*)
3746      printf '%s/%s\n' "$HOME" "${1#~/}"
3747      ;;
3748    *)
3749      printf '%s\n' "$1"
3750      ;;
3751  esac
3752}
3753"#
3754}
3755
3756fn render_remote_script_with_shell_prelude(script: &str, shell_prelude: &str) -> String {
3757    let mut rendered = String::from("set -e\n");
3758
3759    // The runner prelude is intentionally user-managed shell code. We execute
3760    // it before each remote command so PATH setup and toolchain activation stay
3761    // explicit instead of depending on interactive shell startup files that
3762    // SSH automation does not reliably inherit.
3763    if !shell_prelude.trim().is_empty() {
3764        rendered.push_str(shell_prelude);
3765        if !shell_prelude.ends_with('\n') {
3766            rendered.push('\n');
3767        }
3768    }
3769
3770    rendered.push('\n');
3771    rendered.push_str(script.trim_start_matches('\n'));
3772    rendered
3773}
3774
3775fn build_remote_agent_command(preferred_tool: RemoteAgentPreferredTool) -> String {
3776    match preferred_tool {
3777        RemoteAgentPreferredTool::Codex => format!(
3778            "codex --search exec --dangerously-bypass-approvals-and-sandbox -C \"$WORKTREE_PATH\" --json --output-schema \"$RUN_DIR/{REMOTE_SCHEMA_FILE_NAME}\" -o \"$RUN_DIR/{REMOTE_RESULT_FILE_NAME}\" - < \"$RUN_DIR/{REMOTE_PROMPT_FILE_NAME}\" > \"$RUN_DIR/events.jsonl\" 2> \"$RUN_DIR/{REMOTE_STDERR_FILE_NAME}\" &\n"
3779        ),
3780        RemoteAgentPreferredTool::Claude => {
3781            let mut command = String::new();
3782            command.push_str(&format!(
3783                "SCHEMA_CONTENT=\"$(tr -d '\\n' < \"$RUN_DIR/{REMOTE_SCHEMA_FILE_NAME}\")\"\n"
3784            ));
3785            command.push_str("cd \"$WORKTREE_PATH\"\n");
3786            // Claude Code's JSON-schema mode emits request metadata plus the
3787            // validated payload under `structured_output`, so we launch it in
3788            // JSON mode and let the refresh path unwrap that envelope later.
3789            command.push_str(&format!(
3790                "claude -p --dangerously-skip-permissions --add-dir \"$WORKTREE_PATH\" --output-format json --json-schema \"$SCHEMA_CONTENT\" < \"$RUN_DIR/{REMOTE_PROMPT_FILE_NAME}\" > \"$RUN_DIR/{REMOTE_RESULT_FILE_NAME}\" 2> \"$RUN_DIR/{REMOTE_STDERR_FILE_NAME}\" &\n"
3791            ));
3792            command
3793        }
3794    }
3795}
3796
3797#[derive(Debug, Deserialize)]
3798struct ClaudeStructuredOutputEnvelope<T> {
3799    #[serde(rename = "structured_output")]
3800    structured_output: T,
3801}
3802
3803fn parse_remote_agent_output<T>(
3804    raw_result: &str,
3805    preferred_tool: RemoteAgentPreferredTool,
3806    result_label: &str,
3807) -> Result<T, TrackError>
3808where
3809    T: DeserializeOwned,
3810{
3811    match serde_json::from_str::<T>(raw_result) {
3812        Ok(outcome) => Ok(outcome),
3813        Err(direct_error) if preferred_tool == RemoteAgentPreferredTool::Claude => {
3814            // Codex writes the structured payload directly, while Claude wraps
3815            // it in a metadata envelope. Accepting both shapes keeps existing
3816            // fixtures and any transitional persisted results readable.
3817            serde_json::from_str::<ClaudeStructuredOutputEnvelope<T>>(raw_result)
3818                .map(|envelope| envelope.structured_output)
3819                .map_err(|envelope_error| {
3820                    TrackError::new(
3821                        ErrorCode::RemoteDispatchFailed,
3822                        format!(
3823                            "{result_label} did not match the expected direct or Claude structured-output format: direct parse failed with {direct_error}; envelope parse failed with {envelope_error}",
3824                        ),
3825                    )
3826                })
3827        }
3828        Err(error) => Err(TrackError::new(
3829            ErrorCode::RemoteDispatchFailed,
3830            format!("{result_label} is not valid JSON: {error}"),
3831        )),
3832    }
3833}
3834
3835fn build_remote_agent_launcher(
3836    preferred_tool: RemoteAgentPreferredTool,
3837    shell_prelude: &str,
3838) -> String {
3839    let mut launcher = String::from("#!/usr/bin/env bash\n");
3840    launcher.push_str("set -e\n");
3841    if !shell_prelude.trim().is_empty() {
3842        launcher.push_str(shell_prelude);
3843        if !shell_prelude.ends_with('\n') {
3844            launcher.push('\n');
3845        }
3846    }
3847
3848    launcher.push_str("set -eu\n");
3849    launcher.push_str("RUN_DIR=\"$1\"\n");
3850    launcher.push_str("WORKTREE_PATH=\"$2\"\n");
3851    launcher.push_str(&format!(
3852        "printf '%s\\n' \"$$\" > \"$RUN_DIR/{REMOTE_LAUNCHER_PID_FILE_NAME}\"\n"
3853    ));
3854    launcher.push_str("cancel_run() {\n");
3855    launcher.push_str(&format!(
3856        "  if [ -f \"$RUN_DIR/{REMOTE_CODEX_PID_FILE_NAME}\" ]; then\n"
3857    ));
3858    launcher.push_str(&format!(
3859        "    CODEX_PID=\"$(tr -d '[:space:]' < \"$RUN_DIR/{REMOTE_CODEX_PID_FILE_NAME}\")\"\n"
3860    ));
3861    launcher.push_str("    if [ -n \"$CODEX_PID\" ] && kill -0 \"$CODEX_PID\" 2>/dev/null; then\n");
3862    launcher.push_str("      kill \"$CODEX_PID\" 2>/dev/null || true\n");
3863    launcher.push_str("    fi\n");
3864    launcher.push_str("  fi\n");
3865    launcher.push_str(&format!(
3866        "  printf 'canceled\\n' > \"$RUN_DIR/{REMOTE_STATUS_FILE_NAME}\"\n"
3867    ));
3868    launcher.push_str(&format!(
3869        "  date -u +%Y-%m-%dT%H:%M:%SZ > \"$RUN_DIR/{REMOTE_FINISHED_AT_FILE_NAME}\"\n"
3870    ));
3871    launcher.push_str("  exit 130\n");
3872    launcher.push_str("}\n");
3873    launcher.push_str("trap cancel_run TERM INT\n");
3874    launcher.push_str(&format!(
3875        "printf 'running\\n' > \"$RUN_DIR/{REMOTE_STATUS_FILE_NAME}\"\n"
3876    ));
3877    launcher.push_str(&build_remote_agent_command(preferred_tool));
3878    launcher.push_str("CODEX_PID=\"$!\"\n");
3879    launcher.push_str(&format!(
3880        "printf '%s\\n' \"$CODEX_PID\" > \"$RUN_DIR/{REMOTE_CODEX_PID_FILE_NAME}\"\n"
3881    ));
3882    launcher.push_str("if wait \"$CODEX_PID\"; then\n");
3883    launcher.push_str(&format!(
3884        "  printf 'completed\\n' > \"$RUN_DIR/{REMOTE_STATUS_FILE_NAME}\"\n"
3885    ));
3886    launcher.push_str("else\n");
3887    launcher.push_str("  EXIT_CODE=\"$?\"\n");
3888    launcher.push_str(&format!(
3889        "  CURRENT_STATUS=\"$(tr -d '[:space:]' < \"$RUN_DIR/{REMOTE_STATUS_FILE_NAME}\" 2>/dev/null || true)\"\n"
3890    ));
3891    launcher.push_str(
3892        "  if [ \"$CURRENT_STATUS\" != \"canceled\" ] && [ \"$EXIT_CODE\" -ne 130 ]; then\n",
3893    );
3894    launcher.push_str(&format!(
3895        "    printf 'launcher_failed\\n' > \"$RUN_DIR/{REMOTE_STATUS_FILE_NAME}\"\n"
3896    ));
3897    launcher.push_str("  fi\n");
3898    launcher.push_str("fi\n");
3899    launcher.push_str(&format!(
3900        "date -u +%Y-%m-%dT%H:%M:%SZ > \"$RUN_DIR/{REMOTE_FINISHED_AT_FILE_NAME}\"\n"
3901    ));
3902    launcher
3903}
3904
3905struct SshClient {
3906    host: String,
3907    key_path: PathBuf,
3908    known_hosts_path: PathBuf,
3909    port: u16,
3910    shell_prelude: String,
3911    user: String,
3912}
3913
3914impl SshClient {
3915    fn new(config: &crate::types::RemoteAgentRuntimeConfig) -> Result<Self, TrackError> {
3916        if let Some(parent_directory) = config.managed_known_hosts_path.parent() {
3917            fs::create_dir_all(parent_directory).map_err(|error| {
3918                TrackError::new(
3919                    ErrorCode::RemoteDispatchFailed,
3920                    format!(
3921                        "Could not create the managed known_hosts directory at {}: {error}",
3922                        collapse_home_path(parent_directory)
3923                    ),
3924                )
3925            })?;
3926        }
3927
3928        if !config.managed_known_hosts_path.exists() {
3929            fs::write(&config.managed_known_hosts_path, "").map_err(|error| {
3930                TrackError::new(
3931                    ErrorCode::RemoteDispatchFailed,
3932                    format!(
3933                        "Could not create the managed known_hosts file at {}: {error}",
3934                        collapse_home_path(&config.managed_known_hosts_path)
3935                    ),
3936                )
3937            })?;
3938        }
3939
3940        Ok(Self {
3941            host: config.host.clone(),
3942            key_path: config.managed_key_path.clone(),
3943            known_hosts_path: config.managed_known_hosts_path.clone(),
3944            port: config.port,
3945            shell_prelude: config.shell_prelude.clone().unwrap_or_default(),
3946            user: config.user.clone(),
3947        })
3948    }
3949
3950    fn fetch_github_login(&self) -> Result<String, TrackError> {
3951        let login = self.run_script(
3952            r#"
3953set -eu
3954gh api user --jq .login
3955"#,
3956            &[],
3957        )?;
3958
3959        let login = login.trim().to_owned();
3960        if login.is_empty() {
3961            return Err(TrackError::new(
3962                ErrorCode::RemoteDispatchFailed,
3963                "Remote `gh` authentication did not return a GitHub login.",
3964            ));
3965        }
3966
3967        Ok(login)
3968    }
3969
3970    fn fetch_pull_request_metadata(
3971        &self,
3972        pull_request_url: &str,
3973    ) -> Result<GithubPullRequestMetadata, TrackError> {
3974        let reference = parse_github_pull_request_reference(pull_request_url)?;
3975        let pull_request_endpoint = github_pull_request_endpoint(&reference);
3976        let pull_request_json = self
3977            .run_script(
3978                r#"
3979set -eu
3980ENDPOINT="$1"
3981gh api "$ENDPOINT"
3982"#,
3983                std::slice::from_ref(&pull_request_endpoint),
3984            )
3985            .map_err(|error| {
3986                contextualize_track_error(
3987                    error,
3988                    format!(
3989                        "Remote `gh api` on {}@{} could not fetch PR details for {} via endpoint `{}`",
3990                        self.user, self.host, pull_request_url, pull_request_endpoint
3991                    ),
3992                )
3993            })?;
3994        let pull_request =
3995            serde_json::from_str::<GithubPullRequestApiResponse>(&pull_request_json).map_err(
3996                |error| {
3997                    TrackError::new(
3998                        ErrorCode::RemoteDispatchFailed,
3999                        format!(
4000                            "GitHub PR details from endpoint `{pull_request_endpoint}` are not valid JSON: {error}"
4001                        ),
4002                    )
4003                },
4004            )?;
4005
4006        if pull_request.state != "open" || pull_request.merged_at.is_some() {
4007            return Err(TrackError::new(
4008                ErrorCode::RemoteDispatchFailed,
4009                format!("Pull request {pull_request_url} is not open anymore."),
4010            ));
4011        }
4012
4013        Ok(GithubPullRequestMetadata {
4014            pull_request_url: pull_request_url.trim().to_owned(),
4015            pull_request_number: reference.number,
4016            pull_request_title: pull_request.title,
4017            repository_full_name: format!("{}/{}", reference.owner, reference.repository),
4018            repo_url: format!(
4019                "https://github.com/{}/{}",
4020                reference.owner, reference.repository
4021            ),
4022            git_url: format!(
4023                "git@github.com:{}/{}.git",
4024                reference.owner, reference.repository
4025            ),
4026            base_branch: pull_request.base.branch_ref,
4027            head_oid: pull_request.head.sha,
4028        })
4029    }
4030
4031    fn fetch_pull_request_review_state(
4032        &self,
4033        pull_request_url: &str,
4034        main_user: &str,
4035    ) -> Result<GithubPullRequestReviewState, TrackError> {
4036        let reference = parse_github_pull_request_reference(pull_request_url)?;
4037        let pull_request_endpoint = github_pull_request_endpoint(&reference);
4038        let pull_request_json = self
4039            .run_script(
4040                r#"
4041set -eu
4042ENDPOINT="$1"
4043gh api "$ENDPOINT"
4044"#,
4045                std::slice::from_ref(&pull_request_endpoint),
4046            )
4047            .map_err(|error| {
4048                contextualize_track_error(
4049                    error,
4050                    format!(
4051                    "Remote `gh api` on {}@{} could not fetch PR details for {} via endpoint `{}`",
4052                    self.user, self.host, pull_request_url, pull_request_endpoint
4053                ),
4054                )
4055            })?;
4056        let pull_request =
4057            serde_json::from_str::<GithubPullRequestApiResponse>(&pull_request_json).map_err(
4058                |error| {
4059                    TrackError::new(
4060                        ErrorCode::RemoteDispatchFailed,
4061                        format!(
4062                            "GitHub PR details from endpoint `{pull_request_endpoint}` are not valid JSON: {error}"
4063                        ),
4064                    )
4065                },
4066            )?;
4067
4068        let reviews_endpoint = github_pull_request_reviews_endpoint(&reference);
4069        let reviews_json = self
4070            .run_script(
4071                r#"
4072set -eu
4073ENDPOINT="$1"
4074gh api "$ENDPOINT"
4075"#,
4076                std::slice::from_ref(&reviews_endpoint),
4077            )
4078            .map_err(|error| {
4079                contextualize_track_error(
4080                    error,
4081                    format!(
4082                    "Remote `gh api` on {}@{} could not fetch PR reviews for {} via endpoint `{}`",
4083                    self.user, self.host, pull_request_url, reviews_endpoint
4084                ),
4085                )
4086            })?;
4087        let reviews = serde_json::from_str::<Vec<GithubReviewApiResponse>>(&reviews_json).map_err(
4088            |error| {
4089                TrackError::new(
4090                    ErrorCode::RemoteDispatchFailed,
4091                    format!(
4092                        "GitHub PR reviews from endpoint `{reviews_endpoint}` are not valid JSON: {error}"
4093                    ),
4094                )
4095            },
4096        )?;
4097
4098        let latest_eligible_review = reviews
4099            .into_iter()
4100            .filter_map(|review| {
4101                let reviewer = review.user?.login;
4102                if reviewer != main_user {
4103                    return None;
4104                }
4105
4106                if review.state != "COMMENTED" && review.state != "CHANGES_REQUESTED" {
4107                    return None;
4108                }
4109
4110                let submitted_at = review
4111                    .submitted_at
4112                    .as_deref()
4113                    .and_then(|value| parse_iso_8601_seconds(value).ok())?;
4114
4115                Some(GithubSubmittedReview {
4116                    state: review.state,
4117                    submitted_at,
4118                })
4119            })
4120            .max_by_key(|review| review.submitted_at);
4121
4122        Ok(GithubPullRequestReviewState {
4123            is_open: pull_request.state == "open" && pull_request.merged_at.is_none(),
4124            head_oid: pull_request.head.sha,
4125            latest_eligible_review,
4126        })
4127    }
4128
4129    fn post_pull_request_comment(
4130        &self,
4131        pull_request_url: &str,
4132        comment_body: &str,
4133    ) -> Result<(), TrackError> {
4134        let reference = parse_github_pull_request_reference(pull_request_url)?;
4135        let issue_comments_endpoint = github_pull_request_issue_comments_endpoint(&reference);
4136        self.run_script(
4137            r#"
4138set -eu
4139ENDPOINT="$1"
4140BODY="$2"
4141gh api --method POST "$ENDPOINT" -f body="$BODY" >/dev/null
4142"#,
4143            &[issue_comments_endpoint.clone(), comment_body.to_owned()],
4144        )
4145        .map_err(|error| {
4146            contextualize_track_error(
4147                error,
4148                format!(
4149                    "Remote `gh api` on {}@{} could not post a PR comment for {} via endpoint `{}`",
4150                    self.user, self.host, pull_request_url, issue_comments_endpoint
4151                ),
4152            )
4153        })?;
4154
4155        Ok(())
4156    }
4157
4158    fn ensure_checkout(
4159        &self,
4160        metadata: &ProjectMetadata,
4161        repository_name: &str,
4162        checkout_path: &str,
4163        github_login: &str,
4164    ) -> Result<String, TrackError> {
4165        let ensure_checkout_script = format!(
4166            r#"
4167set -eu
4168{path_helpers}
4169REPO_URL="$1"
4170REPOSITORY_NAME="$2"
4171GIT_URL="$3"
4172BASE_BRANCH="$4"
4173CHECKOUT_PATH="$(expand_remote_path "$5")"
4174GITHUB_LOGIN="$6"
4175
4176mkdir -p "$(dirname "$CHECKOUT_PATH")"
4177
4178# Remote automation runs on fresh machines too, so Git cannot assume that
4179# GitHub already exists in the remote user's known_hosts file. We explicitly
4180# manage a predictable known_hosts path here and tell Git to accept the first
4181# key it sees. That keeps the initial clone/fetch flow unattended while still
4182# recording the host key for the next command.
4183REMOTE_SSH_DIR="$HOME/.ssh"
4184REMOTE_KNOWN_HOSTS_PATH="$REMOTE_SSH_DIR/known_hosts"
4185mkdir -p "$REMOTE_SSH_DIR"
4186chmod 700 "$REMOTE_SSH_DIR"
4187touch "$REMOTE_KNOWN_HOSTS_PATH"
4188chmod 600 "$REMOTE_KNOWN_HOSTS_PATH"
4189export GIT_SSH_COMMAND="ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$REMOTE_KNOWN_HOSTS_PATH"
4190
4191resolve_fork_git_url() {{
4192  gh repo view "$GITHUB_LOGIN/$REPOSITORY_NAME" --json sshUrl --jq .sshUrl 2>/dev/null || true
4193}}
4194
4195FORK_GIT_URL="$(resolve_fork_git_url)"
4196if [ -z "$FORK_GIT_URL" ]; then
4197  gh repo fork "$REPO_URL" >/dev/null
4198  FORK_GIT_URL="$(resolve_fork_git_url)"
4199fi
4200
4201if [ -z "$FORK_GIT_URL" ]; then
4202  echo "Could not determine the fork SSH URL for $GITHUB_LOGIN/$REPOSITORY_NAME after creating the fork." >&2
4203  exit 1
4204fi
4205
4206if [ ! -d "$CHECKOUT_PATH/.git" ]; then
4207  git clone "$FORK_GIT_URL" "$CHECKOUT_PATH" >&2
4208fi
4209
4210cd "$CHECKOUT_PATH"
4211if git remote get-url origin >/dev/null 2>&1; then
4212  git remote set-url origin "$FORK_GIT_URL"
4213else
4214  git remote add origin "$FORK_GIT_URL"
4215fi
4216
4217if git remote get-url upstream >/dev/null 2>&1; then
4218  git remote set-url upstream "$GIT_URL"
4219else
4220  git remote add upstream "$GIT_URL"
4221fi
4222
4223git fetch origin --prune >&2
4224git fetch upstream --prune >&2
4225
4226if git show-ref --verify --quiet "refs/heads/$BASE_BRANCH"; then
4227  git checkout "$BASE_BRANCH" >&2
4228else
4229  git checkout -B "$BASE_BRANCH" "upstream/$BASE_BRANCH" >&2
4230fi
4231
4232git reset --hard "upstream/$BASE_BRANCH" >&2
4233git clean -fd >&2
4234
4235printf '%s\n' "$FORK_GIT_URL"
4236"#,
4237            path_helpers = remote_path_helpers_shell(),
4238        );
4239        let fork_git_url = self.run_script(
4240            &ensure_checkout_script,
4241            &[
4242                metadata.repo_url.clone(),
4243                repository_name.to_owned(),
4244                metadata.git_url.clone(),
4245                metadata.base_branch.clone(),
4246                checkout_path.to_owned(),
4247                github_login.to_owned(),
4248            ],
4249        )?;
4250
4251        let fork_git_url = fork_git_url.trim().to_owned();
4252        if fork_git_url.is_empty() {
4253            return Err(TrackError::new(
4254                ErrorCode::RemoteDispatchFailed,
4255                "Remote fork setup did not return a fork Git URL.",
4256            ));
4257        }
4258
4259        Ok(fork_git_url)
4260    }
4261
4262    fn create_worktree(
4263        &self,
4264        checkout_path: &str,
4265        base_branch: &str,
4266        branch_name: &str,
4267        worktree_path: &str,
4268    ) -> Result<(), TrackError> {
4269        let create_worktree_script = format!(
4270            r#"
4271set -eu
4272{path_helpers}
4273CHECKOUT_PATH="$(expand_remote_path "$1")"
4274BASE_BRANCH="$2"
4275BRANCH_NAME="$3"
4276WORKTREE_PATH="$(expand_remote_path "$4")"
4277
4278mkdir -p "$(dirname "$WORKTREE_PATH")"
4279
4280worktree_is_registered() {{
4281  git -C "$CHECKOUT_PATH" worktree list --porcelain | grep -F "worktree $WORKTREE_PATH" >/dev/null 2>&1
4282}}
4283
4284if [ -e "$WORKTREE_PATH" ]; then
4285  if worktree_is_registered; then
4286    git -C "$CHECKOUT_PATH" worktree remove --force "$WORKTREE_PATH" >&2 || true
4287  else
4288    echo "Refusing to overwrite unexpected existing path at $WORKTREE_PATH while preparing a fresh dispatch worktree." >&2
4289    exit 1
4290  fi
4291fi
4292
4293git -C "$CHECKOUT_PATH" worktree prune >&2
4294git -C "$CHECKOUT_PATH" worktree add -B "$BRANCH_NAME" "$WORKTREE_PATH" "upstream/$BASE_BRANCH" >&2
4295"#,
4296            path_helpers = remote_path_helpers_shell(),
4297        );
4298        self.run_script(
4299            &create_worktree_script,
4300            &[
4301                checkout_path.to_owned(),
4302                base_branch.to_owned(),
4303                branch_name.to_owned(),
4304                worktree_path.to_owned(),
4305            ],
4306        )?;
4307
4308        Ok(())
4309    }
4310
4311    fn create_review_worktree(
4312        &self,
4313        checkout_path: &str,
4314        pull_request_number: u64,
4315        branch_name: &str,
4316        worktree_path: &str,
4317        target_head_oid: Option<&str>,
4318    ) -> Result<(), TrackError> {
4319        let create_review_worktree_script = build_create_review_worktree_script();
4320        self.run_script(
4321            &create_review_worktree_script,
4322            &[
4323                checkout_path.to_owned(),
4324                pull_request_number.to_string(),
4325                branch_name.to_owned(),
4326                worktree_path.to_owned(),
4327                target_head_oid.unwrap_or_default().to_owned(),
4328            ],
4329        )?;
4330
4331        Ok(())
4332    }
4333
4334    fn ensure_follow_up_worktree(
4335        &self,
4336        checkout_path: &str,
4337        branch_name: &str,
4338        worktree_path: &str,
4339    ) -> Result<(), TrackError> {
4340        let ensure_follow_up_worktree_script = format!(
4341            r#"
4342set -eu
4343{path_helpers}
4344CHECKOUT_PATH="$(expand_remote_path "$1")"
4345BRANCH_NAME="$2"
4346WORKTREE_PATH="$(expand_remote_path "$3")"
4347
4348mkdir -p "$(dirname "$WORKTREE_PATH")"
4349git -C "$CHECKOUT_PATH" fetch origin --prune >&2 || true
4350git -C "$CHECKOUT_PATH" fetch upstream --prune >&2 || true
4351
4352if [ -e "$WORKTREE_PATH/.git" ]; then
4353  if ! git -C "$WORKTREE_PATH" rev-parse --show-toplevel >/dev/null 2>&1; then
4354    echo "Existing follow-up worktree path $WORKTREE_PATH is not a valid Git worktree." >&2
4355    exit 1
4356  fi
4357
4358  git -C "$WORKTREE_PATH" checkout "$BRANCH_NAME" >&2
4359  exit 0
4360fi
4361
4362if [ -e "$WORKTREE_PATH" ]; then
4363  echo "Follow-up worktree path $WORKTREE_PATH already exists but is not a Git worktree." >&2
4364  exit 1
4365fi
4366
4367git -C "$CHECKOUT_PATH" worktree prune >&2
4368
4369if git -C "$CHECKOUT_PATH" show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then
4370  git -C "$CHECKOUT_PATH" worktree add "$WORKTREE_PATH" "$BRANCH_NAME" >&2
4371  exit 0
4372fi
4373
4374if git -C "$CHECKOUT_PATH" show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then
4375  git -C "$CHECKOUT_PATH" worktree add -B "$BRANCH_NAME" "$WORKTREE_PATH" "origin/$BRANCH_NAME" >&2
4376  exit 0
4377fi
4378
4379echo "Could not restore the follow-up worktree for branch $BRANCH_NAME." >&2
4380exit 1
4381"#,
4382            path_helpers = remote_path_helpers_shell(),
4383        );
4384        self.run_script(
4385            &ensure_follow_up_worktree_script,
4386            &[
4387                checkout_path.to_owned(),
4388                branch_name.to_owned(),
4389                worktree_path.to_owned(),
4390            ],
4391        )?;
4392
4393        Ok(())
4394    }
4395
4396    fn launch_remote_dispatch(
4397        &self,
4398        remote_run_directory: &str,
4399        worktree_path: &str,
4400        preferred_tool: RemoteAgentPreferredTool,
4401    ) -> Result<(), TrackError> {
4402        let launcher_contents = build_remote_agent_launcher(preferred_tool, &self.shell_prelude);
4403        self.upload_remote_file(
4404            &format!("{remote_run_directory}/launch.sh"),
4405            &launcher_contents,
4406        )?;
4407
4408        let launch_script = format!(
4409            r#"
4410set -eu
4411{path_helpers}
4412RUN_DIR="$(expand_remote_path "$1")"
4413WORKTREE_PATH="$(expand_remote_path "$2")"
4414
4415mkdir -p "$RUN_DIR"
4416LAUNCHER_PATH="$RUN_DIR/launch.sh"
4417chmod +x "$LAUNCHER_PATH"
4418nohup bash "$LAUNCHER_PATH" "$RUN_DIR" "$WORKTREE_PATH" >/dev/null 2>&1 </dev/null &
4419"#,
4420            path_helpers = remote_path_helpers_shell(),
4421        );
4422        self.run_script(
4423            &launch_script,
4424            &[remote_run_directory.to_owned(), worktree_path.to_owned()],
4425        )?;
4426
4427        Ok(())
4428    }
4429
4430    fn cancel_remote_dispatch(&self, remote_run_directory: &str) -> Result<(), TrackError> {
4431        let cancel_script = format!(
4432            r#"
4433set -eu
4434{path_helpers}
4435RUN_DIR="$(expand_remote_path "$1")"
4436LAUNCHER_PID_FILE="$RUN_DIR/{launcher_pid_file}"
4437CODEX_PID_FILE="$RUN_DIR/{codex_pid_file}"
4438STATUS_FILE="$RUN_DIR/{status_file}"
4439FINISHED_AT_FILE="$RUN_DIR/{finished_at_file}"
4440
4441kill_if_running() {{
4442  PID="$1"
4443  if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
4444    kill "$PID" 2>/dev/null || true
4445  fi
4446}}
4447
4448if [ -f "$LAUNCHER_PID_FILE" ]; then
4449  LAUNCHER_PID="$(tr -d '[:space:]' < "$LAUNCHER_PID_FILE")"
4450  kill_if_running "$LAUNCHER_PID"
4451fi
4452
4453if [ -f "$CODEX_PID_FILE" ]; then
4454  CODEX_PID="$(tr -d '[:space:]' < "$CODEX_PID_FILE")"
4455  kill_if_running "$CODEX_PID"
4456fi
4457
4458mkdir -p "$RUN_DIR"
4459printf 'canceled\n' > "$STATUS_FILE"
4460date -u +%Y-%m-%dT%H:%M:%SZ > "$FINISHED_AT_FILE"
4461"#,
4462            path_helpers = remote_path_helpers_shell(),
4463            launcher_pid_file = REMOTE_LAUNCHER_PID_FILE_NAME,
4464            codex_pid_file = REMOTE_CODEX_PID_FILE_NAME,
4465            status_file = REMOTE_STATUS_FILE_NAME,
4466            finished_at_file = REMOTE_FINISHED_AT_FILE_NAME,
4467        );
4468        self.run_script(&cancel_script, &[remote_run_directory.to_owned()])?;
4469        Ok(())
4470    }
4471
4472    fn cleanup_task_artifacts(
4473        &self,
4474        checkout_path: &str,
4475        worktree_paths: &[String],
4476        run_directories: &[String],
4477        cleanup_mode: RemoteTaskCleanupMode,
4478    ) -> Result<RemoteArtifactCleanupCounts, TrackError> {
4479        let cleanup_remote_dispatch_directories = cleanup_mode == RemoteTaskCleanupMode::DeleteTask;
4480        let cleanup_script = format!(
4481            r#"
4482set -eu
4483{path_helpers}
4484CHECKOUT_PATH="$(expand_remote_path "$1")"
4485shift
4486
4487WORKTREE_PATHS=()
4488while [ "$#" -gt 0 ]; do
4489  if [ "$1" = "--" ]; then
4490    shift
4491    break
4492  fi
4493
4494  WORKTREE_PATHS+=("$1")
4495  shift
4496done
4497
4498RUN_DIRECTORIES=("$@")
4499WORKTREES_REMOVED=0
4500RUN_DIRECTORIES_REMOVED=0
4501
4502kill_if_running() {{
4503  PID="$1"
4504  if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
4505    kill "$PID" 2>/dev/null || true
4506  fi
4507}}
4508
4509worktree_is_registered() {{
4510  TARGET_WORKTREE="$1"
4511  git -C "$CHECKOUT_PATH" worktree list --porcelain | grep -F "worktree $TARGET_WORKTREE" >/dev/null 2>&1
4512}}
4513
4514for RAW_RUN_DIR in "${{RUN_DIRECTORIES[@]}}"; do
4515  RUN_DIR="$(expand_remote_path "$RAW_RUN_DIR")"
4516  LAUNCHER_PID_FILE="$RUN_DIR/{launcher_pid_file}"
4517  CODEX_PID_FILE="$RUN_DIR/{codex_pid_file}"
4518  STATUS_FILE="$RUN_DIR/{status_file}"
4519  FINISHED_AT_FILE="$RUN_DIR/{finished_at_file}"
4520  CURRENT_STATUS="$(tr -d '[:space:]' < "$STATUS_FILE" 2>/dev/null || true)"
4521
4522  if [ -f "$LAUNCHER_PID_FILE" ]; then
4523    LAUNCHER_PID="$(tr -d '[:space:]' < "$LAUNCHER_PID_FILE")"
4524    kill_if_running "$LAUNCHER_PID"
4525  fi
4526
4527  if [ -f "$CODEX_PID_FILE" ]; then
4528    CODEX_PID="$(tr -d '[:space:]' < "$CODEX_PID_FILE")"
4529    kill_if_running "$CODEX_PID"
4530  fi
4531
4532  if [ -d "$RUN_DIR" ] && {{ [ "$CURRENT_STATUS" = "preparing" ] || [ "$CURRENT_STATUS" = "running" ]; }}; then
4533    printf 'canceled\n' > "$STATUS_FILE"
4534    date -u +%Y-%m-%dT%H:%M:%SZ > "$FINISHED_AT_FILE"
4535  fi
4536done
4537
4538for RAW_WORKTREE_PATH in "${{WORKTREE_PATHS[@]}}"; do
4539  WORKTREE_PATH="$(expand_remote_path "$RAW_WORKTREE_PATH")"
4540  HAD_WORKTREE_PATH="false"
4541  if [ -e "$WORKTREE_PATH" ]; then
4542    HAD_WORKTREE_PATH="true"
4543  fi
4544
4545  if [ -d "$CHECKOUT_PATH/.git" ] && worktree_is_registered "$WORKTREE_PATH"; then
4546    git -C "$CHECKOUT_PATH" worktree remove --force "$WORKTREE_PATH" >&2 || true
4547  fi
4548
4549  if [ -e "$WORKTREE_PATH" ]; then
4550    rm -rf "$WORKTREE_PATH"
4551  fi
4552
4553  if [ "$HAD_WORKTREE_PATH" = "true" ] && [ ! -e "$WORKTREE_PATH" ]; then
4554    WORKTREES_REMOVED=$((WORKTREES_REMOVED + 1))
4555  fi
4556done
4557
4558if [ -d "$CHECKOUT_PATH/.git" ]; then
4559  git -C "$CHECKOUT_PATH" worktree prune >&2 || true
4560fi
4561
4562if [ "{cleanup_remote_dispatch_directories}" = "true" ]; then
4563  for RAW_RUN_DIR in "${{RUN_DIRECTORIES[@]}}"; do
4564    RUN_DIR="$(expand_remote_path "$RAW_RUN_DIR")"
4565    HAD_RUN_DIRECTORY="false"
4566    if [ -e "$RUN_DIR" ]; then
4567      HAD_RUN_DIRECTORY="true"
4568    fi
4569    if [ -e "$RUN_DIR" ]; then
4570      rm -rf "$RUN_DIR"
4571    fi
4572    if [ "$HAD_RUN_DIRECTORY" = "true" ] && [ ! -e "$RUN_DIR" ]; then
4573      RUN_DIRECTORIES_REMOVED=$((RUN_DIRECTORIES_REMOVED + 1))
4574    fi
4575  done
4576fi
4577
4578printf '{{"worktreesRemoved":%s,"runDirectoriesRemoved":%s}}\n' \
4579  "$WORKTREES_REMOVED" \
4580  "$RUN_DIRECTORIES_REMOVED"
4581"#,
4582            path_helpers = remote_path_helpers_shell(),
4583            cleanup_remote_dispatch_directories = if cleanup_remote_dispatch_directories {
4584                "true"
4585            } else {
4586                "false"
4587            },
4588            launcher_pid_file = REMOTE_LAUNCHER_PID_FILE_NAME,
4589            codex_pid_file = REMOTE_CODEX_PID_FILE_NAME,
4590            status_file = REMOTE_STATUS_FILE_NAME,
4591            finished_at_file = REMOTE_FINISHED_AT_FILE_NAME,
4592        );
4593
4594        let mut arguments = vec![checkout_path.to_owned()];
4595        arguments.extend(worktree_paths.iter().cloned());
4596        arguments.push("--".to_owned());
4597        arguments.extend(run_directories.iter().cloned());
4598        let report = self.run_script(&cleanup_script, &arguments)?;
4599        parse_remote_cleanup_counts(&report)
4600    }
4601
4602    fn cleanup_review_artifacts(
4603        &self,
4604        checkout_path: &str,
4605        branch_names: &[String],
4606        worktree_paths: &[String],
4607        run_directories: &[String],
4608    ) -> Result<(), TrackError> {
4609        let cleanup_script = format!(
4610            r#"
4611set -eu
4612{path_helpers}
4613CHECKOUT_PATH="$(expand_remote_path "$1")"
4614shift
4615
4616BRANCH_NAMES=()
4617while [ "$#" -gt 0 ]; do
4618  if [ "$1" = "--worktrees" ]; then
4619    shift
4620    break
4621  fi
4622
4623  BRANCH_NAMES+=("$1")
4624  shift
4625done
4626
4627WORKTREE_PATHS=()
4628while [ "$#" -gt 0 ]; do
4629  if [ "$1" = "--runs" ]; then
4630    shift
4631    break
4632  fi
4633
4634  WORKTREE_PATHS+=("$1")
4635  shift
4636done
4637
4638RUN_DIRECTORIES=("$@")
4639
4640kill_if_running() {{
4641  PID="$1"
4642  if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
4643    kill "$PID" 2>/dev/null || true
4644  fi
4645}}
4646
4647worktree_is_registered() {{
4648  TARGET_WORKTREE="$1"
4649  git -C "$CHECKOUT_PATH" worktree list --porcelain | grep -F "worktree $TARGET_WORKTREE" >/dev/null 2>&1
4650}}
4651
4652for RAW_RUN_DIR in "${{RUN_DIRECTORIES[@]}}"; do
4653  RUN_DIR="$(expand_remote_path "$RAW_RUN_DIR")"
4654  LAUNCHER_PID_FILE="$RUN_DIR/{launcher_pid_file}"
4655  CODEX_PID_FILE="$RUN_DIR/{codex_pid_file}"
4656
4657  if [ -f "$LAUNCHER_PID_FILE" ]; then
4658    LAUNCHER_PID="$(tr -d '[:space:]' < "$LAUNCHER_PID_FILE")"
4659    kill_if_running "$LAUNCHER_PID"
4660  fi
4661
4662  if [ -f "$CODEX_PID_FILE" ]; then
4663    CODEX_PID="$(tr -d '[:space:]' < "$CODEX_PID_FILE")"
4664    kill_if_running "$CODEX_PID"
4665  fi
4666
4667  if [ -e "$RUN_DIR" ]; then
4668    rm -rf "$RUN_DIR"
4669  fi
4670done
4671
4672for RAW_WORKTREE_PATH in "${{WORKTREE_PATHS[@]}}"; do
4673  WORKTREE_PATH="$(expand_remote_path "$RAW_WORKTREE_PATH")"
4674
4675  if [ -d "$CHECKOUT_PATH/.git" ] && worktree_is_registered "$WORKTREE_PATH"; then
4676    git -C "$CHECKOUT_PATH" worktree remove --force "$WORKTREE_PATH" >&2 || true
4677  fi
4678
4679  if [ -e "$WORKTREE_PATH" ]; then
4680    rm -rf "$WORKTREE_PATH"
4681  fi
4682done
4683
4684for BRANCH_NAME in "${{BRANCH_NAMES[@]}}"; do
4685  if [ -d "$CHECKOUT_PATH/.git" ]; then
4686    git -C "$CHECKOUT_PATH" branch -D "$BRANCH_NAME" >&2 || true
4687  fi
4688done
4689
4690if [ -d "$CHECKOUT_PATH/.git" ]; then
4691  git -C "$CHECKOUT_PATH" worktree prune >&2 || true
4692fi
4693"#,
4694            path_helpers = remote_path_helpers_shell(),
4695            launcher_pid_file = REMOTE_LAUNCHER_PID_FILE_NAME,
4696            codex_pid_file = REMOTE_CODEX_PID_FILE_NAME,
4697        );
4698
4699        let mut arguments = vec![checkout_path.to_owned()];
4700        arguments.extend(branch_names.iter().cloned());
4701        arguments.push("--worktrees".to_owned());
4702        arguments.extend(worktree_paths.iter().cloned());
4703        arguments.push("--runs".to_owned());
4704        arguments.extend(run_directories.iter().cloned());
4705        self.run_script(&cleanup_script, &arguments)?;
4706
4707        Ok(())
4708    }
4709
4710    fn cleanup_orphaned_remote_artifacts(
4711        &self,
4712        workspace_root: &str,
4713        kept_worktree_paths: &[String],
4714        kept_run_directories: &[String],
4715    ) -> Result<RemoteArtifactCleanupCounts, TrackError> {
4716        // The remote workspace layout is currently automation-owned:
4717        // `<workspace>/<name>/<name>` for the checkout plus sibling
4718        // task/review worktree and run directories. That lets one broad sweep
4719        // remove forgotten `dispatch-*` artifacts without needing a second
4720        // local registry of every worktree ever created.
4721        // TODO: If the checkout layout ever becomes user-configurable, replace
4722        // this directory derivation with a registry-backed lookup.
4723        let cleanup_script = format!(
4724            r#"
4725set -eu
4726{path_helpers}
4727WORKSPACE_ROOT="$(expand_remote_path "$1")"
4728shift
4729
4730KEEP_WORKTREE_PATHS=()
4731while [ "$#" -gt 0 ]; do
4732  if [ "$1" = "--" ]; then
4733    shift
4734    break
4735  fi
4736
4737  KEEP_WORKTREE_PATHS+=("$(expand_remote_path "$1")")
4738  shift
4739done
4740
4741KEEP_RUN_DIRECTORIES=()
4742for RAW_RUN_DIR in "$@"; do
4743  KEEP_RUN_DIRECTORIES+=("$(expand_remote_path "$RAW_RUN_DIR")")
4744done
4745
4746WORKTREES_REMOVED=0
4747RUN_DIRECTORIES_REMOVED=0
4748
4749path_is_kept() {{
4750  TARGET_PATH="$1"
4751  shift
4752
4753  for KEPT_PATH in "$@"; do
4754    if [ "$KEPT_PATH" = "$TARGET_PATH" ]; then
4755      return 0
4756    fi
4757  done
4758
4759  return 1
4760}}
4761
4762kill_if_running() {{
4763  PID="$1"
4764  if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then
4765    kill "$PID" 2>/dev/null || true
4766  fi
4767}}
4768
4769remove_run_directory() {{
4770  RUN_DIR="$1"
4771  LAUNCHER_PID_FILE="$RUN_DIR/{launcher_pid_file}"
4772  CODEX_PID_FILE="$RUN_DIR/{codex_pid_file}"
4773
4774  if [ -f "$LAUNCHER_PID_FILE" ]; then
4775    LAUNCHER_PID="$(tr -d '[:space:]' < "$LAUNCHER_PID_FILE")"
4776    kill_if_running "$LAUNCHER_PID"
4777  fi
4778
4779  if [ -f "$CODEX_PID_FILE" ]; then
4780    CODEX_PID="$(tr -d '[:space:]' < "$CODEX_PID_FILE")"
4781    kill_if_running "$CODEX_PID"
4782  fi
4783
4784  if [ -e "$RUN_DIR" ]; then
4785    rm -rf "$RUN_DIR"
4786  fi
4787
4788  if [ ! -e "$RUN_DIR" ]; then
4789    RUN_DIRECTORIES_REMOVED=$((RUN_DIRECTORIES_REMOVED + 1))
4790  fi
4791}}
4792
4793remove_worktree_path() {{
4794  WORKTREE_PATH="$1"
4795  PROJECT_DIRECTORY="$(dirname "$(dirname "$WORKTREE_PATH")")"
4796  PROJECT_NAME="$(basename "$PROJECT_DIRECTORY")"
4797  CHECKOUT_PATH="$PROJECT_DIRECTORY/$PROJECT_NAME"
4798
4799  if [ -d "$CHECKOUT_PATH/.git" ]; then
4800    git -C "$CHECKOUT_PATH" worktree remove --force "$WORKTREE_PATH" >&2 || true
4801    git -C "$CHECKOUT_PATH" worktree prune >&2 || true
4802  fi
4803
4804  if [ -e "$WORKTREE_PATH" ]; then
4805    rm -rf "$WORKTREE_PATH"
4806  fi
4807
4808  if [ ! -e "$WORKTREE_PATH" ]; then
4809    WORKTREES_REMOVED=$((WORKTREES_REMOVED + 1))
4810  fi
4811}}
4812
4813for PROJECT_DIRECTORY in "$WORKSPACE_ROOT"/*; do
4814  [ -d "$PROJECT_DIRECTORY" ] || continue
4815
4816  for RUN_DIR in "$PROJECT_DIRECTORY"/dispatches/dispatch-*; do
4817    [ -e "$RUN_DIR" ] || continue
4818    if path_is_kept "$RUN_DIR" "${{KEEP_RUN_DIRECTORIES[@]}}"; then
4819      continue
4820    fi
4821
4822    remove_run_directory "$RUN_DIR"
4823  done
4824
4825  for WORKTREE_PATH in "$PROJECT_DIRECTORY"/worktrees/dispatch-*; do
4826    [ -e "$WORKTREE_PATH" ] || continue
4827    if path_is_kept "$WORKTREE_PATH" "${{KEEP_WORKTREE_PATHS[@]}}"; then
4828      continue
4829    fi
4830
4831    remove_worktree_path "$WORKTREE_PATH"
4832  done
4833
4834  for RUN_DIR in "$PROJECT_DIRECTORY"/{review_run_directory}/dispatch-*; do
4835    [ -e "$RUN_DIR" ] || continue
4836    if path_is_kept "$RUN_DIR" "${{KEEP_RUN_DIRECTORIES[@]}}"; then
4837      continue
4838    fi
4839
4840    remove_run_directory "$RUN_DIR"
4841  done
4842
4843  for WORKTREE_PATH in "$PROJECT_DIRECTORY"/{review_worktree_directory}/dispatch-*; do
4844    [ -e "$WORKTREE_PATH" ] || continue
4845    if path_is_kept "$WORKTREE_PATH" "${{KEEP_WORKTREE_PATHS[@]}}"; then
4846      continue
4847    fi
4848
4849    remove_worktree_path "$WORKTREE_PATH"
4850  done
4851done
4852
4853printf '{{"worktreesRemoved":%s,"runDirectoriesRemoved":%s}}\n' \
4854  "$WORKTREES_REMOVED" \
4855  "$RUN_DIRECTORIES_REMOVED"
4856"#,
4857            path_helpers = remote_path_helpers_shell(),
4858            review_run_directory = REVIEW_RUN_DIRECTORY_NAME,
4859            review_worktree_directory = REVIEW_WORKTREE_DIRECTORY_NAME,
4860            launcher_pid_file = REMOTE_LAUNCHER_PID_FILE_NAME,
4861            codex_pid_file = REMOTE_CODEX_PID_FILE_NAME,
4862        );
4863
4864        let mut arguments = vec![workspace_root.to_owned()];
4865        arguments.extend(kept_worktree_paths.iter().cloned());
4866        arguments.push("--".to_owned());
4867        arguments.extend(kept_run_directories.iter().cloned());
4868        let report = self.run_script(&cleanup_script, &arguments)?;
4869        parse_remote_cleanup_counts(&report)
4870    }
4871
4872    fn cleanup_review_workspace_caches(&self, checkout_paths: &[String]) -> Result<(), TrackError> {
4873        if checkout_paths.is_empty() {
4874            return Ok(());
4875        }
4876
4877        let cleanup_script = format!(
4878            r#"
4879set -eu
4880{path_helpers}
4881
4882for RAW_CHECKOUT_PATH in "$@"; do
4883  CHECKOUT_PATH="$(expand_remote_path "$RAW_CHECKOUT_PATH")"
4884  WORKSPACE_PATH="$(dirname "$CHECKOUT_PATH")"
4885
4886  if [ -d "$CHECKOUT_PATH/.git" ]; then
4887    git -C "$CHECKOUT_PATH" worktree prune >&2 || true
4888  fi
4889
4890  if [ -e "$CHECKOUT_PATH" ]; then
4891    rm -rf "$CHECKOUT_PATH"
4892  fi
4893
4894  if [ -d "$WORKSPACE_PATH" ]; then
4895    rmdir "$WORKSPACE_PATH" 2>/dev/null || true
4896  fi
4897done
4898"#,
4899            path_helpers = remote_path_helpers_shell(),
4900        );
4901        self.run_script(&cleanup_script, checkout_paths)?;
4902
4903        Ok(())
4904    }
4905
4906    fn reset_workspace(
4907        &self,
4908        workspace_root: &str,
4909        projects_registry_path: &str,
4910    ) -> Result<RemoteResetSummary, TrackError> {
4911        let reset_script = format!(
4912            r#"
4913set -eu
4914{path_helpers}
4915WORKSPACE_ROOT="$(expand_remote_path "$1")"
4916REGISTRY_PATH="$(expand_remote_path "$2")"
4917WORKSPACE_ENTRIES_REMOVED=0
4918REGISTRY_REMOVED=false
4919
4920if [ -z "$WORKSPACE_ROOT" ] || [ "$WORKSPACE_ROOT" = "/" ] || [ "$WORKSPACE_ROOT" = "$HOME" ]; then
4921  echo "Refusing to reset an unsafe remote workspace root at $WORKSPACE_ROOT." >&2
4922  exit 1
4923fi
4924
4925mkdir -p "$WORKSPACE_ROOT"
4926
4927for ENTRY in "$WORKSPACE_ROOT"/* "$WORKSPACE_ROOT"/.[!.]* "$WORKSPACE_ROOT"/..?*; do
4928  [ -e "$ENTRY" ] || continue
4929  rm -rf "$ENTRY"
4930  if [ ! -e "$ENTRY" ]; then
4931    WORKSPACE_ENTRIES_REMOVED=$((WORKSPACE_ENTRIES_REMOVED + 1))
4932  fi
4933done
4934
4935if [ -e "$REGISTRY_PATH" ]; then
4936  rm -f "$REGISTRY_PATH"
4937  if [ ! -e "$REGISTRY_PATH" ]; then
4938    REGISTRY_REMOVED=true
4939  fi
4940fi
4941
4942printf '{{"workspaceEntriesRemoved":%s,"registryRemoved":%s}}\n' \
4943  "$WORKSPACE_ENTRIES_REMOVED" \
4944  "$REGISTRY_REMOVED"
4945"#,
4946            path_helpers = remote_path_helpers_shell(),
4947        );
4948        let report = self.run_script(
4949            &reset_script,
4950            &[workspace_root.to_owned(), projects_registry_path.to_owned()],
4951        )?;
4952        parse_remote_reset_summary(&report)
4953    }
4954
4955    fn read_remote_file(&self, remote_path: &str) -> Result<Option<String>, TrackError> {
4956        let read_remote_file_script = format!(
4957            r#"
4958set -eu
4959{path_helpers}
4960REMOTE_PATH="$(expand_remote_path "$1")"
4961if [ -f "$REMOTE_PATH" ]; then
4962  cat "$REMOTE_PATH"
4963else
4964  exit 3
4965fi
4966"#,
4967            path_helpers = remote_path_helpers_shell(),
4968        );
4969        match self.run_script_with_exit_code(&read_remote_file_script, &[remote_path.to_owned()])? {
4970            ScriptOutput::Success(stdout) => Ok(Some(stdout)),
4971            ScriptOutput::ExitCode(3) => Ok(None),
4972            ScriptOutput::ExitCode(code) => Err(TrackError::new(
4973                ErrorCode::RemoteDispatchFailed,
4974                format!(
4975                    "Could not read the remote file at {remote_path}: remote command exited with status code {code}."
4976                ),
4977            )),
4978            ScriptOutput::Failure(stderr) => Err(TrackError::new(
4979                ErrorCode::RemoteDispatchFailed,
4980                format!("Could not read the remote file at {remote_path}: {stderr}"),
4981            )),
4982        }
4983    }
4984
4985    fn upload_remote_file(&self, remote_path: &str, contents: &str) -> Result<(), TrackError> {
4986        let upload_remote_file_script = format!(
4987            r#"
4988set -eu
4989{path_helpers}
4990REMOTE_PATH="$(expand_remote_path "$1")"
4991mkdir -p "$(dirname "$REMOTE_PATH")"
4992"#,
4993            path_helpers = remote_path_helpers_shell(),
4994        );
4995        self.run_script(&upload_remote_file_script, &[remote_path.to_owned()])?;
4996
4997        let local_temp_file = env::temp_dir().join(format!(
4998            "track-remote-upload-{}",
4999            now_utc().unix_timestamp_nanos()
5000        ));
5001        fs::write(&local_temp_file, contents).map_err(|error| {
5002            TrackError::new(
5003                ErrorCode::DispatchWriteFailed,
5004                format!(
5005                    "Could not write a temporary upload file at {}: {error}",
5006                    path_to_string(&local_temp_file)
5007                ),
5008            )
5009        })?;
5010
5011        let output = self
5012            .base_scp_command()
5013            .arg(&local_temp_file)
5014            .arg(format!("{}@{}:{remote_path}", self.user, self.host))
5015            .output()
5016            .map_err(|error| {
5017                TrackError::new(
5018                    ErrorCode::RemoteDispatchFailed,
5019                    format!("Could not start `scp` for remote dispatch: {error}"),
5020                )
5021            })?;
5022        let _ = fs::remove_file(&local_temp_file);
5023
5024        if !output.status.success() {
5025            return Err(TrackError::new(
5026                ErrorCode::RemoteDispatchFailed,
5027                format!(
5028                    "Could not upload the remote file at {remote_path}: {}",
5029                    String::from_utf8_lossy(&output.stderr).trim()
5030                ),
5031            ));
5032        }
5033
5034        Ok(())
5035    }
5036
5037    fn read_dispatch_snapshots(
5038        &self,
5039        run_directories: &[String],
5040    ) -> Result<Vec<RemoteDispatchSnapshot>, TrackError> {
5041        if run_directories.is_empty() {
5042            return Ok(Vec::new());
5043        }
5044
5045        let snapshot_script = format!(
5046            r#"
5047set -eu
5048{path_helpers}
5049
5050emit_file() {{
5051  LABEL="$1"
5052  FILE_PATH="$(expand_remote_path "$2")"
5053
5054  printf '%s\t' "$LABEL"
5055  if [ -f "$FILE_PATH" ]; then
5056    printf 'present\t'
5057    od -An -tx1 -v "$FILE_PATH" | tr -d ' \n'
5058  else
5059    printf 'missing\t'
5060  fi
5061  printf '\n'
5062}}
5063
5064for RAW_RUN_DIR in "$@"; do
5065  RUN_DIR="$(expand_remote_path "$RAW_RUN_DIR")"
5066  printf 'run\t%s\n' "$RAW_RUN_DIR"
5067  emit_file "status" "$RUN_DIR/{status_file}"
5068  emit_file "result" "$RUN_DIR/{result_file}"
5069  emit_file "stderr" "$RUN_DIR/{stderr_file}"
5070  emit_file "finished_at" "$RUN_DIR/{finished_at_file}"
5071done
5072"#,
5073            path_helpers = remote_path_helpers_shell(),
5074            status_file = REMOTE_STATUS_FILE_NAME,
5075            result_file = REMOTE_RESULT_FILE_NAME,
5076            stderr_file = REMOTE_STDERR_FILE_NAME,
5077            finished_at_file = REMOTE_FINISHED_AT_FILE_NAME,
5078        );
5079        let report = self.run_script(&snapshot_script, run_directories)?;
5080
5081        parse_dispatch_snapshot_report(&report)
5082    }
5083
5084    fn run_script(&self, script: &str, args: &[String]) -> Result<String, TrackError> {
5085        match self.run_script_with_exit_code(script, args)? {
5086            ScriptOutput::Success(stdout) => Ok(stdout),
5087            ScriptOutput::ExitCode(code) => Err(TrackError::new(
5088                ErrorCode::RemoteDispatchFailed,
5089                format!("Remote command exited with unexpected status code {code}."),
5090            )),
5091            ScriptOutput::Failure(stderr) => {
5092                Err(TrackError::new(ErrorCode::RemoteDispatchFailed, stderr))
5093            }
5094        }
5095    }
5096
5097    fn run_script_with_exit_code(
5098        &self,
5099        script: &str,
5100        args: &[String],
5101    ) -> Result<ScriptOutput, TrackError> {
5102        let mut command = self.base_ssh_command();
5103        command.arg(format!("{}@{}", self.user, self.host));
5104        command.arg("bash");
5105        command.arg("-s");
5106        command.arg("--");
5107        command.args(args);
5108        command.stdin(Stdio::piped());
5109        command.stdout(Stdio::piped());
5110        command.stderr(Stdio::piped());
5111
5112        let mut child = command.spawn().map_err(|error| {
5113            TrackError::new(
5114                ErrorCode::RemoteDispatchFailed,
5115                format!("Could not start the remote SSH command: {error}"),
5116            )
5117        })?;
5118
5119        let Some(mut stdin) = child.stdin.take() else {
5120            return Err(TrackError::new(
5121                ErrorCode::RemoteDispatchFailed,
5122                "Could not open stdin for the remote SSH command.",
5123            ));
5124        };
5125        let rendered_script = render_remote_script_with_shell_prelude(script, &self.shell_prelude);
5126        stdin
5127            .write_all(rendered_script.as_bytes())
5128            .map_err(|error| {
5129                TrackError::new(
5130                    ErrorCode::RemoteDispatchFailed,
5131                    format!("Could not write the remote shell script to SSH stdin: {error}"),
5132                )
5133            })?;
5134        drop(stdin);
5135
5136        let output = child.wait_with_output().map_err(|error| {
5137            TrackError::new(
5138                ErrorCode::RemoteDispatchFailed,
5139                format!("Could not wait for the remote SSH command to finish: {error}"),
5140            )
5141        })?;
5142
5143        if output.status.success() {
5144            return Ok(ScriptOutput::Success(
5145                String::from_utf8_lossy(&output.stdout).trim().to_owned(),
5146            ));
5147        }
5148
5149        let exit_code = output.status.code();
5150        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
5151        if let Some(exit_code) = exit_code {
5152            if stderr.is_empty() {
5153                return Ok(ScriptOutput::ExitCode(exit_code));
5154            }
5155
5156            if exit_code == 3 {
5157                return Ok(ScriptOutput::ExitCode(exit_code));
5158            }
5159        }
5160
5161        Ok(ScriptOutput::Failure(if stderr.is_empty() {
5162            "Remote command failed without stderr output.".to_owned()
5163        } else {
5164            stderr
5165        }))
5166    }
5167
5168    fn base_ssh_command(&self) -> Command {
5169        let mut command = Command::new("ssh");
5170        command.arg("-i");
5171        command.arg(&self.key_path);
5172        command.arg("-p");
5173        command.arg(self.port.to_string());
5174        command.args([
5175            "-o",
5176            "BatchMode=yes",
5177            "-o",
5178            "IdentitiesOnly=yes",
5179            "-o",
5180            "ConnectTimeout=10",
5181            "-o",
5182            "StrictHostKeyChecking=accept-new",
5183            "-o",
5184        ]);
5185        command.arg(format!(
5186            "UserKnownHostsFile={}",
5187            path_to_string(&self.known_hosts_path)
5188        ));
5189        command
5190    }
5191
5192    fn base_scp_command(&self) -> Command {
5193        let mut command = Command::new("scp");
5194        command.arg("-i");
5195        command.arg(&self.key_path);
5196        command.arg("-P");
5197        command.arg(self.port.to_string());
5198        command.args([
5199            "-o",
5200            "BatchMode=yes",
5201            "-o",
5202            "IdentitiesOnly=yes",
5203            "-o",
5204            "ConnectTimeout=10",
5205            "-o",
5206            "StrictHostKeyChecking=accept-new",
5207            "-o",
5208        ]);
5209        command.arg(format!(
5210            "UserKnownHostsFile={}",
5211            path_to_string(&self.known_hosts_path)
5212        ));
5213        command
5214    }
5215}
5216
5217enum ScriptOutput {
5218    Success(String),
5219    ExitCode(i32),
5220    Failure(String),
5221}
5222
5223fn parse_dispatch_snapshot_report(report: &str) -> Result<Vec<RemoteDispatchSnapshot>, TrackError> {
5224    let mut snapshots = Vec::new();
5225    let mut current_snapshot: Option<RemoteDispatchSnapshot> = None;
5226
5227    for line in report.lines().filter(|line| !line.trim().is_empty()) {
5228        let columns = line.splitn(3, '\t').collect::<Vec<_>>();
5229        match columns.first().copied() {
5230            Some("run") => {
5231                let _run_identifier = columns.get(1).ok_or_else(|| {
5232                    TrackError::new(
5233                        ErrorCode::RemoteDispatchFailed,
5234                        "Remote dispatch refresh report is missing a run directory.",
5235                    )
5236                })?;
5237                if let Some(snapshot) = current_snapshot.take() {
5238                    snapshots.push(snapshot);
5239                }
5240                current_snapshot = Some(RemoteDispatchSnapshot::default());
5241            }
5242            Some("status") | Some("result") | Some("stderr") | Some("finished_at") => {
5243                let field_name = columns
5244                    .first()
5245                    .expect("field-tagged dispatch line should have a tag");
5246                let presence = columns.get(1).ok_or_else(|| {
5247                    TrackError::new(
5248                        ErrorCode::RemoteDispatchFailed,
5249                        "Remote dispatch refresh report is missing a field state.",
5250                    )
5251                })?;
5252                let value = match *presence {
5253                    "missing" => None,
5254                    "present" => Some(decode_hex_string(columns.get(2).copied().unwrap_or(""))?),
5255                    _ => {
5256                        return Err(TrackError::new(
5257                            ErrorCode::RemoteDispatchFailed,
5258                            "Remote dispatch refresh report has an unknown field state.",
5259                        ));
5260                    }
5261                };
5262                let Some(snapshot) = current_snapshot.as_mut() else {
5263                    return Err(TrackError::new(
5264                        ErrorCode::RemoteDispatchFailed,
5265                        "Remote dispatch refresh report emitted a field before the run header.",
5266                    ));
5267                };
5268                match *field_name {
5269                    "status" => snapshot.status = value,
5270                    "result" => snapshot.result = value,
5271                    "stderr" => snapshot.stderr = value,
5272                    "finished_at" => snapshot.finished_at = value,
5273                    _ => {}
5274                }
5275            }
5276            _ => {
5277                return Err(TrackError::new(
5278                    ErrorCode::RemoteDispatchFailed,
5279                    "Remote dispatch refresh report contains an unexpected line.",
5280                ));
5281            }
5282        }
5283    }
5284
5285    if let Some(snapshot) = current_snapshot {
5286        snapshots.push(snapshot);
5287    }
5288
5289    Ok(snapshots)
5290}
5291
5292fn parse_remote_cleanup_counts(report: &str) -> Result<RemoteArtifactCleanupCounts, TrackError> {
5293    let parsed_report = serde_json::from_str::<RemoteArtifactCleanupReport>(report.trim())
5294        .map_err(|error| {
5295            TrackError::new(
5296                ErrorCode::RemoteDispatchFailed,
5297                format!("Could not parse the remote cleanup report: {error}"),
5298            )
5299        })?;
5300
5301    Ok(RemoteArtifactCleanupCounts {
5302        worktrees_removed: parsed_report.worktrees_removed,
5303        run_directories_removed: parsed_report.run_directories_removed,
5304    })
5305}
5306
5307fn parse_remote_reset_summary(report: &str) -> Result<RemoteResetSummary, TrackError> {
5308    let parsed_report =
5309        serde_json::from_str::<RemoteWorkspaceResetReport>(report.trim()).map_err(|error| {
5310            TrackError::new(
5311                ErrorCode::RemoteDispatchFailed,
5312                format!("Could not parse the remote reset report: {error}"),
5313            )
5314        })?;
5315
5316    Ok(RemoteResetSummary {
5317        workspace_entries_removed: parsed_report.workspace_entries_removed,
5318        registry_removed: parsed_report.registry_removed,
5319    })
5320}
5321
5322fn decode_hex_string(hex: &str) -> Result<String, TrackError> {
5323    if hex.len() % 2 != 0 {
5324        return Err(TrackError::new(
5325            ErrorCode::RemoteDispatchFailed,
5326            "Remote dispatch refresh data is not valid hexadecimal.",
5327        ));
5328    }
5329
5330    let mut bytes = Vec::with_capacity(hex.len() / 2);
5331    let mut index = 0;
5332    while index < hex.len() {
5333        let byte = u8::from_str_radix(&hex[index..index + 2], 16).map_err(|error| {
5334            TrackError::new(
5335                ErrorCode::RemoteDispatchFailed,
5336                format!("Remote dispatch refresh data is not valid hexadecimal: {error}"),
5337            )
5338        })?;
5339        bytes.push(byte);
5340        index += 2;
5341    }
5342
5343    String::from_utf8(bytes).map_err(|error| {
5344        TrackError::new(
5345            ErrorCode::RemoteDispatchFailed,
5346            format!("Remote dispatch refresh data is not valid UTF-8: {error}"),
5347        )
5348    })
5349}
5350
5351#[cfg(test)]
5352mod tests {
5353    use std::collections::BTreeMap;
5354    use std::fs;
5355    use std::path::{Path, PathBuf};
5356    use std::sync::{
5357        atomic::{AtomicBool, Ordering},
5358        Arc,
5359    };
5360
5361    use serde_json::json;
5362    use tempfile::TempDir;
5363
5364    use crate::backend_config::RemoteAgentConfigService;
5365    use crate::config::{
5366        ApiConfigFile, LlamaCppConfigFile, RemoteAgentConfigFile, TrackConfigFile,
5367    };
5368    use crate::dispatch_repository::DispatchRepository;
5369    use crate::project_repository::{ProjectMetadata, ProjectRepository};
5370    use crate::review_dispatch_repository::ReviewDispatchRepository;
5371    use crate::review_repository::ReviewRepository;
5372    use crate::task_description::render_task_description;
5373    use crate::task_repository::FileTaskRepository;
5374    use crate::test_support::{set_env_var, track_data_env_lock};
5375    use crate::time_utils::{now_utc, parse_iso_8601_seconds};
5376    use crate::types::{
5377        DispatchStatus, Priority, RemoteAgentPreferredTool, ReviewRecord, ReviewRunRecord, Status,
5378        TaskCreateInput, TaskDispatchRecord, TaskSource, TaskUpdateInput,
5379    };
5380    use time::Duration;
5381
5382    use super::{
5383        build_create_review_worktree_script, build_remote_agent_launcher,
5384        build_remote_dispatch_prompt, build_remote_dispatch_schema, build_remote_review_prompt,
5385        build_remote_review_schema, build_review_follow_up_request, build_review_workspace_key,
5386        describe_remote_reset_blockers, latest_pull_request_for_branch,
5387        parse_dispatch_snapshot_report, parse_github_pull_request_reference,
5388        parse_github_repository_name, refresh_dispatch_record_from_snapshot,
5389        render_remote_script_with_shell_prelude, select_follow_up_base_dispatch,
5390        select_previous_submitted_review_run, GithubPullRequestMetadata, RemoteDispatchService,
5391        RemoteDispatchSnapshot, RemoteReviewService,
5392    };
5393
5394    struct TestContext {
5395        _directory: TempDir,
5396        _env_lock_guard: std::sync::MutexGuard<'static, ()>,
5397        _track_state_dir_guard: crate::test_support::EnvVarGuard,
5398        data_dir: PathBuf,
5399        config_service: RemoteAgentConfigService,
5400        dispatch_repository: DispatchRepository,
5401        project_repository: ProjectRepository,
5402        review_dispatch_repository: ReviewDispatchRepository,
5403        review_repository: ReviewRepository,
5404        task_repository: FileTaskRepository,
5405    }
5406
5407    impl TestContext {
5408        fn new(config: TrackConfigFile) -> Self {
5409            let directory = TempDir::new().expect("tempdir should be created");
5410            let env_lock_guard = track_data_env_lock()
5411                .lock()
5412                .unwrap_or_else(|poisoned| poisoned.into_inner());
5413            let state_root = directory.path().join("state");
5414            let track_state_dir_guard = set_env_var("TRACK_STATE_DIR", &state_root);
5415            let data_dir = state_root.join("issues");
5416            let database_path = state_root.join("track.sqlite");
5417            let config_service =
5418                RemoteAgentConfigService::new(None).expect("config service should resolve");
5419            config_service
5420                .save_remote_agent_config(config.remote_agent.as_ref())
5421                .expect("remote-agent config should save");
5422
5423            Self {
5424                _directory: directory,
5425                _env_lock_guard: env_lock_guard,
5426                _track_state_dir_guard: track_state_dir_guard,
5427                data_dir: data_dir.clone(),
5428                config_service,
5429                dispatch_repository: DispatchRepository::new(Some(database_path.clone()))
5430                    .expect("dispatch repository should resolve"),
5431                project_repository: ProjectRepository::new(Some(database_path.clone()))
5432                    .expect("project repository should resolve"),
5433                review_dispatch_repository: ReviewDispatchRepository::new(Some(
5434                    database_path.clone(),
5435                ))
5436                .expect("review dispatch repository should resolve"),
5437                review_repository: ReviewRepository::new(Some(database_path.clone()))
5438                    .expect("review repository should resolve"),
5439                task_repository: FileTaskRepository::new(Some(database_path))
5440                    .expect("task repository should resolve"),
5441            }
5442        }
5443
5444        fn service(&self) -> RemoteDispatchService<'_> {
5445            RemoteDispatchService {
5446                config_service: &self.config_service,
5447                dispatch_repository: &self.dispatch_repository,
5448                project_repository: &self.project_repository,
5449                task_repository: &self.task_repository,
5450                review_repository: &self.review_repository,
5451                review_dispatch_repository: &self.review_dispatch_repository,
5452            }
5453        }
5454
5455        fn review_service(&self) -> RemoteReviewService<'_> {
5456            RemoteReviewService {
5457                config_service: &self.config_service,
5458                project_repository: &self.project_repository,
5459                review_repository: &self.review_repository,
5460                review_dispatch_repository: &self.review_dispatch_repository,
5461            }
5462        }
5463
5464        fn create_task(&self, project: &str, description: &str) -> crate::types::Task {
5465            self.project_repository
5466                .upsert_project_by_name(
5467                    project,
5468                    ProjectMetadata {
5469                        repo_url: format!("https://github.com/acme/{project}"),
5470                        git_url: format!("git@github.com:acme/{project}.git"),
5471                        base_branch: "main".to_owned(),
5472                        description: None,
5473                    },
5474                    Vec::new(),
5475                )
5476                .expect("project should save");
5477            self.task_repository
5478                .create_task(TaskCreateInput {
5479                    project: project.to_owned(),
5480                    priority: Priority::High,
5481                    description: description.to_owned(),
5482                    source: Some(TaskSource::Web),
5483                })
5484                .expect("task should be created")
5485                .task
5486        }
5487
5488        fn write_project_metadata(&self, project: &str) {
5489            self.project_repository
5490                .upsert_project_by_name(
5491                    project,
5492                    ProjectMetadata {
5493                        repo_url: format!("https://github.com/acme/{project}"),
5494                        git_url: format!("git@github.com:acme/{project}.git"),
5495                        base_branch: "main".to_owned(),
5496                        description: None,
5497                    },
5498                    Vec::new(),
5499                )
5500                .expect("project metadata should save");
5501        }
5502
5503        fn create_running_dispatch(&self, task: &crate::types::Task) -> TaskDispatchRecord {
5504            let mut dispatch = self
5505                .dispatch_repository
5506                .create_dispatch(task, "198.51.100.10", RemoteAgentPreferredTool::Codex)
5507                .expect("dispatch should be created");
5508            dispatch.status = DispatchStatus::Running;
5509            dispatch.branch_name = Some(format!("track/{}", dispatch.dispatch_id));
5510            dispatch.worktree_path = Some(format!(
5511                "~/workspace/{}/worktrees/{}",
5512                task.project, dispatch.dispatch_id
5513            ));
5514            dispatch.summary =
5515                Some("The remote agent is working in the prepared environment.".to_owned());
5516            dispatch.updated_at = now_utc();
5517            self.dispatch_repository
5518                .save_dispatch(&dispatch)
5519                .expect("dispatch should save");
5520            dispatch
5521        }
5522
5523        fn create_review(&self) -> ReviewRecord {
5524            let review = sample_review_record();
5525            self.review_repository
5526                .save_review(&review)
5527                .expect("review should save");
5528            review
5529        }
5530
5531        fn create_review_run(
5532            &self,
5533            review: &ReviewRecord,
5534            status: DispatchStatus,
5535        ) -> ReviewRunRecord {
5536            let timestamp = now_utc();
5537            let dispatch_id = format!("dispatch-{}", timestamp.unix_timestamp_nanos());
5538            let record = ReviewRunRecord {
5539                dispatch_id: dispatch_id.clone(),
5540                review_id: review.id.clone(),
5541                pull_request_url: review.pull_request_url.clone(),
5542                repository_full_name: review.repository_full_name.clone(),
5543                workspace_key: review.workspace_key.clone(),
5544                preferred_tool: review.preferred_tool,
5545                status,
5546                created_at: timestamp,
5547                updated_at: timestamp,
5548                finished_at: None,
5549                remote_host: "198.51.100.10".to_owned(),
5550                branch_name: Some(format!("track-review/{dispatch_id}")),
5551                worktree_path: Some(format!(
5552                    "~/workspace/{}/{}/{}",
5553                    review.workspace_key,
5554                    super::REVIEW_WORKTREE_DIRECTORY_NAME,
5555                    dispatch_id
5556                )),
5557                follow_up_request: None,
5558                target_head_oid: Some("abc123def456".to_owned()),
5559                summary: None,
5560                review_submitted: false,
5561                github_review_id: None,
5562                github_review_url: None,
5563                notes: None,
5564                error_message: None,
5565            };
5566            self.review_dispatch_repository
5567                .save_dispatch(&record)
5568                .expect("review run should save");
5569
5570            record
5571        }
5572    }
5573
5574    fn base_test_config(remote_agent: Option<RemoteAgentConfigFile>) -> TrackConfigFile {
5575        TrackConfigFile {
5576            project_roots: Vec::new(),
5577            project_aliases: BTreeMap::new(),
5578            api: ApiConfigFile { port: 3210 },
5579            llama_cpp: LlamaCppConfigFile {
5580                model_path: Some("/tmp/parser.gguf".to_owned()),
5581                model_hf_repo: None,
5582                model_hf_file: None,
5583            },
5584            remote_agent,
5585        }
5586    }
5587
5588    fn install_dummy_managed_remote_agent_material(data_dir: &Path) {
5589        let remote_agent_dir = data_dir
5590            .parent()
5591            .expect("data dir should have a parent")
5592            .join("remote-agent");
5593        fs::create_dir_all(&remote_agent_dir).expect("remote-agent dir should be created");
5594        fs::write(
5595            remote_agent_dir.join("id_ed25519"),
5596            "not-a-real-private-key",
5597        )
5598        .expect("dummy SSH key should be written");
5599        fs::write(remote_agent_dir.join("known_hosts"), "")
5600            .expect("dummy known_hosts file should be written");
5601    }
5602
5603    fn sample_review_record() -> ReviewRecord {
5604        let created_at = now_utc();
5605        ReviewRecord {
5606            id: "20260326-120000-review-pr-42".to_owned(),
5607            pull_request_url: "https://github.com/acme/project-x/pull/42".to_owned(),
5608            pull_request_number: 42,
5609            pull_request_title: "Fix queue layout".to_owned(),
5610            repository_full_name: "acme/project-x".to_owned(),
5611            repo_url: "https://github.com/acme/project-x".to_owned(),
5612            git_url: "git@github.com:acme/project-x.git".to_owned(),
5613            base_branch: "main".to_owned(),
5614            workspace_key: "project-x".to_owned(),
5615            preferred_tool: RemoteAgentPreferredTool::Codex,
5616            project: Some("project-x".to_owned()),
5617            main_user: "octocat".to_owned(),
5618            default_review_prompt: Some("Focus on regressions and missing tests.".to_owned()),
5619            extra_instructions: Some("Pay special attention to queue rendering.".to_owned()),
5620            created_at,
5621            updated_at: created_at,
5622        }
5623    }
5624
5625    #[test]
5626    fn builds_remote_prompt_with_both_summary_layers() {
5627        let prompt = build_remote_dispatch_prompt(
5628            "project-x",
5629            &ProjectMetadata {
5630                repo_url: "https://github.com/acme/project-x".to_owned(),
5631                git_url: "git@github.com:acme/project-x.git".to_owned(),
5632                base_branch: "main".to_owned(),
5633                description: Some("Main repo".to_owned()),
5634            },
5635            "track/dispatch-1",
5636            "~/workspace/project-x/worktrees/dispatch-1",
5637            &render_task_description(
5638                "Fix a bug in module A",
5639                Some("- Inspect `module_a.rs`"),
5640                Some("proj-x prio high fix a bug in module A"),
5641            ),
5642            Some("https://github.com/acme/project-x/pull/42"),
5643            Some("Address review comments from the latest PR review."),
5644        );
5645
5646        assert!(prompt.contains("## Summary"));
5647        assert!(prompt.contains("## Original note"));
5648        assert!(prompt.contains("## Existing PR"));
5649        assert!(prompt.contains("## Current follow-up request"));
5650        assert!(prompt.contains("fetch that context with `gh`"));
5651        assert!(prompt.contains("only act on that reviewer's feedback"));
5652        assert!(prompt.contains("track/dispatch-1"));
5653        assert!(
5654            prompt.contains("Use conventional commits for both commit messages and the PR title")
5655        );
5656    }
5657
5658    #[test]
5659    fn dispatch_schema_limits_terminal_status_values() {
5660        let schema = build_remote_dispatch_schema();
5661
5662        assert!(schema.contains("\"succeeded\""));
5663        assert!(schema.contains("\"failed\""));
5664        assert!(schema.contains("\"blocked\""));
5665        assert!(schema.contains("\"pullRequestUrl\""));
5666        assert!(schema.contains("\"branchName\""));
5667        assert!(schema.contains("\"notes\""));
5668        assert!(schema.contains("\"required\""));
5669        assert!(!schema.contains("\"running\""));
5670    }
5671
5672    #[test]
5673    fn builds_remote_review_prompt_with_follow_up_guidance_and_saved_context() {
5674        let review = sample_review_record();
5675        let previous_review_run = ReviewRunRecord {
5676            dispatch_id: "review-dispatch-1".to_owned(),
5677            review_id: review.id.clone(),
5678            pull_request_url: review.pull_request_url.clone(),
5679            repository_full_name: review.repository_full_name.clone(),
5680            workspace_key: review.workspace_key.clone(),
5681            preferred_tool: review.preferred_tool,
5682            status: DispatchStatus::Succeeded,
5683            created_at: now_utc(),
5684            updated_at: now_utc(),
5685            finished_at: Some(now_utc()),
5686            remote_host: "198.51.100.10".to_owned(),
5687            branch_name: Some("track-review/review-dispatch-1".to_owned()),
5688            worktree_path: Some(
5689                "~/workspace/project-x/review-worktrees/review-dispatch-1".to_owned(),
5690            ),
5691            follow_up_request: None,
5692            target_head_oid: Some("abc123def456".to_owned()),
5693            summary: Some("Submitted a GitHub review with two inline comments.".to_owned()),
5694            review_submitted: true,
5695            github_review_id: Some("1001".to_owned()),
5696            github_review_url: Some(
5697                "https://github.com/acme/project-x/pull/42#pullrequestreview-1001".to_owned(),
5698            ),
5699            notes: None,
5700            error_message: None,
5701        };
5702        let current_review_run = ReviewRunRecord {
5703            dispatch_id: "review-dispatch-2".to_owned(),
5704            review_id: review.id.clone(),
5705            pull_request_url: review.pull_request_url.clone(),
5706            repository_full_name: review.repository_full_name.clone(),
5707            workspace_key: review.workspace_key.clone(),
5708            preferred_tool: review.preferred_tool,
5709            status: DispatchStatus::Preparing,
5710            created_at: now_utc(),
5711            updated_at: now_utc(),
5712            finished_at: None,
5713            remote_host: "198.51.100.10".to_owned(),
5714            branch_name: Some("track-review/review-dispatch-2".to_owned()),
5715            worktree_path: Some(
5716                "~/workspace/project-x/review-worktrees/review-dispatch-2".to_owned(),
5717            ),
5718            follow_up_request: Some(
5719                "Check whether the main review comments were actually resolved.".to_owned(),
5720            ),
5721            target_head_oid: Some("fedcba654321".to_owned()),
5722            summary: Some(
5723                "Re-review request: Check whether the main review comments were actually resolved."
5724                    .to_owned(),
5725            ),
5726            review_submitted: false,
5727            github_review_id: None,
5728            github_review_url: None,
5729            notes: None,
5730            error_message: None,
5731        };
5732        let prompt =
5733            build_remote_review_prompt(&review, &current_review_run, Some(&previous_review_run));
5734
5735        assert!(prompt.contains("You are responsible for submitting the GitHub review yourself"));
5736        assert!(prompt.contains("Submit one GitHub review in COMMENT mode."));
5737        assert!(prompt.contains("Prefer inline review comments"));
5738        assert!(prompt.contains("The first line of the top-level review body must be `@octocat requested me to review this PR.`"));
5739        assert!(prompt.contains("- Pinned review commit: fedcba654321"));
5740        assert!(prompt.contains("the prepared worktree is intended to match that exact commit"));
5741        assert!(prompt.contains("Capture the submitted GitHub review's durable handle"));
5742        assert!(prompt.contains("Return `reviewSubmitted` as `true` only after GitHub confirms"));
5743        assert!(prompt.contains("## Current re-review request"));
5744        assert!(prompt.contains("Check whether the main review comments were actually resolved."));
5745        assert!(prompt.contains("## Previous bot review context"));
5746        assert!(prompt.contains("https://github.com/acme/project-x/pull/42#pullrequestreview-1001"));
5747        assert!(prompt.contains("## Re-review guidance"));
5748        assert!(prompt.contains("non-blocking input at the discretion of the reviewee unless @octocat explicitly commented"));
5749        assert!(prompt.contains("do not repeat it as a primary finding"));
5750        assert!(prompt.contains("## Default review prompt"));
5751        assert!(prompt.contains("Focus on regressions and missing tests."));
5752        assert!(prompt.contains("## Extra instructions"));
5753        assert!(prompt.contains("Pay special attention to queue rendering."));
5754    }
5755
5756    #[test]
5757    fn review_worktree_script_pins_the_requested_commit_or_fails_explicitly() {
5758        let script = build_create_review_worktree_script();
5759
5760        assert!(script.contains("TARGET_HEAD_OID"));
5761        assert!(script.contains("fetch upstream \"$TARGET_HEAD_OID\""));
5762        assert!(script.contains("TARGET_REF=\"$TARGET_HEAD_OID\""));
5763        assert!(
5764            script.contains("Requested review commit $TARGET_HEAD_OID is not available locally.")
5765        );
5766        assert!(script.contains("review would drift to a newer commit"));
5767        assert!(script.contains("branch -f \"$BRANCH_NAME\" \"$TARGET_REF\""));
5768    }
5769
5770    #[test]
5771    fn review_schema_requires_review_submission_metadata_and_terminal_status_values() {
5772        let schema = build_remote_review_schema();
5773
5774        assert!(schema.contains("\"reviewSubmitted\""));
5775        assert!(schema.contains("\"githubReviewId\""));
5776        assert!(schema.contains("\"githubReviewUrl\""));
5777        assert!(schema.contains("\"succeeded\""));
5778        assert!(schema.contains("\"failed\""));
5779        assert!(schema.contains("\"blocked\""));
5780        assert!(!schema.contains("\"running\""));
5781    }
5782
5783    #[test]
5784    fn parses_github_repository_name() {
5785        assert_eq!(
5786            parse_github_repository_name("https://github.com/acme/project-x")
5787                .expect("github url should parse"),
5788            "project-x"
5789        );
5790    }
5791
5792    #[test]
5793    fn parses_github_pull_request_reference() {
5794        let reference =
5795            parse_github_pull_request_reference("https://github.com/acme/project-x/pull/42")
5796                .expect("github pr url should parse");
5797
5798        assert_eq!(reference.owner, "acme");
5799        assert_eq!(reference.repository, "project-x");
5800        assert_eq!(reference.number, 42);
5801    }
5802
5803    #[test]
5804    fn builds_review_workspace_key_from_repository_name() {
5805        let metadata = GithubPullRequestMetadata {
5806            pull_request_url: "https://github.com/acme/project-x/pull/42".to_owned(),
5807            pull_request_number: 42,
5808            pull_request_title: "Fix queue layout".to_owned(),
5809            repository_full_name: "acme/project-x".to_owned(),
5810            repo_url: "https://github.com/acme/project-x".to_owned(),
5811            git_url: "git@github.com:acme/project-x.git".to_owned(),
5812            base_branch: "main".to_owned(),
5813            head_oid: "abc123".to_owned(),
5814        };
5815
5816        assert_eq!(build_review_workspace_key(&metadata), "acme-project-x");
5817    }
5818
5819    #[test]
5820    fn builds_review_follow_up_request_that_scopes_feedback_to_one_user() {
5821        let request = build_review_follow_up_request(
5822            "https://github.com/acme/project-x/pull/42",
5823            "octocat",
5824            parse_iso_8601_seconds("2026-03-25T12:00:00Z").expect("timestamp should parse"),
5825        );
5826
5827        assert!(request.contains("@octocat"));
5828        assert!(request.contains("COMMENTED or CHANGES_REQUESTED"));
5829        assert!(request.contains("Ignore APPROVED reviews"));
5830        assert!(request.contains("https://github.com/acme/project-x/pull/42"));
5831    }
5832
5833    #[test]
5834    fn saved_review_dispatch_prerequisites_do_not_depend_on_live_review_follow_up_settings() {
5835        let context = TestContext::new(base_test_config(Some(RemoteAgentConfigFile {
5836            host: "127.0.0.1".to_owned(),
5837            user: "builder".to_owned(),
5838            port: 2222,
5839            workspace_root: "~/workspace".to_owned(),
5840            projects_registry_path: "~/track-projects.json".to_owned(),
5841            preferred_tool: RemoteAgentPreferredTool::Codex,
5842            shell_prelude: Some("export PATH=\"$PATH\"".to_owned()),
5843            review_follow_up: None,
5844        })));
5845        let review = context.create_review();
5846
5847        let _track_data_dir = set_env_var("TRACK_DATA_DIR", &context.data_dir);
5848        install_dummy_managed_remote_agent_material(&context.data_dir);
5849
5850        let (remote_agent, loaded_review) = context
5851            .review_service()
5852            .load_review_dispatch_prerequisites(&review.id)
5853            .expect("saved review dispatch prerequisites should load");
5854
5855        assert_eq!(remote_agent.host, "127.0.0.1");
5856        assert_eq!(loaded_review.id, review.id);
5857        assert_eq!(loaded_review.main_user, review.main_user);
5858        assert_eq!(
5859            loaded_review.default_review_prompt,
5860            review.default_review_prompt
5861        );
5862    }
5863
5864    #[test]
5865    fn parses_batched_dispatch_snapshot_report() {
5866        let report = concat!(
5867            "run\t~/workspace/project-x/dispatches/dispatch-1\n",
5868            "status\tpresent\t72756e6e696e670a\n",
5869            "result\tmissing\t\n",
5870            "stderr\tmissing\t\n",
5871            "finished_at\tmissing\t\n",
5872            "run\t~/workspace/project-y/dispatches/dispatch-2\n",
5873            "status\tpresent\t636f6d706c657465640a\n",
5874            "result\tpresent\t7b22737461747573223a22737563636565646564227d\n",
5875            "stderr\tpresent\t\n",
5876            "finished_at\tpresent\t323032362d30332d31385431303a33353a33315a0a\n",
5877        );
5878
5879        let snapshots =
5880            parse_dispatch_snapshot_report(report).expect("dispatch snapshot report should parse");
5881
5882        assert_eq!(
5883            snapshots
5884                .first()
5885                .expect("first dispatch snapshot should exist")
5886                .status
5887                .as_deref(),
5888            Some("running\n")
5889        );
5890        assert_eq!(
5891            snapshots
5892                .get(1)
5893                .expect("second dispatch snapshot should exist")
5894                .result
5895                .as_deref(),
5896            Some("{\"status\":\"succeeded\"}")
5897        );
5898        assert_eq!(
5899            snapshots
5900                .get(1)
5901                .expect("second dispatch snapshot should exist")
5902                .finished_at
5903                .as_deref(),
5904            Some("2026-03-18T10:35:31Z\n")
5905        );
5906    }
5907
5908    #[test]
5909    fn prepends_shell_prelude_before_remote_script_body() {
5910        let rendered = render_remote_script_with_shell_prelude(
5911            "set -eu\nprintf '%s\\n' done\n",
5912            "export NVM_DIR=\"$HOME/.nvm\"\n. \"$HOME/.cargo/env\"\n",
5913        );
5914
5915        assert!(rendered.starts_with("set -e\n"));
5916        assert!(rendered.contains("export NVM_DIR=\"$HOME/.nvm\""));
5917        assert!(rendered.contains(". \"$HOME/.cargo/env\""));
5918        assert!(rendered.contains("printf '%s\\n' done"));
5919    }
5920
5921    #[test]
5922    fn builds_codex_launcher_with_runner_shell_prelude() {
5923        let launcher = build_remote_agent_launcher(
5924            RemoteAgentPreferredTool::Codex,
5925            "export NVM_DIR=\"$HOME/.nvm\"\n. \"$HOME/.cargo/env\"\n",
5926        );
5927
5928        assert!(launcher.starts_with("#!/usr/bin/env bash"));
5929        assert!(launcher.contains("export NVM_DIR=\"$HOME/.nvm\""));
5930        assert!(launcher.contains("codex --search exec"));
5931        assert!(launcher.contains("RUN_DIR=\"$1\""));
5932        assert!(launcher.contains("WORKTREE_PATH=\"$2\""));
5933        assert!(launcher.contains("launcher.pid"));
5934        assert!(launcher.contains("codex.pid"));
5935        assert!(launcher.contains("trap cancel_run TERM INT"));
5936        assert!(launcher.contains("canceled"));
5937    }
5938
5939    #[test]
5940    fn builds_claude_launcher_with_schema_validation_and_yolo_mode() {
5941        let launcher = build_remote_agent_launcher(
5942            RemoteAgentPreferredTool::Claude,
5943            "export PATH=\"$HOME/.local/bin:$PATH\"\n",
5944        );
5945
5946        assert!(launcher.starts_with("#!/usr/bin/env bash"));
5947        assert!(launcher.contains("export PATH=\"$HOME/.local/bin:$PATH\""));
5948        assert!(launcher.contains("SCHEMA_CONTENT=\"$(tr -d '\\n'"));
5949        assert!(launcher.contains("cd \"$WORKTREE_PATH\""));
5950        assert!(launcher.contains("claude -p --dangerously-skip-permissions"));
5951        assert!(launcher.contains("--output-format json"));
5952        assert!(launcher.contains("--json-schema \"$SCHEMA_CONTENT\""));
5953        assert!(launcher.contains("codex.pid"));
5954    }
5955
5956    #[test]
5957    fn refresh_reads_claude_dispatch_outcome_from_structured_output_envelope() {
5958        let created_at = now_utc();
5959        let record = TaskDispatchRecord {
5960            dispatch_id: "dispatch-1".to_owned(),
5961            task_id: "task-1".to_owned(),
5962            preferred_tool: RemoteAgentPreferredTool::Claude,
5963            project: "project-a".to_owned(),
5964            status: DispatchStatus::Running,
5965            created_at,
5966            updated_at: created_at,
5967            finished_at: None,
5968            remote_host: "192.0.2.25".to_owned(),
5969            branch_name: Some("track/dispatch-1".to_owned()),
5970            worktree_path: Some("~/workspace/project-a/worktrees/dispatch-1".to_owned()),
5971            pull_request_url: None,
5972            follow_up_request: None,
5973            summary: None,
5974            notes: None,
5975            error_message: None,
5976            review_request_head_oid: None,
5977            review_request_user: None,
5978        };
5979        let snapshot = RemoteDispatchSnapshot {
5980            status: Some("completed\n".to_owned()),
5981            result: Some(
5982                json!({
5983                    "result": "Mock Claude completed successfully.",
5984                    "structured_output": {
5985                        "status": "succeeded",
5986                        "summary": "Mock Claude completed successfully.",
5987                        "pullRequestUrl": "https://github.com/acme/project-a/pull/42",
5988                        "branchName": "track/dispatch-1",
5989                        "worktreePath": "/tmp/project-a/worktrees/dispatch-1",
5990                        "notes": "Captured from the Claude mock."
5991                    }
5992                })
5993                .to_string(),
5994            ),
5995            stderr: None,
5996            finished_at: Some("2026-03-18T10:35:31Z\n".to_owned()),
5997        };
5998
5999        let refreshed = refresh_dispatch_record_from_snapshot(record, &snapshot)
6000            .expect("Claude envelope should refresh successfully");
6001
6002        assert_eq!(refreshed.status, DispatchStatus::Succeeded);
6003        assert_eq!(
6004            refreshed.summary.as_deref(),
6005            Some("Mock Claude completed successfully.")
6006        );
6007        assert_eq!(
6008            refreshed.pull_request_url.as_deref(),
6009            Some("https://github.com/acme/project-a/pull/42")
6010        );
6011        assert_eq!(
6012            refreshed.worktree_path.as_deref(),
6013            Some("/tmp/project-a/worktrees/dispatch-1")
6014        );
6015        assert_eq!(
6016            refreshed.notes.as_deref(),
6017            Some("Captured from the Claude mock.")
6018        );
6019    }
6020
6021    #[test]
6022    fn refresh_reads_claude_review_outcome_from_structured_output_envelope() {
6023        let context = TestContext::new(base_test_config(None));
6024        let created_at = now_utc();
6025        let record = ReviewRunRecord {
6026            dispatch_id: "review-dispatch-1".to_owned(),
6027            review_id: "review-1".to_owned(),
6028            pull_request_url: "https://github.com/acme/project-a/pull/42".to_owned(),
6029            repository_full_name: "acme/project-a".to_owned(),
6030            workspace_key: "project-a".to_owned(),
6031            preferred_tool: RemoteAgentPreferredTool::Claude,
6032            status: DispatchStatus::Running,
6033            created_at,
6034            updated_at: created_at,
6035            finished_at: None,
6036            remote_host: "192.0.2.25".to_owned(),
6037            branch_name: Some("track-review/review-dispatch-1".to_owned()),
6038            worktree_path: Some("~/workspace/project-a/review-worktrees/review-1".to_owned()),
6039            follow_up_request: None,
6040            target_head_oid: Some("abc123def456".to_owned()),
6041            summary: None,
6042            review_submitted: false,
6043            github_review_id: None,
6044            github_review_url: None,
6045            notes: None,
6046            error_message: None,
6047        };
6048        let snapshot = RemoteDispatchSnapshot {
6049            status: Some("completed\n".to_owned()),
6050            result: Some(
6051                json!({
6052                    "result": "Mock Claude reviewed the pull request successfully.",
6053                    "structured_output": {
6054                        "status": "succeeded",
6055                        "summary": "Mock Claude reviewed the pull request successfully.",
6056                        "reviewSubmitted": true,
6057                        "githubReviewId": "1001",
6058                        "githubReviewUrl": "https://github.com/acme/project-a/pull/42#pullrequestreview-1001",
6059                        "worktreePath": "/tmp/project-a/review-worktrees/review-1",
6060                        "notes": "Captured from the Claude review mock."
6061                    }
6062                })
6063                .to_string(),
6064            ),
6065            stderr: None,
6066            finished_at: Some("2026-03-18T10:35:31Z\n".to_owned()),
6067        };
6068
6069        let refreshed = context
6070            .review_service()
6071            .refresh_review_dispatch_record_from_snapshot(record, &snapshot)
6072            .expect("Claude review envelope should refresh successfully");
6073
6074        assert_eq!(refreshed.status, DispatchStatus::Succeeded);
6075        assert_eq!(
6076            refreshed.summary.as_deref(),
6077            Some("Mock Claude reviewed the pull request successfully.")
6078        );
6079        assert!(refreshed.review_submitted);
6080        assert_eq!(refreshed.github_review_id.as_deref(), Some("1001"));
6081        assert_eq!(
6082            refreshed.github_review_url.as_deref(),
6083            Some("https://github.com/acme/project-a/pull/42#pullrequestreview-1001")
6084        );
6085        assert_eq!(
6086            refreshed.worktree_path.as_deref(),
6087            Some("/tmp/project-a/review-worktrees/review-1")
6088        );
6089        assert_eq!(
6090            refreshed.notes.as_deref(),
6091            Some("Captured from the Claude review mock.")
6092        );
6093    }
6094
6095    #[test]
6096    fn refresh_marks_remote_canceled_runs_as_terminal() {
6097        let created_at = now_utc();
6098        let record = TaskDispatchRecord {
6099            dispatch_id: "dispatch-1".to_owned(),
6100            task_id: "task-1".to_owned(),
6101            preferred_tool: RemoteAgentPreferredTool::Codex,
6102            project: "project-a".to_owned(),
6103            status: DispatchStatus::Running,
6104            created_at,
6105            updated_at: created_at,
6106            finished_at: None,
6107            remote_host: "192.0.2.25".to_owned(),
6108            branch_name: Some("track/dispatch-1".to_owned()),
6109            worktree_path: Some("~/workspace/project-a/worktrees/dispatch-1".to_owned()),
6110            pull_request_url: None,
6111            follow_up_request: None,
6112            summary: None,
6113            notes: None,
6114            error_message: None,
6115            review_request_head_oid: None,
6116            review_request_user: None,
6117        };
6118        let snapshot = RemoteDispatchSnapshot {
6119            status: Some("canceled\n".to_owned()),
6120            result: None,
6121            stderr: None,
6122            finished_at: Some("2026-03-18T10:35:31Z\n".to_owned()),
6123        };
6124
6125        let refreshed = refresh_dispatch_record_from_snapshot(record, &snapshot)
6126            .expect("canceled snapshot should refresh");
6127
6128        assert_eq!(refreshed.status, DispatchStatus::Canceled);
6129        assert_eq!(
6130            refreshed.summary.as_deref(),
6131            Some("Canceled from the web UI.")
6132        );
6133        assert!(refreshed.finished_at.is_some());
6134    }
6135
6136    #[test]
6137    fn follow_up_uses_the_latest_reusable_dispatch_context() {
6138        let created_at = now_utc();
6139        let dispatch_history = vec![
6140            TaskDispatchRecord {
6141                dispatch_id: "dispatch-3".to_owned(),
6142                task_id: "task-1".to_owned(),
6143                preferred_tool: RemoteAgentPreferredTool::Codex,
6144                project: "project-a".to_owned(),
6145                status: DispatchStatus::Failed,
6146                created_at: created_at + Duration::seconds(2),
6147                updated_at: created_at + Duration::seconds(2),
6148                finished_at: Some(created_at + Duration::seconds(2)),
6149                remote_host: "192.0.2.25".to_owned(),
6150                branch_name: None,
6151                worktree_path: None,
6152                pull_request_url: None,
6153                follow_up_request: Some("Address review comments".to_owned()),
6154                summary: Some("Launch failed before the branch was restored.".to_owned()),
6155                notes: None,
6156                error_message: Some("Remote launch failed.".to_owned()),
6157                review_request_head_oid: None,
6158                review_request_user: None,
6159            },
6160            TaskDispatchRecord {
6161                dispatch_id: "dispatch-2".to_owned(),
6162                task_id: "task-1".to_owned(),
6163                preferred_tool: RemoteAgentPreferredTool::Claude,
6164                project: "project-a".to_owned(),
6165                status: DispatchStatus::Succeeded,
6166                created_at: created_at + Duration::seconds(1),
6167                updated_at: created_at + Duration::seconds(1),
6168                finished_at: Some(created_at + Duration::seconds(1)),
6169                remote_host: "192.0.2.25".to_owned(),
6170                branch_name: Some("track/dispatch-2".to_owned()),
6171                worktree_path: Some("~/workspace/project-a/worktrees/dispatch-2".to_owned()),
6172                pull_request_url: Some("https://github.com/acme/project-a/pull/42".to_owned()),
6173                follow_up_request: None,
6174                summary: Some("Opened a PR.".to_owned()),
6175                notes: None,
6176                error_message: None,
6177                review_request_head_oid: None,
6178                review_request_user: None,
6179            },
6180            TaskDispatchRecord {
6181                dispatch_id: "dispatch-1".to_owned(),
6182                task_id: "task-1".to_owned(),
6183                preferred_tool: RemoteAgentPreferredTool::Codex,
6184                project: "project-a".to_owned(),
6185                status: DispatchStatus::Failed,
6186                created_at,
6187                updated_at: created_at,
6188                finished_at: Some(created_at),
6189                remote_host: "192.0.2.25".to_owned(),
6190                branch_name: Some("track/dispatch-1".to_owned()),
6191                worktree_path: Some("~/workspace/project-a/worktrees/dispatch-1".to_owned()),
6192                pull_request_url: Some("https://github.com/acme/project-a/pull/1".to_owned()),
6193                follow_up_request: None,
6194                summary: None,
6195                notes: None,
6196                error_message: Some("Old failure.".to_owned()),
6197                review_request_head_oid: None,
6198                review_request_user: None,
6199            },
6200        ];
6201
6202        let selected = select_follow_up_base_dispatch(&dispatch_history)
6203            .expect("a reusable dispatch should be selected");
6204        let pull_request_url = latest_pull_request_for_branch(
6205            &dispatch_history,
6206            selected
6207                .branch_name
6208                .as_deref()
6209                .expect("selected dispatch should have a branch name"),
6210        );
6211
6212        assert_eq!(selected.dispatch_id, "dispatch-2");
6213        assert_eq!(
6214            pull_request_url.as_deref(),
6215            Some("https://github.com/acme/project-a/pull/42")
6216        );
6217    }
6218
6219    #[test]
6220    fn selects_the_latest_previous_submitted_review_run() {
6221        let review = sample_review_record();
6222        let dispatch_history = vec![
6223            ReviewRunRecord {
6224                dispatch_id: "dispatch-3".to_owned(),
6225                review_id: review.id.clone(),
6226                pull_request_url: review.pull_request_url.clone(),
6227                repository_full_name: review.repository_full_name.clone(),
6228                workspace_key: review.workspace_key.clone(),
6229                preferred_tool: review.preferred_tool,
6230                status: DispatchStatus::Preparing,
6231                created_at: now_utc(),
6232                updated_at: now_utc(),
6233                finished_at: None,
6234                remote_host: "192.0.2.25".to_owned(),
6235                branch_name: Some("track-review/dispatch-3".to_owned()),
6236                worktree_path: Some("~/workspace/project-x/review-worktrees/dispatch-3".to_owned()),
6237                follow_up_request: Some("Re-review the latest fixes.".to_owned()),
6238                target_head_oid: Some("ccc333".to_owned()),
6239                summary: Some("Re-review request: Re-review the latest fixes.".to_owned()),
6240                review_submitted: false,
6241                github_review_id: None,
6242                github_review_url: None,
6243                notes: None,
6244                error_message: None,
6245            },
6246            ReviewRunRecord {
6247                dispatch_id: "dispatch-2".to_owned(),
6248                review_id: review.id.clone(),
6249                pull_request_url: review.pull_request_url.clone(),
6250                repository_full_name: review.repository_full_name.clone(),
6251                workspace_key: review.workspace_key.clone(),
6252                preferred_tool: review.preferred_tool,
6253                status: DispatchStatus::Succeeded,
6254                created_at: now_utc(),
6255                updated_at: now_utc(),
6256                finished_at: Some(now_utc()),
6257                remote_host: "192.0.2.25".to_owned(),
6258                branch_name: Some("track-review/dispatch-2".to_owned()),
6259                worktree_path: Some("~/workspace/project-x/review-worktrees/dispatch-2".to_owned()),
6260                follow_up_request: None,
6261                target_head_oid: Some("bbb222".to_owned()),
6262                summary: Some("Submitted a review.".to_owned()),
6263                review_submitted: true,
6264                github_review_id: Some("1002".to_owned()),
6265                github_review_url: Some(
6266                    "https://github.com/acme/project-x/pull/42#pullrequestreview-1002".to_owned(),
6267                ),
6268                notes: None,
6269                error_message: None,
6270            },
6271            ReviewRunRecord {
6272                dispatch_id: "dispatch-1".to_owned(),
6273                review_id: review.id.clone(),
6274                pull_request_url: review.pull_request_url.clone(),
6275                repository_full_name: review.repository_full_name.clone(),
6276                workspace_key: review.workspace_key.clone(),
6277                preferred_tool: review.preferred_tool,
6278                status: DispatchStatus::Succeeded,
6279                created_at: now_utc(),
6280                updated_at: now_utc(),
6281                finished_at: Some(now_utc()),
6282                remote_host: "192.0.2.25".to_owned(),
6283                branch_name: Some("track-review/dispatch-1".to_owned()),
6284                worktree_path: Some("~/workspace/project-x/review-worktrees/dispatch-1".to_owned()),
6285                follow_up_request: None,
6286                target_head_oid: Some("aaa111".to_owned()),
6287                summary: Some("Submitted an older review.".to_owned()),
6288                review_submitted: true,
6289                github_review_id: Some("1001".to_owned()),
6290                github_review_url: Some(
6291                    "https://github.com/acme/project-x/pull/42#pullrequestreview-1001".to_owned(),
6292                ),
6293                notes: None,
6294                error_message: None,
6295            },
6296        ];
6297
6298        let selected = select_previous_submitted_review_run(&dispatch_history, "dispatch-3")
6299            .expect("a previous submitted review should be selected");
6300
6301        assert_eq!(selected.dispatch_id, "dispatch-2");
6302        assert_eq!(selected.github_review_id.as_deref(), Some("1002"));
6303    }
6304
6305    #[test]
6306    fn closing_task_stays_local_when_remote_cleanup_is_unavailable() {
6307        let context = TestContext::new(base_test_config(None));
6308        let task = context.create_task("project-a", "Investigate a flaky remote cleanup");
6309        let existing_dispatch = context.create_running_dispatch(&task);
6310
6311        let updated_task = context
6312            .service()
6313            .update_task(
6314                &task.id,
6315                TaskUpdateInput {
6316                    status: Some(Status::Closed),
6317                    ..TaskUpdateInput::default()
6318                },
6319            )
6320            .expect("closing should still succeed locally");
6321
6322        let updated_dispatch = context
6323            .dispatch_repository
6324            .get_dispatch(&task.id, &existing_dispatch.dispatch_id)
6325            .expect("dispatch lookup should succeed")
6326            .expect("dispatch should still exist");
6327
6328        assert_eq!(updated_task.status, Status::Closed);
6329        assert_eq!(updated_dispatch.status, DispatchStatus::Canceled);
6330        assert_eq!(
6331            updated_dispatch.summary.as_deref(),
6332            Some("Canceled because the task was closed locally. Remote cleanup was skipped.")
6333        );
6334        assert!(updated_dispatch
6335            .error_message
6336            .as_deref()
6337            .is_some_and(|message| message.contains("remote-agent configuration is missing")));
6338    }
6339
6340    #[test]
6341    fn deleting_task_stays_local_when_remote_cleanup_is_unavailable() {
6342        let context = TestContext::new(base_test_config(None));
6343        let task = context.create_task("project-a", "Delete the task even without remote cleanup");
6344        let _existing_dispatch = context.create_running_dispatch(&task);
6345
6346        context
6347            .service()
6348            .delete_task(&task.id)
6349            .expect("delete should still succeed locally");
6350
6351        let task_error = context
6352            .task_repository
6353            .get_task(&task.id)
6354            .expect_err("deleted task should be gone");
6355        assert_eq!(task_error.code, crate::errors::ErrorCode::TaskNotFound);
6356        assert!(context
6357            .dispatch_repository
6358            .dispatches_for_task(&task.id)
6359            .expect("dispatch lookup should succeed")
6360            .is_empty());
6361    }
6362
6363    #[test]
6364    fn refresh_releases_active_dispatches_when_remote_config_disappears() {
6365        let context = TestContext::new(base_test_config(None));
6366        let task = context.create_task("project-a", "Recover from a missing remote config");
6367        let existing_dispatch = context.create_running_dispatch(&task);
6368
6369        let refreshed = context
6370            .service()
6371            .latest_dispatches_for_tasks(std::slice::from_ref(&task.id))
6372            .expect("dispatch refresh should succeed");
6373        let updated_dispatch = refreshed
6374            .first()
6375            .expect("latest dispatch should still be returned");
6376
6377        assert_eq!(updated_dispatch.dispatch_id, existing_dispatch.dispatch_id);
6378        assert_eq!(updated_dispatch.status, DispatchStatus::Blocked);
6379        assert_eq!(
6380            updated_dispatch.summary.as_deref(),
6381            Some("Remote reconciliation is unavailable locally, so active runs were released.")
6382        );
6383        assert_eq!(
6384            updated_dispatch.error_message.as_deref(),
6385            Some("Remote agent configuration is missing locally.")
6386        );
6387    }
6388
6389    #[test]
6390    fn queue_dispatch_releases_stale_active_dispatch_when_remote_refresh_fails() {
6391        let context = TestContext::new(base_test_config(Some(RemoteAgentConfigFile {
6392            host: "127.0.0.1".to_owned(),
6393            user: "builder".to_owned(),
6394            port: 1,
6395            workspace_root: "~/workspace".to_owned(),
6396            projects_registry_path: "~/track-projects.json".to_owned(),
6397            preferred_tool: RemoteAgentPreferredTool::Codex,
6398            shell_prelude: Some("export PATH=\"$PATH\"".to_owned()),
6399            review_follow_up: None,
6400        })));
6401        let task =
6402            context.create_task("project-a", "Retry after the previous remote run got stuck");
6403        context.write_project_metadata(&task.project);
6404        let existing_dispatch = context.create_running_dispatch(&task);
6405
6406        let _track_data_dir = set_env_var("TRACK_DATA_DIR", &context.data_dir);
6407        install_dummy_managed_remote_agent_material(&context.data_dir);
6408
6409        let queued_dispatch = context
6410            .service()
6411            .queue_dispatch(&task.id, None)
6412            .expect("queueing should release the stale active dispatch first");
6413        let released_dispatch = context
6414            .dispatch_repository
6415            .get_dispatch(&task.id, &existing_dispatch.dispatch_id)
6416            .expect("dispatch lookup should succeed")
6417            .expect("previous dispatch should still exist");
6418
6419        assert_ne!(queued_dispatch.dispatch_id, existing_dispatch.dispatch_id);
6420        assert_eq!(queued_dispatch.status, DispatchStatus::Preparing);
6421        assert_eq!(released_dispatch.status, DispatchStatus::Blocked);
6422        assert_eq!(
6423            released_dispatch.summary.as_deref(),
6424            Some(
6425                "Remote reconciliation could not reach the remote machine, so active runs were released locally."
6426            )
6427        );
6428        assert!(released_dispatch.error_message.is_some());
6429    }
6430
6431    #[test]
6432    fn follow_up_dispatch_keeps_the_original_runner_tool() {
6433        let context = TestContext::new(base_test_config(Some(RemoteAgentConfigFile {
6434            host: "127.0.0.1".to_owned(),
6435            user: "builder".to_owned(),
6436            port: 2222,
6437            workspace_root: "~/workspace".to_owned(),
6438            projects_registry_path: "~/track-projects.json".to_owned(),
6439            preferred_tool: RemoteAgentPreferredTool::Codex,
6440            shell_prelude: Some("export PATH=\"$PATH\"".to_owned()),
6441            review_follow_up: None,
6442        })));
6443        let task = context.create_task("project-a", "Keep using the same runner on follow-up");
6444        context.write_project_metadata(&task.project);
6445        let _track_data_dir = set_env_var("TRACK_DATA_DIR", &context.data_dir);
6446        install_dummy_managed_remote_agent_material(&context.data_dir);
6447
6448        let mut first_dispatch = context
6449            .service()
6450            .queue_dispatch(&task.id, Some(RemoteAgentPreferredTool::Claude))
6451            .expect("initial dispatch should queue");
6452        first_dispatch.status = DispatchStatus::Succeeded;
6453        first_dispatch.finished_at = Some(first_dispatch.updated_at);
6454        context
6455            .dispatch_repository
6456            .save_dispatch(&first_dispatch)
6457            .expect("initial dispatch should save as terminal");
6458
6459        let follow_up_dispatch = context
6460            .service()
6461            .queue_follow_up_dispatch(&task.id, "Address the review comments.")
6462            .expect("follow-up dispatch should queue");
6463
6464        assert_eq!(
6465            first_dispatch.preferred_tool,
6466            RemoteAgentPreferredTool::Claude
6467        );
6468        assert_eq!(
6469            follow_up_dispatch.preferred_tool,
6470            RemoteAgentPreferredTool::Claude
6471        );
6472    }
6473
6474    #[test]
6475    fn reset_blockers_include_active_review_runs() {
6476        let context = TestContext::new(base_test_config(None));
6477        let task = context.create_task("project-a", "Keep reset from interrupting live work");
6478        let task_dispatch = context.create_running_dispatch(&task);
6479        let review = context.create_review();
6480        let review_dispatch = context.create_review_run(&review, DispatchStatus::Running);
6481
6482        let blockers = describe_remote_reset_blockers(&[task_dispatch], &[review_dispatch]);
6483
6484        assert_eq!(blockers.len(), 2);
6485        assert!(blockers
6486            .iter()
6487            .any(|blocker| blocker.contains(&task.id) && blocker.contains("task")));
6488        assert!(blockers
6489            .iter()
6490            .any(|blocker| blocker.contains(&review.id) && blocker.contains("review")));
6491    }
6492
6493    #[test]
6494    fn task_dispatch_start_guard_serializes_same_task() {
6495        let acquired_in_second_thread = Arc::new(AtomicBool::new(false));
6496        let guard = super::TaskDispatchStartGuard::acquire("task-1");
6497
6498        std::thread::scope(|scope| {
6499            let acquired_in_second_thread_for_join = Arc::clone(&acquired_in_second_thread);
6500            let join_handle = scope.spawn(move || {
6501                let _guard = super::TaskDispatchStartGuard::acquire("task-1");
6502                acquired_in_second_thread_for_join.store(true, Ordering::SeqCst);
6503            });
6504
6505            std::thread::sleep(std::time::Duration::from_millis(50));
6506            assert!(
6507                !acquired_in_second_thread.load(Ordering::SeqCst),
6508                "the second same-task start should stay blocked while the first guard is held",
6509            );
6510
6511            drop(guard);
6512            join_handle
6513                .join()
6514                .expect("second thread should acquire the guard after release");
6515        });
6516
6517        assert!(
6518            acquired_in_second_thread.load(Ordering::SeqCst),
6519            "the waiting same-task start should proceed after the first guard releases",
6520        );
6521    }
6522
6523    #[test]
6524    fn review_dispatch_start_guard_serializes_same_review() {
6525        let acquired_in_second_thread = Arc::new(AtomicBool::new(false));
6526        let guard = super::ReviewDispatchStartGuard::acquire("review-1");
6527
6528        std::thread::scope(|scope| {
6529            let acquired_in_second_thread_for_join = Arc::clone(&acquired_in_second_thread);
6530            let join_handle = scope.spawn(move || {
6531                let _guard = super::ReviewDispatchStartGuard::acquire("review-1");
6532                acquired_in_second_thread_for_join.store(true, Ordering::SeqCst);
6533            });
6534
6535            std::thread::sleep(std::time::Duration::from_millis(50));
6536            assert!(
6537                !acquired_in_second_thread.load(Ordering::SeqCst),
6538                "the second same-review start should stay blocked while the first guard is held",
6539            );
6540
6541            drop(guard);
6542            join_handle
6543                .join()
6544                .expect("second thread should acquire the guard after release");
6545        });
6546
6547        assert!(
6548            acquired_in_second_thread.load(Ordering::SeqCst),
6549            "the waiting same-review start should proceed after the first guard releases",
6550        );
6551    }
6552}