Skip to main content

orch/engine/
mod.rs

1//! Engine — the core orchestration loop.
2//!
3//! The engine is the central coordinator. It owns the tick loop, backend
4//! connections, channel registry, transport layer, and tmux session manager.
5//!
6//! Submodules handle the heavy lifting:
7//! - [`tick`] — core tick phases (session polling, stuck recovery, routing, dispatch)
8//! - [`sync`] — periodic sync operations (cleanup, PR review, mentions)
9//! - [`review`] — PR review pipeline (review agent, auto-merge, change handling)
10//! - [`cleanup`] — worktree cleanup and merged-PR detection
11//!
12//! This file contains struct definitions, project initialization, and the
13//! main event loop (`serve()`).
14
15pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40/// Standard Orch attribution footer for issue bodies, PR bodies, and comments.
41///
42/// Append to every user-visible string posted to GitHub so activity is
43/// clearly attributed to orch rather than appearing as a human post.
44use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{
58    local_branch_exists, remove_worktree_and_branch, remove_worktree_and_branch_for,
59};
60use crate::engine::router::{Router, RouterConfig};
61use crate::engine::runner::worktree::{
62    abort_worktree_rebase, list_project_worktrees, task_id_from_worktree_name,
63    validate_worktree_gitdir,
64};
65use crate::engine::tasks::TaskManager;
66use crate::github::http::{GhHttp, rate_limit_metrics};
67use crate::repo_context::REPO_CONTEXT;
68use crate::store::{TaskStatus, TaskStore};
69use crate::tmux::TmuxManager;
70use runner::WeightSignal;
71// AtomicBool/Ordering removed — shutdown is now immediate (reset tasks + break)
72use crate::backends::Status;
73use crate::backends::github::GitHubBackend;
74use crate::channels::github;
75use crate::channels::github::is_transient_bind_error;
76use crate::channels::github::webhook_backoff_delay;
77use crate::channels::notification;
78use crate::channels::notification::TaskNotification;
79use crate::channels::transport;
80use crate::cluster;
81use crate::cluster::local;
82use crate::engine::cooldown as engine_cooldown;
83use crate::engine::review as engine_review;
84use crate::engine::runner::git_ops;
85use crate::engine::runner::worktree;
86use crate::engine::suspend as engine_suspend;
87use crate::engine::suspend::SuspendDetector;
88use crate::engine::tasks as engine_tasks;
89use crate::github::app;
90use crate::home;
91use crate::secret_ref;
92use crate::store;
93use crate::webhook_status::WebhookStatus;
94use futures::future;
95use std::collections::HashMap;
96use std::collections::HashSet;
97use std::fmt;
98use std::fmt::Debug;
99use std::fmt::Formatter;
100use std::fs;
101use std::path::Path;
102use std::path::PathBuf;
103use std::sync::Arc;
104use std::sync::atomic::AtomicBool;
105use std::sync::atomic::AtomicU64;
106use std::sync::atomic::Ordering;
107use std::time as std_time;
108use std::time::Duration;
109use std::time::Instant;
110use std::time::SystemTime;
111use tokio::process::Command;
112use tokio::signal;
113use tokio::signal::unix;
114use tokio::signal::unix::SignalKind;
115use tokio::sync::Mutex;
116use tokio::sync::broadcast::Sender;
117use tokio::sync::broadcast::error::RecvError;
118use tokio::sync::mpsc::Receiver;
119use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
120use tokio::task;
121use tokio::time;
122
123pub fn orch_footer() -> &'static str {
124    "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
125}
126
127/// Build a standard attribution footer for GitHub comments posted by orch bots.
128///
129/// `verb` is "Created", "Reviewed", or "Commented" depending on context.
130pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
131    match model {
132        Some(m) => format!(
133            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
134        ),
135        None => format!(
136            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
137        ),
138    }
139}
140
141/// Lightweight reference tuple for channel message handling.
142type EngineRef = (
143    String,
144    Arc<dyn ExternalBackend>,
145    Arc<TaskManager>,
146    Option<Arc<TaskStore>>,
147);
148
149/// Per-project engine state.
150///
151/// Each project has its own backend, task runner, and task manager,
152/// but they share the global tmux manager, transport, and semaphore.
153#[derive(Clone)]
154pub struct ProjectEngine {
155    pub repo: String,
156    pub project_dir: PathBuf,
157    pub backend: Arc<dyn ExternalBackend>,
158    pub task_manager: Arc<TaskManager>,
159    pub runner: Arc<runner::TaskRunner>,
160    pub store: Arc<TaskStore>,
161}
162
163impl Debug for ProjectEngine {
164    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
165        f.debug_struct("ProjectEngine")
166            .field("repo", &self.repo)
167            .field("project_dir", &self.project_dir)
168            .finish_non_exhaustive()
169    }
170}
171
172/// A restart ends every review session, so an `in_review` task with no session has no reviewer.
173/// Hand it back to `needs_review` at start, so it does not wait for the stuck-review recovery
174pub(crate) async fn reset_orphaned_reviews(
175    task_manager: &TaskManager,
176    store: &Arc<TaskStore>,
177    repo: &str,
178    tmux: &TmuxManager,
179) -> usize {
180    // The store holds external and internal tasks, the second list is the legacy internal table
181    let mut ids: Vec<ExternalId> = store
182        .list_by_status(repo, TaskStatus::InReview)
183        .await
184        .unwrap_or_default()
185        .iter()
186        .map(|t| engine_tasks::store_task_to_external(t).id)
187        .collect();
188    if let Ok(legacy) = task_manager
189        .list_internal_by_status(TaskStatus::InReview)
190        .await
191    {
192        ids.extend(legacy.into_iter().map(|t| t.id));
193    }
194    ids.sort_by(|a, b| a.0.cmp(&b.0));
195    ids.dedup_by(|a, b| a.0 == b.0);
196
197    let mut reset = 0;
198    for id in &ids {
199        let session = tmux.session_name(repo, &format!("{}-review", id.0));
200
201        if tmux.session_exists(&session).await {
202            continue;
203        }
204
205        if let Err(e) = task_manager
206            .update_task_status(id, Status::NeedsReview)
207            .await
208        {
209            tracing::warn!(
210                task_id = id.0,
211                ?e,
212                "failed to reset an orphaned review at start"
213            );
214            continue;
215        }
216
217        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
218            let reason = "startup: review session gone, reset to needs_review";
219
220            if let Err(e) = store
221                .finalize_incomplete_runs(store_id, "aborted", reason)
222                .await
223            {
224                tracing::warn!(
225                    task_id = id.0,
226                    ?e,
227                    "failed to finalize the orphaned review's runs"
228                );
229            }
230        }
231
232        reset += 1;
233    }
234
235    reset
236}
237
238/// Starts the relay client when `relay.url` is set. The token is read in the background, so a
239/// passbox prompt nobody answers never holds up the engine
240fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
241    // The global config only: a project file must never point a secret at its own endpoint
242    let url = config::get_global("relay.url").unwrap_or_default();
243    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
244        return;
245    };
246
247    if url.is_empty() {
248        return;
249    }
250
251    let token_ref = config::get_global("relay.token").unwrap_or_default();
252    let notify = Arc::clone(notify);
253    tokio::spawn(async move {
254        let mut warned = false;
255        let token = loop {
256            let reference = token_ref.clone();
257            let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
258                .await
259                .map_err(anyhow::Error::from)
260                .and_then(|r| r);
261
262            match read {
263                Ok(token) if !token.is_empty() => break token,
264                Ok(_) => {
265                    tracing::warn!("relay.token is empty, the relay client stays off");
266                    return;
267                }
268                Err(e) if !warned => {
269                    tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
270                    warned = true;
271                }
272                Err(_) => {}
273            }
274
275            time::sleep(Duration::from_secs(300)).await;
276        };
277
278        relay::run(url, token, store, notify).await;
279    });
280}
281
282/**
283 * Runs `sync_tick` for every project at once, then the tail that is the same for all of them,
284 * in a spawned task. Sync time is the slowest project instead of the sum of all of them
285 * (#3864). The caller must already hold `sync_in_progress`; this releases it when done
286 **/
287#[expect(
288    clippy::too_many_arguments,
289    reason = "the spawned task takes its own handle to each piece of engine state"
290)]
291fn spawn_sync_pass(
292    project_engines: &[ProjectEngine],
293    tmux: &Arc<TmuxManager>,
294    engine_config: &EngineConfig,
295    router: &Arc<RwLock<Router>>,
296    dispatching: &Arc<dashmap::DashMap<String, String>>,
297    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
298    sync_in_progress: &Arc<AtomicBool>,
299    trigger: &'static str,
300) {
301    let engines = project_engines.to_vec();
302    let tmux = Arc::clone(tmux);
303    let engine_config = engine_config.clone();
304    let router = Arc::clone(router);
305    let dispatching = Arc::clone(dispatching);
306    let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
307    let guard = Arc::clone(sync_in_progress);
308
309    tokio::spawn(async move {
310        let started = Instant::now();
311        let mut passes = Vec::with_capacity(engines.len());
312        for engine in &engines {
313            passes.push(REPO_CONTEXT.scope(
314                engine.repo.clone(),
315                sync_one_project(
316                    engine,
317                    &tmux,
318                    &engine_config,
319                    &router,
320                    &dispatching,
321                    &auto_merge_in_flight,
322                ),
323            ));
324        }
325        future::join_all(passes).await;
326
327        sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
328        tracing::info!(
329            trigger,
330            elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
331            "sync tick complete"
332        );
333        guard.store(false, Ordering::Release);
334    });
335}
336
337async fn sync_one_project(
338    engine: &ProjectEngine,
339    tmux: &Arc<TmuxManager>,
340    engine_config: &EngineConfig,
341    router: &Arc<RwLock<Router>>,
342    dispatching: &Arc<dashmap::DashMap<String, String>>,
343    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
344) {
345    if let Err(e) = Box::pin(sync::sync_tick(
346        &engine.backend,
347        tmux,
348        &engine.repo,
349        engine_config,
350        router,
351        &engine.task_manager,
352        &engine.store,
353        dispatching,
354        auto_merge_in_flight,
355    ))
356    .await
357    {
358        tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
359    }
360}
361
362/// Milliseconds a group of tick or sync phases may take before it is worth an operator's eye
363const SLOW_PHASE_GROUP_MS: u128 = 1_000;
364
365/**
366 * Logs how long each phase of a tick or sync group took, for one project. Tick and sync time
367 * grows with the project count, and the totals alone did not say which project or phase spent
368 * it (#3864). Info once a group crosses `SLOW_PHASE_GROUP_MS`, so a quiet install stays quiet
369 **/
370pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
371    let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
372    if total_ms >= SLOW_PHASE_GROUP_MS {
373        tracing::info!(group, repo, total_ms, ?phases, "phase timings");
374    } else {
375        tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
376    }
377}
378
379/// Return the configured agent list from top-level `agents` in config.yml,
380/// falling back to [`router::config::DEFAULT_AGENTS`] when not set.
381///
382/// This is the canonical source of truth for "which agents does orch know
383/// about" — use it instead of `DEFAULT_AGENTS` directly so that
384/// Claude-compatible agents added via config (e.g. `olm`) are recognized
385/// without code changes.
386///
387/// The key is top-level (`agents:`) rather than nested (`engine.agents:`)
388/// because `config::get` serializes nested YAML arrays via `serde_norway::to_string`
389/// which produces YAML block format, not JSON — breaking downstream parsing.
390pub fn configured_agents() -> Vec<String> {
391    if let Ok(agents_str) = config::get("agents")
392        && !agents_str.is_empty()
393        && agents_str != "[]"
394    {
395        // config::get returns serde_norway::to_string for arrays, which is
396        // YAML block format ("- claude\n- codex\n..."). Parse as YAML first.
397        if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
398            && !agents_arr.is_empty()
399        {
400            return agents_arr;
401        }
402        // Fallback: try JSON (inline arrays) or comma-separated
403        if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
404            && !agents_arr.is_empty()
405        {
406            return agents_arr;
407        }
408    }
409    router::config::DEFAULT_AGENTS
410        .iter()
411        .map(ToString::to_string)
412        .collect()
413}
414
415/// Seconds a live session may run past its own timeout before stuck recovery takes it. The
416/// `timeout` wrapper ends the agent at the limit, and the runner still collects the result
417pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
418
419/// The limit the `timeout` wrapper puts on one agent run, per complexity
420#[derive(Clone, Copy, Debug, PartialEq, Eq)]
421pub struct RunTimeouts {
422    pub base: u64,
423    pub medium: u64,
424    pub complex: u64,
425}
426
427impl Default for RunTimeouts {
428    fn default() -> Self {
429        Self {
430            base: 1800,
431            medium: 1800,
432            complex: 1800,
433        }
434    }
435}
436
437impl RunTimeouts {
438    /// `workflow.timeout_seconds` (at least 1800), raised by
439    /// `workflow.timeout_by_complexity.{medium,complex}` but never below the base
440    pub fn from_config() -> Self {
441        let read =
442            |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
443        let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
444        Self {
445            base,
446            medium: read("workflow.timeout_by_complexity.medium")
447                .unwrap_or(base)
448                .max(base),
449            complex: read("workflow.timeout_by_complexity.complex")
450                .unwrap_or(base)
451                .max(base),
452        }
453    }
454
455    pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
456        match complexity {
457            Some("complex") => self.complex,
458            Some("medium") => self.medium,
459            _ => self.base,
460        }
461    }
462}
463
464/// Engine configuration.
465#[derive(Clone, Debug)]
466pub struct EngineConfig {
467    /// Main tick interval
468    pub tick_interval: Duration,
469    /// GitHub sync interval (cleanup, PR review, mentions)
470    pub sync_interval: Duration,
471    /// Webhook health check interval (seconds)
472    pub webhook_health_check_interval: Option<Duration>,
473    /// Maximum parallel task executions
474    pub max_parallel: usize,
475    /// Stuck task timeout for tasks with an active tmux session (seconds)
476    pub stuck_timeout: u64,
477    /// Stuck task timeout for tasks with no active tmux session (seconds).
478    /// Shorter than `stuck_timeout` because no session means the agent has already exited.
479    pub no_session_stuck_timeout: u64,
480    /// Stuck task timeout for in_review tasks with no active tmux session (seconds).
481    /// Longer than `no_session_stuck_timeout` because a review agent ends its tmux session on
482    /// normal completion before its result reaches the engine. A shorter limit resets a task
483    /// whose finished review is still being delivered, and the result is dropped as stale.
484    /// `from_config` keeps it above the review agent's own timeout plus
485    /// `review::REVIEW_RESULT_DELIVERY_SECS`.
486    pub in_review_no_session_stuck_timeout: u64,
487    /// Per-run agent timeouts. A live session is not stuck before its own timeout runs out
488    pub run_timeouts: RunTimeouts,
489    /// Auto-create follow-up tasks when PR reviews request changes
490    pub auto_create_followup_on_changes: bool,
491    /// Graceful shutdown timeout — how long to wait for running agents before exiting.
492    pub graceful_shutdown_timeout: Duration,
493    /// Grace period before silence detection kicks in (seconds).
494    /// Allows time for agent startup, model download, API handshake.
495    pub silence_grace_period: u64,
496    /// Cooldown duration for a model detected as silent (seconds).
497    pub silence_cooldown: u64,
498    /// How often to check for a newer orch release and notify channels (seconds).
499    /// Set to 0 to disable. Default: 3600 (1 hour).
500    pub upgrade_check_interval: u64,
501    /// Seconds of continuous startup failure (GitHub unreachable / auth errors)
502    /// before the engine escalates from WARN to ERROR-level logging and sends a
503    /// one-time push notification to configured channels. Set to 0 to disable.
504    /// Default: 3600 (1 hour).
505    pub startup_failure_escalation_secs: u64,
506}
507
508impl Default for EngineConfig {
509    fn default() -> Self {
510        Self {
511            tick_interval: Duration::from_secs(10),
512            sync_interval: Duration::from_secs(45),
513            webhook_health_check_interval: Some(Duration::from_secs(60)),
514            max_parallel: 4,
515            stuck_timeout: 1800,
516            no_session_stuck_timeout: 600,
517            in_review_no_session_stuck_timeout: 1920,
518            run_timeouts: RunTimeouts::default(),
519            auto_create_followup_on_changes: true,
520            graceful_shutdown_timeout: Duration::from_secs(600),
521            silence_grace_period: 300,
522            silence_cooldown: 3600,
523            upgrade_check_interval: 3600,
524            startup_failure_escalation_secs: 3600,
525        }
526    }
527}
528
529impl EngineConfig {
530    /// Stuck limit for an in_progress task whose session is alive: `stuck_timeout`, raised to the
531    /// run's own timeout plus `RUN_TIMEOUT_GRACE_SECS`
532    pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
533        self.stuck_timeout.max(
534            self.run_timeouts
535                .for_complexity(complexity)
536                .saturating_add(RUN_TIMEOUT_GRACE_SECS),
537        )
538    }
539
540    /// Load engine configuration from config files, falling back to defaults.
541    pub fn from_config() -> Self {
542        let mut config = Self::default();
543
544        if let Ok(val) = config::get("engine.tick_interval") {
545            if let Ok(secs) = val.parse::<u64>() {
546                config.tick_interval = Duration::from_secs(secs);
547            } else {
548                tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
549            }
550        }
551
552        if let Ok(val) = config::get("engine.sync_interval") {
553            if let Ok(secs) = val.parse::<u64>() {
554                config.sync_interval = Duration::from_secs(secs);
555            } else {
556                tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
557            }
558        }
559
560        if let Ok(val) = config::get("engine.max_parallel") {
561            if let Ok(n) = val.parse::<usize>() {
562                config.max_parallel = n;
563            } else {
564                tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
565            }
566        }
567
568        if let Ok(val) = config::get("engine.stuck_timeout") {
569            if let Ok(secs) = val.parse::<u64>() {
570                config.stuck_timeout = secs;
571            } else {
572                tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
573            }
574        }
575
576        if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
577            if let Ok(secs) = val.parse::<u64>() {
578                config.no_session_stuck_timeout = secs;
579            } else {
580                tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
581            }
582        }
583
584        let mut in_review_from_config = false;
585        if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
586            if let Ok(secs) = val.parse::<u64>() {
587                config.in_review_no_session_stuck_timeout = secs;
588                in_review_from_config = true;
589            } else {
590                tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
591            }
592        }
593
594        // Keep the no-session review limit over the review agent's own timeout plus delivery
595        // time, whatever either is set to. Under that, recovery resets a task whose finished
596        // review is still on its way to the engine and the result is dropped as stale
597        let review_timeout = engine_review::review_timeout_seconds();
598        let floored = engine_review::no_session_review_floor(
599            config.in_review_no_session_stuck_timeout,
600            review_timeout,
601        );
602        if floored != config.in_review_no_session_stuck_timeout {
603            if in_review_from_config {
604                tracing::warn!(
605                    configured_secs = config.in_review_no_session_stuck_timeout,
606                    review_timeout_secs = review_timeout,
607                    raised_to_secs = floored,
608                    "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
609                );
610            } else {
611                tracing::debug!(
612                    review_timeout_secs = review_timeout,
613                    raised_to_secs = floored,
614                    "raised the default in_review no-session limit over a long review timeout"
615                );
616            }
617            config.in_review_no_session_stuck_timeout = floored;
618        }
619
620        config.run_timeouts = RunTimeouts::from_config();
621
622        if let Ok(val) = config::get("engine.webhook_health_check_interval") {
623            if let Ok(secs) = val.parse::<u64>() {
624                if secs > 0 {
625                    config.webhook_health_check_interval = Some(Duration::from_secs(secs));
626                } else {
627                    config.webhook_health_check_interval = None;
628                }
629            } else {
630                tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
631            }
632        }
633
634        if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
635            config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
636        }
637
638        if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
639            if let Ok(secs) = val.parse::<u64>() {
640                config.graceful_shutdown_timeout = Duration::from_secs(secs);
641            } else {
642                tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
643            }
644        }
645
646        if let Ok(val) = config::get("engine.silence_grace_period") {
647            if let Ok(secs) = val.parse::<u64>() {
648                config.silence_grace_period = secs;
649            } else {
650                tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
651            }
652        }
653
654        if let Ok(val) = config::get("engine.silence_cooldown") {
655            if let Ok(secs) = val.parse::<u64>() {
656                config.silence_cooldown = secs;
657            } else {
658                tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
659            }
660        }
661
662        if let Ok(val) = config::get("engine.upgrade_check_interval") {
663            if let Ok(secs) = val.parse::<u64>() {
664                config.upgrade_check_interval = secs;
665            } else {
666                tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
667            }
668        }
669
670        if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
671            if let Ok(secs) = val.parse::<u64>() {
672                config.startup_failure_escalation_secs = secs;
673            } else {
674                tracing::warn!(
675                    key = "engine.startup_failure_escalation_secs",
676                    value = %val,
677                    default_secs = config.startup_failure_escalation_secs,
678                    "invalid value for engine.startup_failure_escalation_secs, using default"
679                );
680            }
681        }
682
683        config
684    }
685}
686
687/// KV key holding the epoch second the GitHub core window resets, so `orch task status`
688/// can show the limit from a separate process.
689pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
690
691/// Format an epoch second as a local `HH:MM:SS` for operator-facing output.
692pub fn format_rate_limited_until(reset_at: i64) -> String {
693    chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
694        || reset_at.to_string(),
695        |t| {
696            t.with_timezone(&chrono::Local)
697                .format("%H:%M:%S %Z")
698                .to_string()
699        },
700    )
701}
702
703/// Announce a GitHub rate-limit window on every operator surface: the log (`orch log`),
704/// a KV flag (`orch task status`) and the channels (Telegram, Discord).
705async fn report_rate_limited(
706    reset_at: i64,
707    store: Option<&Arc<TaskStore>>,
708    transport: Option<&Arc<Transport>>,
709) {
710    let m = rate_limit_metrics();
711    let until = format_rate_limited_until(reset_at);
712    tracing::warn!(
713        reset_at,
714        rate_limit_hits = m.hits,
715        skipped_calls = m.skipped_calls,
716        five_xx_hits = m.five_xx_hits,
717        rest_remaining = ?m.rest_remaining,
718        graphql_remaining = ?m.graphql_remaining,
719        "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
720    );
721
722    if let Some(store) = store
723        && let Err(e) = store
724            .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
725            .await
726    {
727        tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
728    }
729
730    if let Some(transport) = transport {
731        transport.push_notification(TaskNotification {
732            task_id: "github".to_string(),
733            title: "GitHub rate limit".to_string(),
734            status: "blocked".to_string(),
735            agent: "orch".to_string(),
736            duration_seconds: 0.0,
737            summary: format!(
738                "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
739            ),
740            repo: None,
741            notify_target: None,
742            pr_number: None,
743        });
744    }
745}
746
747/// Grow or shrink the dispatch pool toward `target` without cancelling running work.
748///
749/// Growing adds permits at once. Shrinking can only take back the permits that
750/// are free right now, so the return value is the limit actually in force; call
751/// again as running tasks release theirs
752fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
753    if target > in_force {
754        semaphore.add_permits(target - in_force);
755        return target;
756    }
757
758    if target < in_force {
759        return in_force - semaphore.forget_permits(in_force - target);
760    }
761
762    in_force
763}
764
765/// Config subtrees the engine only reads at startup: the channel gateways hold
766/// long-lived websockets and the webhook listener binds its port once.
767const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
768
769/// Read the restart-only subtrees so a later reload can tell they changed.
770fn restart_only_snapshot() -> Vec<Option<String>> {
771    RESTART_ONLY_KEYS
772        .iter()
773        .map(|key| config::get(key).ok())
774        .collect()
775}
776
777/// Report the restart-only keys an edit changed. Only the key is logged, never
778/// the value, because `channels.*` holds bot tokens
779fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
780    let changed: Vec<&'static str> = RESTART_ONLY_KEYS
781        .iter()
782        .zip(old)
783        .zip(new)
784        .filter(|((_, old), new)| old != new)
785        .map(|((key, _), _)| *key)
786        .collect();
787
788    for key in &changed {
789        tracing::info!(key = key, "config changed, takes effect after restart");
790    }
791
792    changed
793}
794
795/// Log every engine setting whose value changed across a config reload, so
796/// `orch log` shows what the operator's edit actually did.
797fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
798    macro_rules! log_changed {
799        ($($key:literal => $field:ident),+ $(,)?) => {
800            $(if old.$field != new.$field {
801                tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
802            })+
803        };
804    }
805
806    log_changed!(
807        "engine.tick_interval" => tick_interval,
808        "engine.sync_interval" => sync_interval,
809        "engine.webhook_health_check_interval" => webhook_health_check_interval,
810        "engine.max_parallel" => max_parallel,
811        "engine.stuck_timeout" => stuck_timeout,
812        "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
813        "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
814        "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
815        "engine.silence_grace_period" => silence_grace_period,
816        "engine.silence_cooldown" => silence_cooldown,
817        "engine.upgrade_check_interval" => upgrade_check_interval,
818        "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
819        "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
820    );
821}
822
823fn should_run_upgrade_check(
824    last_upgrade_check: Option<Instant>,
825    upgrade_check_interval: u64,
826) -> bool {
827    if upgrade_check_interval == 0 {
828        return false;
829    }
830
831    last_upgrade_check
832        .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
833}
834
835/// Connect one project's backend and build its task manager and runner over
836/// the shared store.
837///
838/// `event_tx` wires status transitions into the event bus; pass `None` during
839/// startup, where the bus does not exist yet and task managers are rebuilt
840/// once it does
841async fn connect_project_engine(
842    repo: String,
843    project_dir: PathBuf,
844    store: Arc<TaskStore>,
845    event_tx: Option<Sender<events::TaskEvent>>,
846) -> anyhow::Result<ProjectEngine> {
847    let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
848
849    // A failed health check (rate limit, network, auth) must not drop the project: it would have
850    // no review subscriber until the next restart. Later ticks hit the API again and retry on
851    // their own, so build the engine and warn while the check keeps failing.
852    match backend.health_check().await {
853        Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
854        Err(e) => tracing::warn!(
855            repo = %repo,
856            error = %e,
857            "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
858        ),
859    }
860
861    let task_manager = Arc::new(match event_tx {
862        Some(tx) => {
863            TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
864        }
865        None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
866    });
867    let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
868
869    Ok(ProjectEngine {
870        repo,
871        project_dir,
872        backend,
873        task_manager,
874        runner,
875        store,
876    })
877}
878
879/**
880 * Split the configured project list into the ones not running yet and the names of the ones that
881 * went away. Kept separate from `sync_project_engines` so the comparison can be tested without a
882 * GitHub backend: getting it wrong either spawns a second set of subscribers for a project already
883 * running, or silently never picks a new one up
884 **/
885fn diff_configured_projects(
886    configured: &[(String, PathBuf)],
887    active_repos: &HashSet<String>,
888) -> (Vec<(String, PathBuf)>, Vec<String>) {
889    let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
890
891    let added = configured
892        .iter()
893        .filter(|(repo, _)| !active_repos.contains(repo))
894        .cloned()
895        .collect();
896    let removed = active_repos
897        .iter()
898        .filter(|repo| !configured_repos.contains(repo.as_str()))
899        .cloned()
900        .collect();
901
902    (added, removed)
903}
904
905/// Pick up projects added to `projects:` since startup, and report the ones
906/// that were removed.
907///
908/// A removed project keeps running until the next restart: its subscriber
909/// tasks and any agent it already dispatched outlive the config read
910#[expect(
911    clippy::too_many_arguments,
912    reason = "a new project engine needs every shared engine handle"
913)]
914async fn sync_project_engines(
915    project_engines: &mut Vec<ProjectEngine>,
916    active_repos: &mut HashSet<String>,
917    event_bus: &events::EventBus,
918    tmux: &Arc<TmuxManager>,
919    capture: &Arc<CaptureService>,
920    semaphore: &Arc<Semaphore>,
921    review_semaphore: &Arc<Semaphore>,
922    weight_tx: &mpsc::Sender<WeightSignal>,
923    router: &Arc<RwLock<Router>>,
924    dispatching: &Arc<dashmap::DashMap<String, String>>,
925) {
926    let configured = match config::get_projects_with_paths() {
927        Ok(p) if !p.is_empty() => p,
928        Ok(_) => return,
929        Err(e) => {
930            tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
931            return;
932        }
933    };
934    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
935        return;
936    };
937
938    let (added, removed) = diff_configured_projects(&configured, active_repos);
939    for repo in removed {
940        tracing::info!(
941            repo = %repo,
942            "project removed from config, takes effect after restart"
943        );
944    }
945
946    for (repo, project_dir) in added {
947        config::watch(&project_dir.join(".orch.yml"));
948        let engine = match connect_project_engine(
949            repo.clone(),
950            project_dir,
951            Arc::clone(&store),
952            Some(event_bus.sender()),
953        )
954        .await
955        {
956            Ok(engine) => engine,
957            Err(e) => {
958                // A failed health check no longer lands here, so this is a backend that could
959                // not be built at all, such as a repo slug that is not `owner/name`
960                tracing::warn!(
961                    repo = %repo,
962                    err = %e,
963                    "project added to config but its backend could not be built, retrying on the next config change"
964                );
965                continue;
966            }
967        };
968
969        subscribers::dispatch::spawn(
970            event_bus.subscribe(),
971            Arc::clone(&engine.backend),
972            Arc::clone(tmux),
973            Arc::clone(&engine.runner),
974            Arc::clone(capture),
975            Arc::clone(semaphore),
976            Arc::clone(&engine.task_manager),
977            weight_tx.clone(),
978            Arc::clone(router),
979            Arc::clone(dispatching),
980            Arc::clone(&engine.store),
981            engine.repo.clone(),
982        );
983        subscribers::review::spawn(
984            event_bus.subscribe(),
985            Arc::clone(&engine.backend),
986            Arc::clone(tmux),
987            Arc::clone(review_semaphore),
988            Arc::clone(&engine.task_manager),
989            Arc::clone(router),
990            Arc::clone(dispatching),
991            Arc::clone(&engine.store),
992            engine.repo.clone(),
993        );
994        subscribers::unblock::spawn(
995            event_bus.subscribe(),
996            Arc::clone(&engine.backend),
997            Arc::clone(&engine.task_manager),
998            Arc::clone(&engine.store),
999            engine.repo.clone(),
1000        );
1001
1002        active_repos.insert(repo.clone());
1003        project_engines.push(engine);
1004        tracing::info!(repo = %repo, "project added from config, now syncing");
1005    }
1006}
1007
1008/// Initialize all project engines from config.
1009///
1010/// Returns a vector of ProjectEngine, one for each configured project.
1011async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1012    let projects = config::get_projects_with_paths()?;
1013    let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1014    tracing::info!(repos = ?repos, "loading projects from config");
1015
1016    if projects.is_empty() {
1017        let config_path = home::config_path().map_or_else(
1018            |_| "~/.orch/config.yml".to_string(),
1019            |p| p.display().to_string(),
1020        );
1021        anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1022    }
1023
1024    let mut engines = Vec::new();
1025
1026    // Initialize unified task store once — all project engines share the same SQLite file.
1027    // Creating one pool here avoids N separate connection pools for the same database.
1028    let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1029    if cluster::enabled() {
1030        cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1031        local::serve((*store).clone()).await?;
1032    }
1033
1034    // Load persisted model cooldowns and register store for future writes.
1035    // Runs once here so the global cooldown store is not overwritten per project.
1036    engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1037
1038    for (repo, project_dir) in projects {
1039        tracing::info!(repo = %repo, "initializing project engine");
1040        config::watch(&project_dir.join(".orch.yml"));
1041
1042        engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1043    }
1044
1045    Ok(engines)
1046}
1047
1048async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1049    for engine in project_engines {
1050        let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1051        let repo_root_path = PathBuf::from(&repo_root);
1052        let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1053        let worktrees = list_project_worktrees(&repo_root_path).await?;
1054
1055        // Fetch refs once for all worktrees in this project, so rebase below
1056        // operates on up-to-date origin/* refs without N sequential fetches.
1057        if let Err(e) = git_ops::git()
1058            .await
1059            .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1060            .output_with_context()
1061            .await
1062        {
1063            tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1064        }
1065
1066        let mut orphans_removed: u32 = 0;
1067        let mut invalid_removed: u32 = 0;
1068        let mut cleaned: u32 = 0;
1069        let mut valid_kept: u32 = 0;
1070
1071        for worktree_dir in worktrees {
1072            let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1073                continue;
1074            };
1075
1076            // Validate gitdir link before doing anything else — a broken .git file
1077            // means git commands on this worktree will fail with "not a git repository".
1078            if !validate_worktree_gitdir(&worktree_dir).await {
1079                tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1080                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1081                    .await;
1082                invalid_removed += 1;
1083                continue;
1084            }
1085
1086            let Some(task_id) = task_id_from_worktree_name(name) else {
1087                tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1088                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1089                    .await;
1090                orphans_removed += 1;
1091                continue;
1092            };
1093
1094            let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1095                tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1096                remove_worktree_and_branch(
1097                    &task_id,
1098                    &worktree_dir,
1099                    Some(name),
1100                    &repo_root_path,
1101                    false,
1102                )
1103                .await;
1104                orphans_removed += 1;
1105                continue;
1106            };
1107
1108            let task = engine.store.get(store_id).await?;
1109            let branch_name = if task.branch.is_empty() {
1110                name
1111            } else {
1112                task.branch.as_str()
1113            };
1114
1115            match task.status {
1116                TaskStatus::New
1117                | TaskStatus::Routed
1118                | TaskStatus::InProgress
1119                | TaskStatus::NeedsReview
1120                | TaskStatus::InReview => {
1121                    let keep_conflict = worktree::keep_conflicted_rebase(
1122                        task.merge_conflict_retries,
1123                        task.pr_number,
1124                    );
1125                    if keep_conflict && git_ops::rebase_in_progress(&worktree_dir).await {
1126                        tracing::info!(
1127                            repo = %engine.repo,
1128                            task_id = %task_id,
1129                            worktree = %worktree_dir.display(),
1130                            "keeping the conflicted rebase auto_merge left for the agent"
1131                        );
1132                        valid_kept += 1;
1133                        continue;
1134                    }
1135                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1136                    let origin_branch = format!("origin/{default_branch}");
1137                    let outcome = git_ops::stash_rebase_restore(
1138                        &worktree_dir,
1139                        &origin_branch,
1140                        git_ops::StashRebaseOpts {
1141                            abort_on_failure: false,
1142                        },
1143                    )
1144                    .await;
1145                    let conflict_left = git_ops::rebase_in_progress(&worktree_dir).await;
1146                    let failed_err = match outcome {
1147                        Ok(
1148                            git_ops::RebaseOutcome::Succeeded | git_ops::RebaseOutcome::Skipped(_),
1149                        ) => None,
1150                        Ok(git_ops::RebaseOutcome::Failed(e)) if keep_conflict && conflict_left => {
1151                            tracing::info!(
1152                                repo = %engine.repo,
1153                                task_id = %task_id,
1154                                err = %e,
1155                                "startup rebase hit a content conflict, leaving it for the agent"
1156                            );
1157                            None
1158                        }
1159                        Ok(git_ops::RebaseOutcome::Failed(e)) => Some(e),
1160                        Err(e) => Some(e.to_string()),
1161                    };
1162                    if let Some(e) = failed_err {
1163                        reset_startup_task_after_failed_rebase(
1164                            engine,
1165                            &task_id,
1166                            &worktree_dir,
1167                            branch_name,
1168                            &repo_root_path,
1169                            task.pr_number.is_some(),
1170                            &e,
1171                        )
1172                        .await;
1173                        cleaned += 1;
1174                    } else {
1175                        valid_kept += 1;
1176                    }
1177                }
1178                // A blocked or waiting task resumes from this work, and without a PR the branch
1179                // may hold the only copy of it. An approved task's PR is updated on GitHub
1180                TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1181                    valid_kept += 1;
1182                }
1183                TaskStatus::Done => {
1184                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1185                    remove_worktree_and_branch_for(
1186                        &task_id,
1187                        &worktree_dir,
1188                        Some(branch_name),
1189                        &repo_root_path,
1190                        false,
1191                        task.pr_number.is_some(),
1192                    )
1193                    .await;
1194                    cleaned += 1;
1195                }
1196            }
1197        }
1198
1199        tracing::info!(
1200            repo = %engine.repo,
1201            orphans_removed,
1202            invalid_removed,
1203            cleaned,
1204            valid_kept,
1205            "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1206        );
1207
1208        // Prune stale .git/worktrees/ entries whose directories no longer exist.
1209        // This handles entries left behind by crashes or manual directory removal.
1210        let prune = Command::new("git")
1211            .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1212            .output_with_context()
1213            .await;
1214        match prune {
1215            Ok(output) if output.status.success() => {
1216                tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1217            }
1218            Ok(output) => {
1219                let stderr = String::from_utf8_lossy(&output.stderr);
1220                tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1221            }
1222            Err(e) => {
1223                tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1224            }
1225        }
1226    }
1227
1228    Ok(())
1229}
1230
1231async fn reset_startup_task_after_failed_rebase(
1232    engine: &ProjectEngine,
1233    task_id: &str,
1234    worktree_dir: &Path,
1235    branch_name: &str,
1236    repo_root: &Path,
1237    keep_remote: bool,
1238    err: &str,
1239) {
1240    tracing::warn!(repo = %engine.repo, task_id, err, "startup rebase failed, resetting task");
1241    abort_worktree_rebase(worktree_dir).await;
1242    remove_worktree_and_branch(
1243        task_id,
1244        worktree_dir,
1245        Some(branch_name),
1246        repo_root,
1247        keep_remote,
1248    )
1249    .await;
1250    let keep_branch = local_branch_exists(repo_root, branch_name).await;
1251    match engine.store.resolve_task_id(&engine.repo, task_id).await {
1252        Ok(Some(reset_id)) => {
1253            let result = if keep_branch {
1254                engine.store.reset_to_new_keep_branch(reset_id).await
1255            } else {
1256                engine.store.reset_to_new(reset_id).await
1257            };
1258            if let Err(e) = result {
1259                tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1260            }
1261        }
1262        Ok(None) => {
1263            tracing::warn!(task_id, "task not found in store during reconciliation");
1264        }
1265        Err(e) => {
1266            tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1267        }
1268    }
1269    if let Err(e) = engine
1270        .task_manager
1271        .update_task_status(&ExternalId(task_id.to_string()), Status::New)
1272        .await
1273    {
1274        tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1275    }
1276}
1277
1278/// Sync Fibonacci estimates stored in SQLite to the GitHub Projects board for
1279/// all tasks that have a positive estimate.
1280///
1281/// Runs once on startup to catch tasks that were routed before the estimate
1282/// sync feature was available or before `project_estimate_field_id` was
1283/// configured. The underlying mutations are idempotent, so re-syncing an
1284/// already-correct estimate is harmless.
1285async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1286    for engine in project_engines {
1287        let tasks = match engine.store.list_external_tasks_with_estimates().await {
1288            Ok(t) => t,
1289            Err(e) => {
1290                tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1291                continue;
1292            }
1293        };
1294
1295        if tasks.is_empty() {
1296            continue;
1297        }
1298
1299        let mut synced: u32 = 0;
1300        let mut failed: u32 = 0;
1301        for (external_id, estimate) in &tasks {
1302            match engine
1303                .backend
1304                .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1305                .await
1306            {
1307                Ok(()) => synced += 1,
1308                Err(e) => {
1309                    tracing::debug!(
1310                        repo = %engine.repo,
1311                        task_id = %external_id,
1312                        estimate,
1313                        err = %e,
1314                        "startup estimate reconciliation: sync failed"
1315                    );
1316                    failed += 1;
1317                }
1318            }
1319        }
1320
1321        if synced > 0 || failed > 0 {
1322            tracing::info!(
1323                repo = %engine.repo,
1324                synced,
1325                failed,
1326                "startup estimate reconciliation complete"
1327            );
1328        }
1329    }
1330}
1331
1332/// Read per-project channel configuration from `.orch.yml`.
1333fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1334    let config_path = project_dir.join(".orch.yml");
1335    if !config_path.exists() {
1336        return ProjectChannelConfig::default();
1337    }
1338    let content = match fs::read_to_string(&config_path) {
1339        Ok(c) => c,
1340        Err(e) => {
1341            tracing::warn!(
1342                path = %config_path.display(),
1343                error = %e,
1344                "failed to read .orch.yml; channel config will use defaults"
1345            );
1346            return ProjectChannelConfig::default();
1347        }
1348    };
1349    let val: serde_norway::Value = match serde_norway::from_str(&content) {
1350        Ok(v) => v,
1351        Err(e) => {
1352            tracing::warn!(
1353                path = %config_path.display(),
1354                error = %e,
1355                "failed to parse .orch.yml; channel config will use defaults"
1356            );
1357            return ProjectChannelConfig::default();
1358        }
1359    };
1360    let Some(channels) = val.get("channels") else {
1361        return ProjectChannelConfig::default();
1362    };
1363    let get_str = |section: &str, key: &str| -> Option<String> {
1364        channels
1365            .get(section)
1366            .and_then(|s| s.get(key))
1367            .and_then(|v| v.as_str())
1368            .map(String::from)
1369    };
1370    ProjectChannelConfig {
1371        telegram_topic_id: get_str("telegram", "topic_id"),
1372        discord_channel_id: get_str("discord", "channel_id"),
1373    }
1374}
1375
1376/// Fetch the latest orch release tag from GitHub via the `gh` CLI (async).
1377///
1378/// Returns the version string (without the leading "v"), or `None` when the
1379/// check fails (no network, unauthenticated, `gh` not installed, etc.).
1380async fn fetch_latest_release_version() -> Option<String> {
1381    let output = Command::new("gh")
1382        .args([
1383            "api",
1384            "repos/gabrielkoerich/orch/releases/latest",
1385            "--jq",
1386            ".tag_name",
1387        ])
1388        .output()
1389        .await
1390        .ok()?;
1391    if !output.status.success() {
1392        return None;
1393    }
1394    let tag = String::from_utf8(output.stdout).ok()?;
1395    let version = tag.trim().trim_start_matches('v').to_string();
1396    if version.is_empty() {
1397        None
1398    } else {
1399        Some(version)
1400    }
1401}
1402
1403/// Check if a newer orch release is available and send channel notifications
1404/// asking the operator to upgrade manually. Upgrading is operator-only — the
1405/// service never runs `brew` itself.
1406///
1407/// Uses the store's KV to throttle checks (`upgrade:last_check_at`) and
1408/// deduplicate notifications (`upgrade:last_notified_version`). Each distinct
1409/// latest version is notified about once; the notification repeats only when
1410/// an even newer version is released.
1411async fn check_and_notify_upgrade(
1412    store: &Arc<TaskStore>,
1413    channels: &Arc<ChannelRegistry>,
1414    current_version: &str,
1415    check_interval: Duration,
1416) {
1417    let last_check_key = "upgrade:last_check_at";
1418    let last_notified_key = "upgrade:last_notified_version";
1419
1420    // Throttle: only check once per interval.
1421    let now = chrono::Utc::now().timestamp();
1422    if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1423        && let Ok(last_check) = last_check_str.parse::<i64>()
1424        && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1425    {
1426        return;
1427    }
1428
1429    // Record check timestamp (best-effort).
1430    let _ = store.kv_set(last_check_key, &now.to_string()).await;
1431
1432    let Some(latest) = fetch_latest_release_version().await else {
1433        tracing::debug!("upgrade check: failed to fetch latest release version");
1434        return;
1435    };
1436
1437    if latest == current_version {
1438        tracing::debug!(current = %current_version, "orch is up to date");
1439        return;
1440    }
1441
1442    tracing::warn!(
1443        current_version = %current_version,
1444        latest_version = %latest,
1445        "orch upgrade available"
1446    );
1447
1448    // Operator-only upgrade: send channel notifications so the operator can
1449    // upgrade on their own schedule. The service never runs brew itself.
1450
1451    // Deduplicate: don't re-notify for the same latest version.
1452    if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1453        && last_notified == latest
1454    {
1455        tracing::debug!(latest = %latest, "upgrade already notified");
1456        return;
1457    }
1458
1459    let msg_telegram = format!(
1460        "⚠️ <b>Orch Upgrade Available</b>\n\n\
1461         Service: <code>{current_version}</code>\n\
1462         Latest:  <code>{latest}</code>\n\n\
1463         Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1464    );
1465
1466    let msg_discord = format!(
1467        "⚠️ **Orch Upgrade Available**\n\n\
1468         Service: `{current_version}`\n\
1469         Latest:  `{latest}`\n\n\
1470         Run: `brew update && brew upgrade orch && brew services restart orch`"
1471    );
1472
1473    let msg_slack = format!(
1474        "⚠️ *Orch Upgrade Available*\n\n\
1475         Service: `{current_version}`\n\
1476         Latest:  `{latest}`\n\n\
1477         Run: `brew update && brew upgrade orch && brew services restart orch`"
1478    );
1479
1480    for channel in channels.iter() {
1481        let (body, metadata, topic_id) = match channel.name() {
1482            "telegram" => (
1483                msg_telegram.clone(),
1484                serde_json::json!({"preformatted_html": true}),
1485                None,
1486            ),
1487            "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1488            "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1489            _ => continue,
1490        };
1491
1492        let msg = OutgoingMessage {
1493            thread_id: "upgrade".to_string(),
1494            body,
1495            reply_to: None,
1496            metadata,
1497            topic_id,
1498        };
1499
1500        if let Err(e) = channel.send(&msg).await {
1501            tracing::warn!(
1502                channel = channel.name(),
1503                ?e,
1504                "failed to send upgrade notification"
1505            );
1506        }
1507    }
1508
1509    // Record that we notified for this latest version.
1510    let _ = store.kv_set(last_notified_key, &latest).await;
1511}
1512
1513/// Send a one-time escalation notification when the startup retry loop has been
1514/// spinning for longer than `startup_failure_escalation_secs`.
1515///
1516/// This is called *before* the channel registry is initialised (the retry loop
1517/// sits above it in `serve()`), so we construct channel objects directly from
1518/// config — mirroring the approach used during normal channel registration, but
1519/// without relying on `ChannelRegistry`.
1520async fn send_startup_escalation_notification(elapsed_secs: u64) {
1521    #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1522    let duration = notification::format_duration(elapsed_secs as f64);
1523
1524    let msg_telegram = format!(
1525        "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1526         The engine has been retrying backend connections for {duration}\n\
1527         without success. All task processing, routing, and scheduled\n\
1528         jobs are paused. Check service logs with `orch log -f`."
1529    );
1530
1531    let msg_discord = format!(
1532        "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1533         The engine has been retrying backend connections for {duration}\n\
1534         without success. All task processing, routing, and scheduled\n\
1535         jobs are paused. Check service logs with `orch log -f`."
1536    );
1537
1538    let msg_slack = format!(
1539        "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1540         The engine has been retrying backend connections for {duration}\n\
1541         without success. All task processing, routing, and scheduled\n\
1542         jobs are paused. Check service logs with `orch log -f`."
1543    );
1544
1545    // Telegram
1546    let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1547    let tg_chat_id = config::get("channels.telegram.chat_id");
1548    if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1549        && !token.is_empty()
1550        && !chat_id.is_empty()
1551        && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1552    {
1553        let msg = OutgoingMessage {
1554            thread_id: "startup_escalation".to_string(),
1555            body: msg_telegram.clone(),
1556            reply_to: None,
1557            metadata: serde_json::json!({ "preformatted_html": true }),
1558            topic_id: config::get("channels.telegram.general_topic_id").ok(),
1559        };
1560        match channel.send(&msg).await {
1561            Err(e) => {
1562                tracing::warn!(?e, "failed to send startup escalation via telegram");
1563            }
1564            _ => {
1565                tracing::info!("startup escalation notification sent to telegram");
1566            }
1567        }
1568    }
1569
1570    // Discord
1571    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1572        let channel_id = config::get("channels.discord.channel_id")
1573            .or_else(|_| config::get("channels.discord.general_channel_id"));
1574        if !token.is_empty()
1575            && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1576        {
1577            let shard_id = config::get("channels.discord.shard_id")
1578                .ok()
1579                .and_then(|v| v.parse::<u64>().ok())
1580                .unwrap_or(0);
1581            let shard_count = config::get("channels.discord.shard_count")
1582                .ok()
1583                .and_then(|v| v.parse::<u64>().ok())
1584                .unwrap_or(1);
1585            if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1586                let msg = OutgoingMessage {
1587                    thread_id: "startup_escalation".to_string(),
1588                    body: msg_discord.clone(),
1589                    reply_to: None,
1590                    metadata: serde_json::Value::Null,
1591                    topic_id: None,
1592                };
1593                match gateway.send(&msg).await {
1594                    Err(e) => {
1595                        tracing::warn!(?e, "failed to send startup escalation via discord");
1596                    }
1597                    _ => {
1598                        tracing::info!("startup escalation notification sent to discord");
1599                    }
1600                }
1601            }
1602        }
1603    }
1604
1605    // Slack
1606    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1607        let channel_id = config::get("channels.slack.channel_id");
1608        if !token.is_empty()
1609            && let Ok(cid) = channel_id
1610            && let Ok(channel) = SlackChannel::new(token, Some(cid))
1611        {
1612            let msg = OutgoingMessage {
1613                thread_id: "startup_escalation".to_string(),
1614                body: msg_slack.clone(),
1615                reply_to: None,
1616                metadata: serde_json::Value::Null,
1617                topic_id: None,
1618            };
1619            match channel.send(&msg).await {
1620                Err(e) => {
1621                    tracing::warn!(?e, "failed to send startup escalation via slack");
1622                }
1623                _ => {
1624                    tracing::info!("startup escalation notification sent to slack");
1625                }
1626            }
1627        }
1628    }
1629}
1630
1631/// Start the orch service.
1632///
1633/// This is the main entry point — called by `orch serve`.
1634pub async fn serve() -> anyhow::Result<()> {
1635    let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1636    if killed > 0 {
1637        tracing::info!(count = killed, "terminated stale orch serve processes");
1638    }
1639
1640    tracing::info!("orch engine starting");
1641
1642    // Pre-create standard directories with async I/O so subsequent synchronous
1643    // callers (state_dir, orch_home, etc.) find them already present and their
1644    // create_dir_all calls become fast no-ops.
1645    if let Err(e) = home::init_dirs().await {
1646        tracing::warn!("failed to initialize orch directories: {e}");
1647    }
1648
1649    // Pre-warm the config cache with async I/O so get() / get_list() calls in
1650    // hot async paths never block a Tokio thread on std::fs::read_to_string.
1651    if let Err(e) = config::warm_cache().await {
1652        tracing::warn!("failed to warm config cache: {e}");
1653    }
1654
1655    let mut config = EngineConfig::from_config();
1656
1657    tracing::info!("internal database ready");
1658
1659    // Initialize project engines — retry with backoff so a network outage at
1660    // startup doesn't cause a crash-loop (launchd KeepAlive would restart us
1661    // immediately, filling the error log with 600+ identical messages).
1662    //
1663    // IMPORTANT: errors from init_project_engines() must NEVER propagate past
1664    // this loop to main().  If they did, anyhow would write "Error: …" to stderr,
1665    // which brew routes to orch.error.log — polluting it even when projects ARE
1666    // configured correctly but GitHub is temporarily unreachable.
1667    let startup_escalation_secs = config.startup_failure_escalation_secs;
1668    let mut project_engines = {
1669        let mut delay_secs = 5u64;
1670        let mut attempt = 0u32;
1671        let mut escalated = false;
1672        let start = Instant::now();
1673        loop {
1674            attempt += 1;
1675            match init_project_engines().await {
1676                Ok(engines) => break engines,
1677                Err(e) => {
1678                    let elapsed = start.elapsed().as_secs();
1679
1680                    // Short blips (under the escalation threshold) stay at WARN
1681                    // so they don't flood the operator's attention. Once the
1682                    // outage crosses the threshold, escalate to ERROR-level
1683                    // logging and send a one-time push notification to configured
1684                    // channels — "transient" has clearly become "prolonged."
1685                    if startup_escalation_secs > 0
1686                        && !escalated
1687                        && elapsed >= startup_escalation_secs
1688                    {
1689                        escalated = true;
1690                        #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1691                        let outage = notification::format_duration(elapsed as f64);
1692                        tracing::error!(
1693                            delay_secs,
1694                            attempt,
1695                            elapsed_secs = elapsed,
1696                            threshold_secs = startup_escalation_secs,
1697                            "project backends unavailable for {outage} — escalating startup failure: {e}",
1698                        );
1699                        // Send one-time notification to configured channels.
1700                        // Channels aren't registered yet at this stage, so we
1701                        // construct them directly from config (mirrors orch notify).
1702                        send_startup_escalation_notification(elapsed).await;
1703                    } else {
1704                        tracing::warn!(
1705                            delay_secs,
1706                            attempt,
1707                            "project backends unavailable, retrying: {e}"
1708                        );
1709                    }
1710
1711                    time::sleep(Duration::from_secs(delay_secs)).await;
1712                    delay_secs = (delay_secs * 2).min(120);
1713                }
1714            }
1715        }
1716    };
1717
1718    tracing::info!(
1719        project_count = project_engines.len(),
1720        "initialized project engines"
1721    );
1722
1723    // Full set of currently-configured project repos, used by the per-tick global
1724    // sweeps (e.g. `dispatch_routed_tasks_global`) to distinguish "orphaned" tasks
1725    // (repo no longer in config) from tasks belonging to some other active project.
1726    let mut active_repos: HashSet<String> =
1727        project_engines.iter().map(|e| e.repo.clone()).collect();
1728
1729    // Create the event bus for task status transitions
1730    let event_bus = events::EventBus::new(256);
1731    if let Err(e) = event_bus.start_ws_server().await {
1732        tracing::warn!(
1733            ?e,
1734            "failed to start event websocket server, continuing without it"
1735        );
1736    }
1737
1738    // Check for a newer orch release in the background and log if the service is behind.
1739    // The periodic checker sends channel notifications after channels are initialized;
1740    // upgrading is operator-only and never runs automatically.
1741    {
1742        let current = env!("ORCH_VERSION").to_string();
1743        tokio::spawn(async move {
1744            if let Some(latest) = fetch_latest_release_version().await
1745                && latest != current
1746            {
1747                tracing::warn!(
1748                    current_version = %current,
1749                    latest_version = %latest,
1750                    "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1751                );
1752            }
1753        });
1754    }
1755
1756    // Re-create task managers with shared store and event bus
1757    for engine in &mut project_engines {
1758        engine.task_manager = Arc::new(TaskManager::with_events(
1759            Arc::clone(&engine.backend),
1760            Arc::clone(&engine.store),
1761            engine.repo.clone(),
1762            event_bus.sender(),
1763        ));
1764    }
1765
1766    // Startup reconciliation decides task status, so only the leader runs it
1767    let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1768    if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1769        tracing::warn!(err = %e, "startup worktree reconciliation failed");
1770    }
1771
1772    if leads {
1773        // Run estimate reconciliation in the background so it never blocks startup.
1774        // Each GraphQL mutation takes ~0.8 s; with hundreds of tasks this would
1775        // otherwise delay the engine reaching its main loop by several minutes.
1776        let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1777        tokio::spawn(async move {
1778            reconcile_startup_estimates(&engines_snapshot).await;
1779        });
1780    }
1781
1782    // Build ChannelRouter from global config + per-project configs
1783    let global_channel_config = GlobalChannelConfig {
1784        telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1785        discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1786        control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1787        control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1788    };
1789    let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1790        .iter()
1791        .map(|e| {
1792            let cfg = read_project_channel_config(&e.project_dir);
1793            (e.repo.clone(), cfg)
1794        })
1795        .collect();
1796    let channel_router = Arc::new(ChannelRouter::new(
1797        &global_channel_config,
1798        &project_channel_configs,
1799    ));
1800    tracing::info!(
1801        projects = ?channel_router.projects(),
1802        "channel router initialized"
1803    );
1804
1805    // Initialize tmux manager (shared across all projects)
1806    let tmux = Arc::new(TmuxManager::new());
1807    // Kill leftover sessions from a previous process. This is daemon start only,
1808    // never a tick, because nextest makes a once-flag in tick_all kill live agents
1809    tick::startup_cleanup(&tmux).await;
1810
1811    // Initialize transport
1812    let transport = Arc::new(Transport::new());
1813
1814    // Initialize capture service and start background loop
1815    let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1816    let capture_for_tick = Arc::clone(&capture);
1817    tokio::spawn(async move {
1818        capture.start().await;
1819    });
1820
1821    // Initialize channel registry
1822    let mut channel_registry = ChannelRegistry::new();
1823
1824    // Try to initialize Telegram channel
1825    if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1826        && !token.is_empty()
1827    {
1828        let chat_id = config::get("channels.telegram.chat_id").ok();
1829        match TelegramChannel::new(token, chat_id) {
1830            Ok(telegram) => {
1831                if let Err(e) = telegram.health_check().await {
1832                    tracing::warn!(?e, "telegram channel health check failed, skipping");
1833                } else {
1834                    channel_registry.register(Box::new(telegram));
1835                    tracing::info!("telegram channel registered");
1836                }
1837            }
1838            Err(e) => {
1839                tracing::warn!(?e, "failed to create telegram channel, skipping");
1840            }
1841        }
1842    }
1843
1844    // Try to initialize Discord Gateway channel (websocket)
1845    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1846        && !token.is_empty()
1847    {
1848        let channel_id = config::get("channels.discord.channel_id").ok();
1849        let shard_id = config::get("channels.discord.shard_id")
1850            .ok()
1851            .and_then(|v| v.parse::<u64>().ok())
1852            .unwrap_or(0);
1853        let shard_count = config::get("channels.discord.shard_count")
1854            .ok()
1855            .and_then(|v| v.parse::<u64>().ok())
1856            .unwrap_or(1);
1857        match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1858            Ok(discord) => {
1859                if let Err(e) = discord.health_check().await {
1860                    tracing::warn!(?e, "discord gateway health check failed, skipping");
1861                } else {
1862                    channel_registry.register(Box::new(discord));
1863                    tracing::info!(shard_id, shard_count, "discord gateway registered");
1864                }
1865            }
1866            Err(e) => {
1867                tracing::warn!(?e, "failed to create discord gateway, skipping");
1868            }
1869        }
1870    }
1871
1872    // Try to initialize Slack channel
1873    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1874        && !token.is_empty()
1875    {
1876        let channel_id = config::get("channels.slack.channel_id").ok();
1877        match SlackChannel::new(token, channel_id) {
1878            Ok(slack) => {
1879                if let Err(e) = slack.health_check().await {
1880                    tracing::warn!(?e, "slack channel health check failed, skipping");
1881                } else {
1882                    channel_registry.register(Box::new(slack));
1883                    tracing::info!("slack channel registered");
1884                }
1885            }
1886            Err(e) => {
1887                tracing::warn!(?e, "failed to create slack channel, skipping");
1888            }
1889        }
1890    }
1891
1892    // Initialize tmux channel with transport for output streaming
1893    let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1894    channel_registry.register(Box::new(tmux_channel));
1895    tracing::info!("tmux channel registered");
1896
1897    // Start all channels and collect their message receivers
1898    let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1899    for channel in channel_registry.iter() {
1900        match channel.start().await {
1901            Ok(rx) => {
1902                tracing::info!(channel = channel.name(), "channel started");
1903                channel_receivers.push(rx);
1904            }
1905            Err(e) => {
1906                tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1907            }
1908        }
1909    }
1910
1911    // Wrap channel registry in Arc for shared access (notification dispatcher needs it)
1912    let channel_registry = Arc::new(channel_registry);
1913
1914    // Spawn tasks to handle incoming channel messages (if any channels are active)
1915    let transport_for_messages = Arc::clone(&transport);
1916    let tmux_for_messages = Arc::clone(&tmux);
1917    let capture_for_messages = Arc::clone(&capture_for_tick);
1918    let channels_for_messages = Arc::clone(&channel_registry);
1919    // Lightweight engine references for command execution and task creation
1920    let engine_refs: Vec<EngineRef> = project_engines
1921        .iter()
1922        .map(|e| {
1923            (
1924                e.repo.clone(),
1925                Arc::clone(&e.backend),
1926                Arc::clone(&e.task_manager),
1927                Some(Arc::clone(&e.store)),
1928            )
1929        })
1930        .collect();
1931    // Shared in-memory map for pending project picks (General channel multi-project picker).
1932    let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1933    let router_for_messages = Arc::clone(&channel_router);
1934    for mut rx in channel_receivers {
1935        let transport = Arc::clone(&transport_for_messages);
1936        let tmux = Arc::clone(&tmux_for_messages);
1937        let capture = Arc::clone(&capture_for_messages);
1938        let channels = Arc::clone(&channels_for_messages);
1939        let engine_refs = engine_refs.clone();
1940        let ch_router = Arc::clone(&router_for_messages);
1941        let picks = Arc::clone(&pending_picks);
1942        tokio::spawn(async move {
1943            while let Some(msg) = rx.recv().await {
1944                tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1945                channel_handler::handle_channel_message(
1946                    msg,
1947                    &transport,
1948                    &tmux,
1949                    &capture,
1950                    &channels,
1951                    &engine_refs,
1952                    &ch_router,
1953                    &picks,
1954                )
1955                .await;
1956            }
1957        });
1958    }
1959
1960    // Post-restart confirmation: if the service was restarted via the /restart control command,
1961    // a KV marker was stored before shutdown. Send a confirmation to the originating channel now
1962    // that we're back online. Use a short delay to let the channel connections re-establish.
1963    {
1964        let restart_channels = Arc::clone(&channel_registry);
1965        let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1966        if let Some(store) = restart_store {
1967            tokio::spawn(async move {
1968                time::sleep(Duration::from_secs(5)).await;
1969                match store.kv_get("control:restart_pending").await {
1970                    Ok(Some(conv_key)) => {
1971                        if let Some((ch_name, thread_id, topic_id)) =
1972                            transport::parse_conversation_key(&conv_key)
1973                        {
1974                            for ch in restart_channels.iter() {
1975                                if ch.name() == ch_name {
1976                                    let msg = OutgoingMessage {
1977                                        thread_id: thread_id.to_string(),
1978                                        body: "orch restarted successfully.".to_string(),
1979                                        reply_to: None,
1980                                        metadata: serde_json::json!({}),
1981                                        topic_id: topic_id.map(ToString::to_string),
1982                                    };
1983                                    if let Err(e) = ch.send(&msg).await {
1984                                        tracing::warn!(
1985                                            channel = ch_name,
1986                                            error = %e,
1987                                            "failed to send restart confirmation"
1988                                        );
1989                                    }
1990                                    break;
1991                                }
1992                            }
1993                        }
1994                        if let Err(e) = store.kv_delete("control:restart_pending").await {
1995                            tracing::warn!(error = %e, "failed to clear control:restart_pending");
1996                        }
1997                    }
1998                    Ok(None) => {} // no pending restart, nothing to do
1999                    Err(e) => {
2000                        tracing::warn!(error = %e, "failed to read control:restart_pending");
2001                    }
2002                }
2003            });
2004        }
2005    }
2006
2007    // Spawn notification dispatcher — reads task completion notifications
2008    // from transport and broadcasts to configured channels.
2009    //
2010    // Routing priority:
2011    // 1. Dedicated channel: if the notification has a `repo` and the
2012    //    ChannelRouter maps it to a topic/channel, send there with `topic_id`.
2013    // 2. Subscribed channels: send to each (channel, thread_id) subscribed to
2014    //    the repo, using `format_with_project()` so recipients know the source.
2015    // 3. Fallback: if no repo or no dedicated channel, broadcast to all
2016    //    channels without a topic_id (legacy behaviour).
2017    {
2018        let mut notification_rx = transport.subscribe_notifications();
2019        let channels = Arc::clone(&channel_registry);
2020        let notif_router = Arc::clone(&channel_router);
2021        // Grab a store reference for subscription lookups.
2022        let notif_store: Option<Arc<TaskStore>> =
2023            project_engines.first().map(|e| Arc::clone(&e.store));
2024        tokio::spawn(async move {
2025            loop {
2026                match notification_rx.recv().await {
2027                    Ok(notification) => {
2028                        let level = NotificationLevel::from_config();
2029                        if !level.should_notify(&notification.status) {
2030                            tracing::debug!(
2031                                task_id = %notification.task_id,
2032                                status = %notification.status,
2033                                "notification suppressed by level={:?}",
2034                                level
2035                            );
2036                            continue;
2037                        }
2038
2039                        tracing::info!(
2040                            task_id = %notification.task_id,
2041                            status = %notification.status,
2042                            repo = ?notification.repo,
2043                            "routing notification to channels"
2044                        );
2045
2046                        // Track whether we sent to at least one dedicated/subscribed target.
2047                        let mut routed = false;
2048
2049                        // 0. Direct target: if notify_target is set, send straight to
2050                        //    that Telegram chat_id and skip all other routing.
2051                        if let Some(ref target) = notification.notify_target {
2052                            let telegram = channels.iter().find(|c| c.name() == "telegram");
2053                            if let Some(channel) = telegram {
2054                                let body = notification.format_telegram();
2055                                let metadata = serde_json::json!({
2056                                    "preformatted_html": true,
2057                                    "chat_id_override": target
2058                                });
2059                                let msg = OutgoingMessage {
2060                                    thread_id: notification.task_id.clone(),
2061                                    body,
2062                                    reply_to: None,
2063                                    metadata,
2064                                    topic_id: None,
2065                                };
2066                                match channel.send(&msg).await {
2067                                    Err(e) => {
2068                                        tracing::warn!(
2069                                            task_id = %notification.task_id,
2070                                            target,
2071                                            ?e,
2072                                            "failed to send to notify_target"
2073                                        );
2074                                    }
2075                                    _ => {
2076                                        routed = true;
2077                                    }
2078                                }
2079                            }
2080                        }
2081
2082                        if routed {
2083                            // notify_target handled — skip further routing
2084                        } else if let Some(repo) = notification.repo.as_deref() {
2085                            // Track destinations already sent to in this iteration so
2086                            // the dedicated (path 1) and subscribed (path 2) loops
2087                            // don't double-fire the same Telegram/Discord destination
2088                            // when a repo's configured topic and a `channel_subscriptions`
2089                            // row resolve to the same `(channel, topic_id)`.
2090                            let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2091
2092                            // 1. Dedicated channel targets for this repo.
2093                            for channel in channels.iter() {
2094                                let ch_name = channel.name();
2095                                if let Some(topic_id) =
2096                                    notif_router.target_for_project(repo, ch_name)
2097                                {
2098                                    let body = match ch_name {
2099                                        "telegram" => notification.format_telegram(),
2100                                        "discord" => notification.format_discord(),
2101                                        "slack" => notification.format_slack(),
2102                                        _ => continue,
2103                                    };
2104                                    let metadata = if ch_name == "telegram" {
2105                                        serde_json::json!({"preformatted_html": true})
2106                                    } else {
2107                                        serde_json::json!({})
2108                                    };
2109                                    let resolved_topic = Some(topic_id.to_string());
2110                                    let msg = OutgoingMessage {
2111                                        thread_id: notification.task_id.clone(),
2112                                        body,
2113                                        reply_to: None,
2114                                        metadata,
2115                                        topic_id: resolved_topic.clone(),
2116                                    };
2117                                    if let Err(e) = channel.send(&msg).await {
2118                                        tracing::warn!(
2119                                            channel = ch_name,
2120                                            task_id = %notification.task_id,
2121                                            ?e,
2122                                            "failed to send to dedicated channel"
2123                                        );
2124                                    } else {
2125                                        sent.insert((ch_name.to_string(), resolved_topic));
2126                                        routed = true;
2127                                    }
2128                                }
2129                            }
2130
2131                            // 2. Subscribed channels for this repo.
2132                            if let Some(store) = &notif_store {
2133                                match store.list_subscribers_for_repo(repo).await {
2134                                    Ok(subscribers) => {
2135                                        for (ch_name, thread_id, topic_id) in subscribers {
2136                                            let channel =
2137                                                channels.iter().find(|c| c.name() == ch_name);
2138                                            let Some(channel) = channel else {
2139                                                continue;
2140                                            };
2141                                            let resolved_topic = if topic_id.is_empty() {
2142                                                None
2143                                            } else {
2144                                                Some(topic_id.clone())
2145                                            };
2146                                            // Skip if a dedicated send to the same
2147                                            // (channel, topic_id) already happened.
2148                                            if sent.contains(&(
2149                                                ch_name.clone(),
2150                                                resolved_topic.clone(),
2151                                            )) {
2152                                                tracing::debug!(
2153                                                    channel = %ch_name,
2154                                                    task_id = %notification.task_id,
2155                                                    topic_id = ?resolved_topic,
2156                                                    "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2157                                                );
2158                                                continue;
2159                                            }
2160                                            let body = notification.format_with_project(&ch_name);
2161                                            let metadata = if ch_name == "telegram" {
2162                                                serde_json::json!({"preformatted_html": true})
2163                                            } else {
2164                                                serde_json::json!({})
2165                                            };
2166                                            let msg = OutgoingMessage {
2167                                                thread_id: thread_id.clone(),
2168                                                body,
2169                                                reply_to: None,
2170                                                metadata,
2171                                                topic_id: resolved_topic.clone(),
2172                                            };
2173                                            if let Err(e) = channel.send(&msg).await {
2174                                                tracing::warn!(
2175                                                    channel = %ch_name,
2176                                                    task_id = %notification.task_id,
2177                                                    ?e,
2178                                                    "failed to send to subscribed channel"
2179                                                );
2180                                            } else {
2181                                                sent.insert((ch_name.clone(), resolved_topic));
2182                                                routed = true;
2183                                            }
2184                                        }
2185                                    }
2186                                    Err(e) => {
2187                                        tracing::warn!(
2188                                            repo,
2189                                            ?e,
2190                                            "failed to list subscribers for repo"
2191                                        );
2192                                    }
2193                                }
2194                            }
2195                        }
2196
2197                        // 3. Fallback: broadcast to all channels when no routing happened.
2198                        if !routed {
2199                            for channel in channels.iter() {
2200                                let (body, should_send, is_telegram) = match channel.name() {
2201                                    "telegram" => (notification.format_telegram(), true, true),
2202                                    "discord" => (notification.format_discord(), true, false),
2203                                    "slack" => (notification.format_slack(), true, false),
2204                                    // GitHub is already handled by backend.post_comment()
2205                                    // tmux doesn't need task completion notifications
2206                                    _ => (String::new(), false, false),
2207                                };
2208
2209                                if !should_send {
2210                                    continue;
2211                                }
2212
2213                                let metadata = if is_telegram {
2214                                    serde_json::json!({"preformatted_html": true})
2215                                } else {
2216                                    serde_json::json!({})
2217                                };
2218                                let msg = OutgoingMessage {
2219                                    thread_id: notification.task_id.clone(),
2220                                    body,
2221                                    reply_to: None,
2222                                    metadata,
2223                                    topic_id: None,
2224                                };
2225
2226                                if let Err(e) = channel.send(&msg).await {
2227                                    tracing::warn!(
2228                                        channel = channel.name(),
2229                                        task_id = %notification.task_id,
2230                                        ?e,
2231                                        "failed to send notification"
2232                                    );
2233                                }
2234                            }
2235                        }
2236                    }
2237                    Err(RecvError::Lagged(n)) => {
2238                        tracing::warn!(missed = n, "notification receiver lagged");
2239                    }
2240                    Err(RecvError::Closed) => {
2241                        tracing::debug!("notification channel closed");
2242                        break;
2243                    }
2244                }
2245            }
2246        });
2247        tracing::info!("notification dispatcher started");
2248    }
2249
2250    // Notify used by webhook events to wake up the engine tick immediately
2251    let webhook_notify = Arc::new(Notify::new());
2252
2253    // `WebhookStatus::fallback_mode` is only shown by `orch webhook status`, the sync
2254    // tick always runs every `engine.sync_interval` whether or not webhooks work
2255    let webhook_port: Option<u16>;
2256    let mut webhook_healthy: bool;
2257    let mut last_webhook_health_check = Instant::now();
2258    // Start webhook server if configured
2259    let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2260    // No readable secret keeps the webhook server off, it never accepts unsigned requests
2261    let webhook_secret = if webhook_enabled {
2262        secret_ref::config_secret("webhook.secret")
2263            .await
2264            .ok()
2265            .filter(|s| !s.trim().is_empty())
2266    } else {
2267        None
2268    };
2269    if webhook_enabled && webhook_secret.is_none() {
2270        tracing::error!(
2271            "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2272        );
2273    }
2274    let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2275    let webhook_secret = webhook_secret.unwrap_or_default();
2276
2277    // Shared status updated by the server-spawn task and the health-check loop.
2278    let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2279
2280    if webhook_enabled {
2281        // Spawn the HTTP server with exponential-backoff retry on transient
2282        // bind errors (e.g. EADDRINUSE).  After MAX_ATTEMPTS the engine falls
2283        // back to polling and updates the shared status accordingly.
2284        const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2285        let port: u16 = config::get("webhook.port")
2286            .ok()
2287            .and_then(|v| v.parse().ok())
2288            .unwrap_or(8080);
2289        webhook_port = Some(port);
2290        let secret = webhook_secret.clone();
2291        let webhook_repo = project_engines
2292            .first()
2293            .map(|e| e.repo.clone())
2294            .unwrap_or_default();
2295
2296        let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2297
2298        // Initialise status: configured, not yet healthy.
2299        let to_save = {
2300            let mut s = webhook_status.lock().await;
2301            s.configured = true;
2302            s.port = Some(port);
2303            s.healthy = false;
2304            s.fallback_mode = false;
2305            s.clone()
2306        };
2307        to_save.save().await;
2308
2309        let status_for_spawn = Arc::clone(&webhook_status);
2310        tokio::spawn(async move {
2311            let mut attempt = 0u32;
2312            loop {
2313                attempt += 1;
2314                tracing::info!(attempt, port, "attempting to start webhook server");
2315                match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2316                    .await
2317                {
2318                    Ok(()) => {
2319                        tracing::info!(attempt, "webhook server exited cleanly");
2320                        break;
2321                    }
2322                    Err(e) => {
2323                        let reason = e.to_string();
2324                        tracing::error!(attempt, %reason, "webhook server failed");
2325                        if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2326                            let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2327                                "max attempts reached"
2328                            } else {
2329                                "non-transient error"
2330                            };
2331                            tracing::error!(
2332                                attempt,
2333                                %reason,
2334                                kind,
2335                                orch_webhook_in_fallback = true,
2336                                "webhook server giving up, switching to polling fallback"
2337                            );
2338                            let to_save = {
2339                                let mut s = status_for_spawn.lock().await;
2340                                s.fallback_mode = true;
2341                                s.healthy = false;
2342                                s.last_failure_reason = Some(reason);
2343                                s.startup_attempts = attempt;
2344                                s.clone()
2345                            };
2346                            to_save.save().await;
2347                            break;
2348                        }
2349                        let delay = webhook_backoff_delay(attempt);
2350                        tracing::warn!(
2351                            attempt,
2352                            delay_ms = delay.as_millis(),
2353                            %reason,
2354                            "webhook bind failed (transient), retrying with backoff"
2355                        );
2356                        {
2357                            let to_save = {
2358                                let mut s = status_for_spawn.lock().await;
2359                                s.startup_attempts = attempt;
2360                                s.last_failure_reason = Some(reason);
2361                                s.clone()
2362                            };
2363                            to_save.save().await;
2364                        }
2365                        time::sleep(delay).await;
2366                    }
2367                }
2368            }
2369        });
2370
2371        // Spawn the message forwarding task (reads from webhook channel)
2372        let notify = Arc::clone(&webhook_notify);
2373        let transport_for_webhook = Arc::clone(&transport);
2374        tokio::spawn(async move {
2375            while let Some(msg) = rx.recv().await {
2376                tracing::info!(
2377                    channel = %msg.channel,
2378                    thread = %msg.thread_id,
2379                    event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2380                    action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2381                    "webhook event received, triggering immediate tick"
2382                );
2383                let route = transport_for_webhook.route(&msg).await;
2384                tracing::info!(
2385                    channel = %msg.channel,
2386                    route = ?route,
2387                    "webhook event routed"
2388                );
2389                // Wake up the engine tick immediately instead of waiting up to 10s
2390                notify.notify_one();
2391            }
2392        });
2393
2394        webhook_healthy = true;
2395        tracing::info!(port, orch_webhook_up = true, "webhook server started");
2396    } else {
2397        webhook_port = None;
2398        webhook_healthy = false;
2399        {
2400            let to_save = {
2401                let mut s = webhook_status.lock().await;
2402                s.configured = false;
2403                s.fallback_mode = true;
2404                s.clone()
2405            };
2406            to_save.save().await;
2407        }
2408        tracing::info!(
2409            orch_webhook_in_fallback = true,
2410            "webhook server disabled, using polling fallback mode"
2411        );
2412    }
2413
2414    start_relay_client(&project_engines, &webhook_notify);
2415    app::start_loader();
2416
2417    // Agent router (selects agent + model per task) - shared across projects.
2418    // Router::from_config() may run `opencode models` when the config uses opencode.
2419    // spawn_blocking keeps the Tokio runtime thread free during that I/O
2420    let router = Arc::new(RwLock::new(
2421        match task::spawn_blocking(Router::from_config).await {
2422            Ok(r) => r,
2423            Err(e) => {
2424                tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2425                task::spawn_blocking(|| Router::new(RouterConfig::default()))
2426                    .await
2427                    .unwrap_or_else(|e2| {
2428                        panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2429                    })
2430            }
2431        },
2432    ));
2433    {
2434        let r = router.read().await;
2435        tracing::info!(
2436            mode = %r.config.mode,
2437            agents = ?r.available_agents,
2438            fallback = %r.config.fallback_executor,
2439            "router initialized"
2440        );
2441    }
2442
2443    // Concurrency limiter (shared across all projects). `parallel_in_force` is the
2444    // limit actually handed out, which lags `config.max_parallel` after a shrink
2445    // until the running tasks give their permits back.
2446    let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2447    let mut parallel_in_force = config.max_parallel;
2448    // Reviews get their own pool, one reviewer per worker, so a busy worker pool never
2449    // starves a review (#3971)
2450    let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2451    let mut review_in_force = config.max_parallel;
2452
2453    // In-memory dispatch guard: tracks task IDs currently being dispatched.
2454    // Guards against GitHub API eventual consistency — after update_status(InProgress),
2455    // the label-removal webhook can trigger an immediate tick where list_by_status(Routed)
2456    // still returns the task (search index propagation delay). The tmux session does not
2457    // exist until the runner completes worktree setup (~10s later), so session_exists
2458    // alone is insufficient. Keyed by "{repo}/{task_id}".
2459    let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2460
2461    // In-memory guard to prevent double-spawn of auto-merge background tasks.
2462    // Since review_open_prs runs every sync_tick (~10s) but CI polling can take
2463    // up to 10 minutes, we track which tasks already have an auto-merge in flight.
2464    let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2465
2466    // Subscribe to config file changes for hot reload
2467    let mut config_rx = config::subscribe();
2468
2469    // Track sync interval
2470    let mut last_sync = Instant::now();
2471
2472    // Track upgrade check interval
2473    let mut last_upgrade_check: Option<Instant> = None;
2474
2475    // Reset epoch of the GitHub rate-limit window already reported, so the log line,
2476    // the KV flag and the channel notification fire once per window instead of per tick
2477    let mut rate_limit_reported: Option<i64> = None;
2478
2479    // Channel for weight signals from task runners back to the router
2480    let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2481
2482    // A restart ends every review session, so reset each in_review task left without one
2483    for engine in project_engines.iter().filter(|_| leads) {
2484        let reset =
2485            reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2486
2487        if reset > 0 {
2488            tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2489        }
2490    }
2491
2492    // Spawn dispatch subscribers — react to Routed events immediately
2493    for engine in &project_engines {
2494        subscribers::dispatch::spawn(
2495            event_bus.subscribe(),
2496            Arc::clone(&engine.backend),
2497            Arc::clone(&tmux),
2498            Arc::clone(&engine.runner),
2499            Arc::clone(&capture_for_tick),
2500            Arc::clone(&semaphore),
2501            Arc::clone(&engine.task_manager),
2502            weight_tx.clone(),
2503            Arc::clone(&router),
2504            Arc::clone(&dispatching),
2505            Arc::clone(&engine.store),
2506            engine.repo.clone(),
2507        );
2508    }
2509    tracing::info!("dispatch subscriber started");
2510
2511    // Spawn review subscribers — react to NeedsReview events immediately
2512    for engine in &project_engines {
2513        subscribers::review::spawn(
2514            event_bus.subscribe(),
2515            Arc::clone(&engine.backend),
2516            Arc::clone(&tmux),
2517            Arc::clone(&review_semaphore),
2518            Arc::clone(&engine.task_manager),
2519            Arc::clone(&router),
2520            Arc::clone(&dispatching),
2521            Arc::clone(&engine.store),
2522            engine.repo.clone(),
2523        );
2524    }
2525    tracing::info!("review subscriber started");
2526
2527    // Spawn unblock subscribers — react to Done events, unblock parent tasks immediately
2528    for engine in &project_engines {
2529        subscribers::unblock::spawn(
2530            event_bus.subscribe(),
2531            Arc::clone(&engine.backend),
2532            Arc::clone(&engine.task_manager),
2533            Arc::clone(&engine.store),
2534            engine.repo.clone(),
2535        );
2536    }
2537    tracing::info!("unblock subscriber started");
2538
2539    // Spawn notify subscriber — reacts to ALL events, pushes to transport.
2540    // Spawned once (not per-project) since the transport handles all repos.
2541    subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2542    tracing::info!("notify subscriber started");
2543
2544    // Main loop
2545    tracing::info!(
2546        tick = ?config.tick_interval,
2547        sync = ?config.sync_interval,
2548        parallel = config.max_parallel,
2549        "entering main loop"
2550    );
2551    // Values of the settings that only take effect at startup, so a reload can
2552    // tell the operator their edit needs a restart.
2553    let mut restart_only = restart_only_snapshot();
2554
2555    let mut interval = time::interval(config.tick_interval);
2556
2557    // Signal handlers (launchd/systemd send SIGTERM to stop services, SIGHUP for reload/restart)
2558    let mut sigterm = unix::signal(SignalKind::terminate())?;
2559    let mut sighup = unix::signal(SignalKind::hangup())?;
2560
2561    // Guard to prevent concurrent sync_tick runs (sync is spawned as a background task).
2562    let sync_in_progress = Arc::new(AtomicBool::new(false));
2563
2564    // Tick watchdog: detects when the main loop hasn't completed a tick in > 60s.
2565    // Updates an atomic timestamp at the end of each tick iteration.
2566    let last_tick_epoch = Arc::new(AtomicU64::new(
2567        SystemTime::now()
2568            .duration_since(std_time::UNIX_EPOCH)
2569            .unwrap_or_default()
2570            .as_secs(),
2571    ));
2572    {
2573        let last_tick_epoch = Arc::clone(&last_tick_epoch);
2574        let tick_interval_secs = config.tick_interval.as_secs();
2575        tokio::spawn(async move {
2576            // Check every 30s whether the main loop is still ticking.
2577            let mut watchdog_interval = time::interval(Duration::from_secs(30));
2578            // Own suspend baseline, independent of the main loop's checkpoint(). The
2579            // watchdog's interval timer also freezes during a host suspend, so this
2580            // detects the gap from its own wake-up instead of racing the main loop's
2581            // checkpoint() for the same GAP_LOG entry (see #3544).
2582            let mut suspend_detector = SuspendDetector::new();
2583            loop {
2584                watchdog_interval.tick().await;
2585                let own_gap = suspend_detector.check();
2586                let last = last_tick_epoch.load(Ordering::Relaxed);
2587                let now = SystemTime::now()
2588                    .duration_since(std_time::UNIX_EPOCH)
2589                    .unwrap_or_default()
2590                    .as_secs();
2591                let stale_secs = now.saturating_sub(last);
2592                // Warn if the tick loop hasn't completed in > 6× the tick interval (at least 60s).
2593                let threshold = (tick_interval_secs * 6).max(60);
2594                if stale_secs > threshold {
2595                    // A host suspend/resume gap fully explains a stale window that
2596                    // isn't an actual stall — the process (and its timers) were
2597                    // simply not scheduled while the host was asleep. Prefer this
2598                    // watchdog's own detector (immune to the checkpoint race above);
2599                    // fall back to the shared log in case the main loop's checkpoint()
2600                    // already ran and recorded a larger gap.
2601                    let shared_gap =
2602                        engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2603                    let gap = match (own_gap, shared_gap) {
2604                        (Some(a), Some(b)) => Some(a.max(b)),
2605                        (Some(a), None) => Some(a),
2606                        (None, Some(b)) => Some(b),
2607                        (None, None) => None,
2608                    };
2609                    if let Some(gap) = gap {
2610                        let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2611                        if gap_secs >= stale_secs {
2612                            tracing::info!(
2613                                stale_secs,
2614                                threshold,
2615                                suspend_gap_secs = gap_secs,
2616                                "tick loop stale window explained by host suspend/resume, not a stall"
2617                            );
2618                        } else {
2619                            tracing::error!(
2620                                stale_secs,
2621                                threshold,
2622                                suspend_gap_secs = gap_secs,
2623                                "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2624                                stale_secs,
2625                                threshold,
2626                                gap_secs,
2627                            );
2628                        }
2629                    } else {
2630                        tracing::error!(
2631                            stale_secs,
2632                            threshold,
2633                            "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2634                            stale_secs,
2635                            threshold,
2636                        );
2637                    }
2638                }
2639            }
2640        });
2641    }
2642
2643    loop {
2644        tokio::select! {
2645            _ = interval.tick() => {
2646                let tick_start = Instant::now();
2647
2648                // Detect a host suspend/resume gap once per iteration, in one place.
2649                // The watchdog and stuck-task recovery both read from this instead of
2650                // re-deriving suspend information from their own wall-clock deltas.
2651                if let Some(gap) = engine_suspend::checkpoint() {
2652                    tracing::info!(
2653                        gap_secs = gap.num_seconds(),
2654                        "host suspend/resume gap detected, discounting from stuck-task age checks"
2655                    );
2656                }
2657
2658                // Finish a pending shrink of the dispatch pool as running tasks
2659                // release their permits. Only logs when a permit was actually
2660                // taken back, so a long-running task does not log every tick.
2661                if review_in_force != config.max_parallel {
2662                    review_in_force =
2663                        resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2664                }
2665                if parallel_in_force != config.max_parallel {
2666                    let resized =
2667                        resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2668                    if resized != parallel_in_force {
2669                        parallel_in_force = resized;
2670                        tracing::info!(
2671                            in_force = parallel_in_force,
2672                            target = config.max_parallel,
2673                            "dispatch pool resized"
2674                        );
2675                    }
2676                }
2677
2678                // Tick weight recovery for rate-limited agents
2679                {
2680                    let mut rw = router.write().await;
2681                    rw.tick_weight_recovery();
2682                }
2683
2684                // Report the GitHub rate limit once per window, then tick anyway. Calls
2685                // into GitHub now fail fast instead of sleeping until the reset, so the
2686                // local phases keep moving and the watchdog does not see a stalled loop.
2687                if let Some(remaining) = GhHttp::is_rate_limited() {
2688                    let reset_at = chrono::Utc::now()
2689                        .timestamp()
2690                        .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2691                    // Tolerate jitter in the header so one window reports once
2692                    if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2693                        rate_limit_reported = Some(reset_at);
2694                        report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2695                    }
2696                } else if let Some(prev) = rate_limit_reported.take() {
2697                    tracing::info!(
2698                        limited_until = prev,
2699                        "GitHub API rate limit cleared"
2700                    );
2701                    if let Some(engine) = project_engines.first() {
2702                        let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2703                    }
2704                }
2705
2706                // Core tick: poll tasks for all projects
2707                tick::tick_all(
2708                    &project_engines,
2709                    &tmux,
2710                    &capture_for_tick,
2711                    &semaphore,
2712                    &config,
2713                    &router,
2714                    &weight_tx,
2715                    &dispatching,
2716                    Some(&transport),
2717                    &active_repos,
2718                    &auto_merge_in_flight,
2719                ).await;
2720
2721                // Drain pending weight signals after tick processing so the router
2722                // learns from outcomes produced by this iteration's task dispatches.
2723                while let Ok(signal) = weight_rx.try_recv() {
2724                    let mut rw = router.write().await;
2725                    match signal {
2726                        WeightSignal::RateLimited { ref agent } => {
2727                            rw.record_rate_limit(agent);
2728                        }
2729                        WeightSignal::Success { ref agent } => {
2730                            rw.record_success(agent);
2731                        }
2732                        WeightSignal::Failed { .. }
2733                        | WeightSignal::Rerouted
2734                        | WeightSignal::Blocked
2735                        | WeightSignal::None => {}
2736                    }
2737                }
2738
2739                // Periodic sync (less frequent) — spawned as a background task so it
2740                // cannot block the main tick loop. A stalled sync (hung HTTP call, slow
2741                // DB query, lock contention) previously parked all tokio workers for 12+
2742                // minutes because it ran inline. See issue #2574.
2743                if last_sync.elapsed() >= config.sync_interval {
2744                    if sync_in_progress.compare_exchange(
2745                        false,
2746                        true,
2747                        Ordering::AcqRel,
2748                        Ordering::Relaxed,
2749                    ).is_ok() {
2750                        spawn_sync_pass(
2751                            &project_engines,
2752                            &tmux,
2753                            &config,
2754                            &router,
2755                            &dispatching,
2756                            &auto_merge_in_flight,
2757                            &sync_in_progress,
2758                            "periodic",
2759                        );
2760                    } else {
2761                        tracing::debug!("sync tick still in progress, skipping");
2762                    }
2763                    // Records sync *schedule* time (not completion). This prevents
2764                    // re-entering the sync branch every tick while one is in flight.
2765                    // The sync_in_progress guard handles actual concurrency control.
2766                    last_sync = Instant::now();
2767                }
2768
2769                // Periodic webhook health check
2770                if webhook_enabled {
2771                    let health_check_interval = config.webhook_health_check_interval
2772                        .unwrap_or(Duration::from_secs(60));
2773
2774                    if last_webhook_health_check.elapsed() >= health_check_interval {
2775                        if let Some(port) = webhook_port {
2776                            let (health, failure_reason) =
2777                                github::check_webhook_health(port).await;
2778                            if health != webhook_healthy {
2779                                webhook_healthy = health;
2780                                if webhook_healthy {
2781                                    tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2782                                } else {
2783                                    tracing::warn!(
2784                                        port,
2785                                        orch_webhook_up = false,
2786                                        reason = failure_reason.as_deref().unwrap_or("unknown"),
2787                                        "webhook health check failed"
2788                                    );
2789                                }
2790                            }
2791                            // Persist updated status.
2792                            {
2793                                let to_save = {
2794                                    let mut s = webhook_status.lock().await;
2795                                    s.healthy = health;
2796                                    s.last_check_utc = Some(chrono::Utc::now());
2797                                    if health {
2798                                        s.last_failure_reason = None;
2799                                    } else if failure_reason.is_some() {
2800                                        s.last_failure_reason = failure_reason;
2801                                    }
2802                                    s.clone()
2803                                };
2804                                to_save.save().await;
2805                            }
2806                        }
2807                        last_webhook_health_check = Instant::now();
2808                    }
2809                }
2810
2811                // Periodic upgrade check — notify channels when a newer release is available.
2812                if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2813                    if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2814                        let channels = Arc::clone(&channel_registry);
2815                        let current = env!("ORCH_VERSION").to_string();
2816                        let check_interval =
2817                            Duration::from_secs(config.upgrade_check_interval);
2818                        tokio::spawn(async move {
2819                            check_and_notify_upgrade(&store, &channels, &current, check_interval)
2820                                .await;
2821                        });
2822                    }
2823                    last_upgrade_check = Some(Instant::now());
2824                }
2825
2826                // Update watchdog timestamp + log tick duration.
2827                let tick_elapsed = tick_start.elapsed();
2828                if tick_elapsed.as_secs() > 30 {
2829                    tracing::warn!(
2830                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2831                        "slow tick"
2832                    );
2833                } else {
2834                    tracing::debug!(
2835                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2836                        "tick complete"
2837                    );
2838                }
2839                last_tick_epoch.store(
2840                    SystemTime::now()
2841                        .duration_since(std_time::UNIX_EPOCH)
2842                        .unwrap_or_default()
2843                        .as_secs(),
2844                    Ordering::Relaxed,
2845                );
2846            }
2847            // Webhook events trigger an immediate tick (bypass polling interval)
2848            () = webhook_notify.notified() => {
2849                if let Some(remaining) = GhHttp::is_rate_limited() {
2850                    tracing::warn!(
2851                        remaining_secs = remaining.as_secs(),
2852                        "GitHub API rate-limited, skipping webhook-triggered tick"
2853                    );
2854                } else {
2855                    tracing::info!("webhook event triggered immediate tick");
2856
2857                    tick::tick_all(
2858                        &project_engines,
2859                        &tmux,
2860                        &capture_for_tick,
2861                        &semaphore,
2862                        &config,
2863                        &router,
2864                        &weight_tx,
2865                        &dispatching,
2866                        Some(&transport),
2867                        &active_repos,
2868                        &auto_merge_in_flight,
2869                    ).await;
2870
2871                // Drain pending weight signals after webhook-triggered tick processing.
2872                while let Ok(signal) = weight_rx.try_recv() {
2873                    let mut rw = router.write().await;
2874                    match signal {
2875                        WeightSignal::RateLimited { ref agent } => {
2876                            rw.record_rate_limit(agent);
2877                        }
2878                        WeightSignal::Success { ref agent } => {
2879                            rw.record_success(agent);
2880                        }
2881                        WeightSignal::Failed { .. }
2882                        | WeightSignal::Rerouted
2883                        | WeightSignal::Blocked
2884                        | WeightSignal::None => {}
2885                    }
2886                }
2887
2888                // Also attempt to run an immediate sync (ingest/cleanup) in the
2889                // background so webhook-driven events (e.g. new issues) are
2890                // picked up without waiting for the periodic sync interval.
2891                // Reuse the same sync_in_progress guard used by the periodic
2892                // branch to avoid concurrent syncs.
2893                if sync_in_progress.compare_exchange(
2894                    false,
2895                    true,
2896                    Ordering::AcqRel,
2897                    Ordering::Relaxed,
2898                ).is_ok() {
2899                    spawn_sync_pass(
2900                        &project_engines,
2901                        &tmux,
2902                        &config,
2903                        &router,
2904                        &dispatching,
2905                        &auto_merge_in_flight,
2906                        &sync_in_progress,
2907                        "webhook",
2908                    );
2909                    // Record schedule time so periodic sync doesn't immediately re-run.
2910                    last_sync = Instant::now();
2911                } else {
2912                    tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2913                }
2914                }
2915
2916                // Also reset the interval so we don't get a redundant tick right after
2917                interval.reset();
2918            }
2919            result = config_rx.recv() => {
2920                // A notification only arrives for a file that parsed, so there is no
2921                // broken-config case to guard against here (see `config::reload_file`).
2922                let reload = match result {
2923                    Ok(path) => {
2924                        tracing::info!(path = %path.display(), "config file changed, reloading");
2925                        true
2926                    }
2927                    Err(RecvError::Lagged(n)) => {
2928                        tracing::warn!(missed = n, "config change receiver lagged, reloading");
2929                        true
2930                    }
2931                    Err(RecvError::Closed) => {
2932                        tracing::warn!("config change channel closed");
2933                        false
2934                    }
2935                };
2936
2937                if reload {
2938                    // Reload engine config
2939                    let new_config = EngineConfig::from_config();
2940                    log_engine_config_changes(&config, &new_config);
2941
2942                    let new_restart_only = restart_only_snapshot();
2943                    log_restart_only_changes(&restart_only, &new_restart_only);
2944                    restart_only = new_restart_only;
2945
2946                    let tick_changed = new_config.tick_interval != config.tick_interval;
2947                    config = new_config;
2948
2949                    // Reset tick interval if it changed
2950                    if tick_changed {
2951                        interval = time::interval(config.tick_interval);
2952                        tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2953                    }
2954
2955                    // Resize the dispatch pool. A shrink only takes the permits that are
2956                    // free right now; the rest are taken as running tasks finish.
2957                    if review_in_force != config.max_parallel {
2958                        review_in_force = resize_dispatch_pool(
2959                            &review_semaphore,
2960                            review_in_force,
2961                            config.max_parallel,
2962                        );
2963                    }
2964                    if parallel_in_force != config.max_parallel {
2965                        parallel_in_force = resize_dispatch_pool(
2966                            &semaphore,
2967                            parallel_in_force,
2968                            config.max_parallel,
2969                        );
2970                        tracing::info!(
2971                            in_force = parallel_in_force,
2972                            target = config.max_parallel,
2973                            "dispatch pool resized"
2974                        );
2975                    }
2976
2977                    // Reload router config (async-safe: use reload_async to avoid blocking the runtime)
2978                    {
2979                        let mut router_guard = router.write().await;
2980                        router_guard.reload_async().await;
2981                    }
2982
2983                    // Pick up projects added to `projects:`
2984                    sync_project_engines(
2985                        &mut project_engines,
2986                        &mut active_repos,
2987                        &event_bus,
2988                        &tmux,
2989                        &capture_for_tick,
2990                        &semaphore,
2991                        &review_semaphore,
2992                        &weight_tx,
2993                        &router,
2994                        &dispatching,
2995                    ).await;
2996
2997                    tracing::info!(
2998                        tick = ?config.tick_interval,
2999                        sync = ?config.sync_interval,
3000                        parallel = config.max_parallel,
3001                        projects = project_engines.len(),
3002                        "config reloaded"
3003                    );
3004                }
3005            }
3006            signal_name = async {
3007                tokio::select! {
3008                    _ = signal::ctrl_c() => "SIGINT",
3009                    _ = sigterm.recv() => "SIGTERM",
3010                    _ = sighup.recv() => "SIGHUP",
3011                }
3012            } => {
3013                tracing::info!(signal = signal_name, "beginning graceful shutdown");
3014
3015                // Wrap the entire shutdown sequence in a timeout to enforce the configured limit.
3016                // This prevents indefinite blocking if status updates or tmux operations hang.
3017                let shutdown_result = time::timeout(
3018                    config.graceful_shutdown_timeout,
3019                    async {
3020                        // Reset in_progress tasks to routed so they re-dispatch after restart.
3021                        // Also reset in_review tasks to needs_review — their review agent
3022                        // tmux sessions will be killed when the process exits.
3023                        let mut reset_count = 0u32;
3024                        let mut review_reset_count = 0u32;
3025                        for engine in &project_engines {
3026                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
3027                                for task in &tasks {
3028                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
3029                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
3030                                    } else {
3031                                        if let Ok(Some(store_id)) =
3032                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3033                                            && let Err(e) = engine
3034                                                .store
3035                                                .finalize_incomplete_runs(
3036                                                    store_id,
3037                                                    "aborted",
3038                                                    "graceful shutdown: reset in_progress task to routed",
3039                                                )
3040                                                .await
3041                                            {
3042                                                tracing::warn!(
3043                                                    task_id = task.id.0,
3044                                                    ?e,
3045                                                    "failed to finalize incomplete runs on shutdown"
3046                                                );
3047                                            }
3048                                        reset_count += 1;
3049                                    }
3050                                }
3051                            }
3052                            // Also reset internal in_progress tasks
3053                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
3054                                for task in &tasks {
3055                                    let task_id = format!("internal:{}", task.id);
3056                                    if let Err(e) = engine.task_manager.update_task_status(
3057                                        &ExternalId(task_id.clone()),
3058                                        Status::Routed,
3059                                    ).await {
3060                                        tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3061                                    } else {
3062                                        if let Err(e) = engine
3063                                            .store
3064                                            .finalize_incomplete_runs(
3065                                                task.id,
3066                                                "aborted",
3067                                                "graceful shutdown: reset in_progress task to routed",
3068                                            )
3069                                            .await
3070                                        {
3071                                            tracing::warn!(
3072                                                task_id,
3073                                                ?e,
3074                                                "failed to finalize incomplete internal runs on shutdown"
3075                                            );
3076                                        }
3077                                        reset_count += 1;
3078                                    }
3079                                }
3080                            }
3081                            // Reset in_review tasks — review agent sessions die on shutdown
3082                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3083                                for task in &tasks {
3084                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3085                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3086                                    } else {
3087                                        if let Ok(Some(store_id)) =
3088                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3089                                            && let Err(e) = engine
3090                                                .store
3091                                                .finalize_incomplete_runs(
3092                                                    store_id,
3093                                                    "aborted",
3094                                                    "graceful shutdown: reset in_review task to needs_review",
3095                                                )
3096                                                .await
3097                                            {
3098                                                tracing::warn!(
3099                                                    task_id = task.id.0,
3100                                                    ?e,
3101                                                    "failed to finalize incomplete review runs on shutdown"
3102                                                );
3103                                            }
3104                                        review_reset_count += 1;
3105                                    }
3106                                }
3107                            }
3108                            // Also reset internal in_review tasks
3109                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3110                                for task in &tasks {
3111                                    let task_id = format!("internal:{}", task.id);
3112                                    if let Err(e) = engine.task_manager.update_task_status(
3113                                        &ExternalId(task_id.clone()),
3114                                        Status::NeedsReview,
3115                                    ).await {
3116                                        tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3117                                    } else {
3118                                        if let Err(e) = engine
3119                                            .store
3120                                            .finalize_incomplete_runs(
3121                                                task.id,
3122                                                "aborted",
3123                                                "graceful shutdown: reset in_review task to needs_review",
3124                                            )
3125                                            .await
3126                                        {
3127                                            tracing::warn!(
3128                                                task_id,
3129                                                ?e,
3130                                                "failed to finalize incomplete internal review runs on shutdown"
3131                                            );
3132                                        }
3133                                        review_reset_count += 1;
3134                                    }
3135                                }
3136                            }
3137                        }
3138                        if reset_count > 0 {
3139                            tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3140                        }
3141                        if review_reset_count > 0 {
3142                            tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3143                        }
3144
3145                        // Kill all orch-managed tmux sessions so stale sessions don't
3146                        // block dispatch after restart (session_exists check).
3147                        let mut killed_sessions = Vec::new();
3148                        if let Ok(sessions) = tmux.list_sessions().await {
3149                            for session in &sessions {
3150                                if session.name.starts_with("orch-") {
3151                                    match tmux.kill_session(&session.name).await { Err(e) => {
3152                                        tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3153                                    } _ => {
3154                                        killed_sessions.push(session.name.clone());
3155                                    }}
3156                                }
3157                            }
3158                            if !killed_sessions.is_empty() {
3159                                tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3160                            }
3161                        }
3162
3163                        // Wait for sessions to actually die before exiting.
3164                        // Prevents race where process exits before tmux finishes cleanup.
3165                        if !killed_sessions.is_empty() {
3166                            let still_alive = tmux
3167                                .wait_for_sessions_dead(
3168                                    &killed_sessions,
3169                                    Duration::from_millis(100),
3170                                    Duration::from_secs(5),
3171                                )
3172                                .await;
3173                            if still_alive > 0 {
3174                                tracing::warn!(
3175                                    still_alive,
3176                                    "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3177                                );
3178                            }
3179                            tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3180                        }
3181                    },
3182                ).await;
3183
3184                if shutdown_result.is_err() {
3185                    tracing::error!(
3186                        timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3187                        "graceful shutdown timed out, forcing exit"
3188                    );
3189                }
3190
3191                break;
3192            }
3193        }
3194    }
3195
3196    // Clean up event bus port file and service version file
3197    events::cleanup_port_file().await;
3198    local::cleanup().await;
3199    events::cleanup_version_file();
3200
3201    // transport and channels drop here at end of scope
3202    let _ = transport;
3203    tracing::info!("orch engine stopped");
3204    Ok(())
3205}
3206
3207#[cfg(test)]
3208mod tests {
3209    use std::env;
3210    use std::process;
3211    use std::slice;
3212    use std::sync::Arc;
3213
3214    use super::*;
3215    use crate::backends::ExternalBackend;
3216    use crate::backends::test_helpers::NoopBackend;
3217    use crate::channels::capture::CaptureService;
3218    use crate::channels::transport::Transport;
3219    use crate::engine::runner::git_ops;
3220    use crate::store::UpsertExternal;
3221    use crate::tmux::SessionGuard;
3222    use crate::tmux::TmuxManager;
3223
3224    async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3225        let id = store
3226            .upsert_external(&UpsertExternal {
3227                repo: "owner/repo",
3228                ext_id: number,
3229                title: "Under review",
3230                body: "",
3231                author: "",
3232                url: "",
3233                labels: &[],
3234                origin: "github",
3235            })
3236            .await
3237            .unwrap();
3238        store.update_status(id, TaskStatus::InReview).await.unwrap();
3239        id
3240    }
3241
3242    #[tokio::test]
3243    async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3244        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3245        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3246        let task_manager =
3247            TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3248        let tmux = TmuxManager::isolated();
3249        let orphaned = in_review_task(&store, "11").await;
3250        let live = in_review_task(&store, "12").await;
3251        let session = tmux
3252            .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3253            .await
3254            .unwrap();
3255        let _guard = SessionGuard(session);
3256
3257        let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3258
3259        assert_eq!(reset, 1);
3260        assert_eq!(
3261            store.get(orphaned).await.unwrap().status,
3262            TaskStatus::NeedsReview
3263        );
3264        assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3265    }
3266
3267    #[test]
3268    fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3269        let semaphore = Arc::new(Semaphore::new(4));
3270        let held: Vec<_> = (0..4)
3271            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3272            .collect();
3273        assert!(
3274            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3275            "pool is full"
3276        );
3277
3278        let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3279        assert_eq!(in_force, 8);
3280
3281        let grown: Vec<_> = (0..4)
3282            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3283            .collect();
3284        assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3285        assert!(
3286            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3287            "pool is full"
3288        );
3289    }
3290
3291    #[test]
3292    fn shrinking_max_parallel_never_cancels_a_running_task() {
3293        let semaphore = Arc::new(Semaphore::new(8));
3294        let mut held: Vec<_> = (0..8)
3295            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3296            .collect();
3297
3298        // Nothing is free, so all 8 tasks keep their permits and keep running.
3299        let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3300        assert_eq!(in_force, 8);
3301        assert_eq!(held.len(), 8);
3302
3303        // Three finish. Their permits are taken back instead of handed out again.
3304        held.truncate(5);
3305        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3306        assert_eq!(in_force, 5);
3307        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3308
3309        // One more finishes and the pool settles at the new limit.
3310        held.truncate(4);
3311        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3312        assert_eq!(in_force, 4);
3313        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3314
3315        // A slot freed after that is handed out, since 4 is now the real limit.
3316        held.truncate(3);
3317        assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3318    }
3319
3320    #[test]
3321    fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3322        let configured = vec![
3323            ("owner/one".to_string(), PathBuf::from("/srv/one")),
3324            ("owner/two".to_string(), PathBuf::from("/srv/two")),
3325        ];
3326        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3327
3328        let (added, removed) = diff_configured_projects(&configured, &active);
3329        assert_eq!(
3330            added,
3331            vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3332            "only the new project is started, so owner/one keeps its single set of subscribers"
3333        );
3334        assert!(removed.is_empty(), "{removed:?}");
3335    }
3336
3337    #[test]
3338    fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3339        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3340        let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3341            .into_iter()
3342            .collect();
3343
3344        let (added, removed) = diff_configured_projects(&configured, &active);
3345        assert!(added.is_empty(), "nothing new to start");
3346        assert_eq!(removed, vec!["owner/gone".to_string()]);
3347    }
3348
3349    #[test]
3350    fn an_unchanged_project_list_starts_nothing() {
3351        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3352        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3353
3354        assert_eq!(
3355            diff_configured_projects(&configured, &active),
3356            (vec![], vec![])
3357        );
3358    }
3359
3360    #[test]
3361    fn restart_only_keys_are_the_settings_read_once_at_startup() {
3362        // Guards the docs table: every key listed here is documented as needing
3363        // `orch service restart`, and nothing reloadable may be added to it.
3364        assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3365    }
3366
3367    #[test]
3368    fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3369        let before = vec![Some("bot_token: a\n".to_string()), None];
3370
3371        assert!(
3372            log_restart_only_changes(&before, &before.clone()).is_empty(),
3373            "a reload that did not touch these keys must stay quiet"
3374        );
3375
3376        // An edited token and a newly added webhook block are both changes
3377        let after = vec![
3378            Some("bot_token: b\n".to_string()),
3379            Some("port: 8080\n".to_string()),
3380        ];
3381        assert_eq!(
3382            log_restart_only_changes(&before, &after),
3383            ["channels", "webhook"]
3384        );
3385
3386        // Only the edited key is reported when the other is untouched
3387        let token_only = vec![Some("bot_token: b\n".to_string()), None];
3388        assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3389    }
3390
3391    #[test]
3392    fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3393        let semaphore = Arc::new(Semaphore::new(4));
3394        assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3395        assert_eq!(semaphore.available_permits(), 4);
3396    }
3397
3398    #[test]
3399    fn live_session_stuck_limit_follows_the_run_timeout() {
3400        let config = EngineConfig {
3401            stuck_timeout: 1800,
3402            run_timeouts: RunTimeouts {
3403                base: 1800,
3404                medium: 2700,
3405                complex: 3600,
3406            },
3407            ..EngineConfig::default()
3408        };
3409        assert_eq!(
3410            config.live_session_stuck_limit(Some("simple")),
3411            1800 + RUN_TIMEOUT_GRACE_SECS
3412        );
3413        assert_eq!(
3414            config.live_session_stuck_limit(Some("medium")),
3415            2700 + RUN_TIMEOUT_GRACE_SECS
3416        );
3417        assert_eq!(
3418            config.live_session_stuck_limit(Some("complex")),
3419            3600 + RUN_TIMEOUT_GRACE_SECS
3420        );
3421
3422        let long_stuck = EngineConfig {
3423            stuck_timeout: 7200,
3424            ..config
3425        };
3426        assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3427    }
3428
3429    #[test]
3430    fn engine_config_defaults() {
3431        let config = EngineConfig::default();
3432        assert_eq!(config.tick_interval, Duration::from_secs(10));
3433        assert_eq!(config.sync_interval, Duration::from_secs(45));
3434        assert_eq!(config.max_parallel, 4);
3435        assert_eq!(config.stuck_timeout, 1800);
3436        assert_eq!(config.no_session_stuck_timeout, 600);
3437        assert!(
3438            config.no_session_stuck_timeout < config.stuck_timeout,
3439            "no_session_stuck_timeout must be shorter than stuck_timeout"
3440        );
3441        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3442        assert_eq!(config.upgrade_check_interval, 3600);
3443    }
3444
3445    #[test]
3446    fn engine_config_from_config_uses_defaults_when_no_config() {
3447        let config = EngineConfig::from_config();
3448        assert_eq!(config.tick_interval, Duration::from_secs(10));
3449        assert_eq!(config.sync_interval, Duration::from_secs(45));
3450        // max_parallel may be overridden by user config; just check it's reasonable
3451        assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3452        assert_eq!(config.stuck_timeout, 1800);
3453        assert_eq!(config.no_session_stuck_timeout, 600);
3454        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3455        assert_eq!(config.upgrade_check_interval, 3600);
3456    }
3457
3458    #[test]
3459    fn engine_config_no_session_timeout_is_under_15_minutes() {
3460        let config = EngineConfig::default();
3461        assert!(
3462            config.no_session_stuck_timeout <= 900,
3463            "no_session_stuck_timeout ({}) exceeds 15 minutes",
3464            config.no_session_stuck_timeout
3465        );
3466    }
3467
3468    #[test]
3469    fn upgrade_check_runs_immediately_when_never_checked() {
3470        assert!(should_run_upgrade_check(None, 3600));
3471    }
3472
3473    #[test]
3474    fn upgrade_check_waits_for_interval_after_first_run() {
3475        assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3476    }
3477
3478    #[test]
3479    fn upgrade_check_can_be_disabled() {
3480        assert!(!should_run_upgrade_check(None, 0));
3481    }
3482
3483    #[test]
3484    fn configured_agents_parses_yaml_block_array() {
3485        // config::get returns serde_norway::to_string for arrays, which is YAML block format
3486        let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3487        let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3488        assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3489    }
3490
3491    #[test]
3492    fn configured_agents_parses_json_inline_array() {
3493        let json_str = r#"["claude","codex","olm"]"#;
3494        let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3495        assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3496    }
3497
3498    #[test]
3499    fn configured_agents_fallback_returns_default_agents() {
3500        // When no config is present, configured_agents() returns DEFAULT_AGENTS
3501        let defaults = configured_agents();
3502        assert!(defaults.contains(&"claude".to_string()));
3503        assert!(defaults.contains(&"codex".to_string()));
3504        assert!(defaults.contains(&"opencode".to_string()));
3505    }
3506
3507    /// Integration test that requires a fully working tmux environment with
3508    /// reliable capture-pane timing. Runs locally but not in CI (GitHub Actions
3509    /// has tmux installed but capture-pane timing is unreliable).
3510    ///
3511    /// DO NOT REMOVE #[ignore] — this test was un-ignored in PR #608 which
3512    /// broke CI. The `tmux -V` skip check is insufficient because tmux CAN
3513    /// create sessions on CI runners, but capture-pane output is empty or
3514    /// delayed, causing the assertion to fail after a 10s timeout.
3515    #[tokio::test]
3516    #[ignore = "capture-pane timing is unreliable on CI runners"]
3517    async fn integration_channel_to_tmux_to_capture() {
3518        // Create transport and capture service
3519        let transport = Arc::new(Transport::new());
3520        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3521        let tmux = TmuxManager::new();
3522
3523        let task_id = "testtask1".to_string();
3524        let session_name = format!("orch-test-{task_id}");
3525
3526        // Start a detached tmux session
3527        let _ = tmux
3528            .command()
3529            .args(["new-session", "-d", "-s", &session_name])
3530            .output()
3531            .await
3532            .expect("failed to start tmux session");
3533
3534        // Guard ensures session cleanup even on panic
3535        let _guard = SessionGuard(session_name.clone());
3536
3537        // Register session with capture service and transport binding
3538        capture
3539            .register_session("owner/repo", &task_id, &session_name)
3540            .await;
3541        transport
3542            .bind(
3543                "owner/repo",
3544                &task_id,
3545                &session_name,
3546                "telegram",
3547                "12345",
3548                None,
3549            )
3550            .await;
3551
3552        // Spawn capture.run() which exits when session is unregistered
3553        let capture_clone = Arc::clone(&capture);
3554        let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3555
3556        // Subscribe to transport output for this task
3557        let mut rx = transport
3558            .subscribe("owner/repo", &task_id)
3559            .await
3560            .expect("no subscription");
3561
3562        // Send a message into the tmux session via tmux send-keys (simulate channel input)
3563        let send = tmux
3564            .command()
3565            .args([
3566                "send-keys",
3567                "-t",
3568                &session_name,
3569                "echo from-channel",
3570                "Enter",
3571            ])
3572            .output()
3573            .await
3574            .expect("failed to send keys");
3575        assert!(send.status.success());
3576
3577        // Wait for an output chunk from capture -> transport
3578        let mut got = false;
3579        for _ in 0..20 {
3580            if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3581                && c.content.contains("from-channel")
3582            {
3583                got = true;
3584                break;
3585            }
3586        }
3587
3588        // Clean up: unregister session (tmux kill handled by SessionGuard on drop)
3589        capture.unregister_session("owner/repo", &task_id).await;
3590
3591        // Wait for capture.run to finish
3592        let _ = capture_handle.await;
3593
3594        assert!(got, "did not observe tmux output via capture/transport");
3595    }
3596
3597    // ── read_project_channel_config tests ─────────────────────────────────
3598
3599    #[test]
3600    fn read_project_channel_config_returns_defaults_when_no_file() {
3601        let dir = tempfile::tempdir().expect("tempdir");
3602        let cfg = read_project_channel_config(dir.path());
3603        assert!(cfg.telegram_topic_id.is_none());
3604        assert!(cfg.discord_channel_id.is_none());
3605    }
3606
3607    #[test]
3608    fn read_project_channel_config_parses_telegram_and_discord() {
3609        let dir = tempfile::tempdir().expect("tempdir");
3610        let orch_yml = dir.path().join(".orch.yml");
3611        fs::write(
3612            &orch_yml,
3613            "channels:\n  telegram:\n    topic_id: \"42\"\n  discord:\n    channel_id: \"9999\"\n",
3614        )
3615        .expect("write .orch.yml");
3616        let cfg = read_project_channel_config(dir.path());
3617        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3618        assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3619    }
3620
3621    #[test]
3622    fn read_project_channel_config_handles_partial_channels() {
3623        let dir = tempfile::tempdir().expect("tempdir");
3624        let orch_yml = dir.path().join(".orch.yml");
3625        fs::write(&orch_yml, "channels:\n  telegram:\n    topic_id: \"7\"\n")
3626            .expect("write .orch.yml");
3627        let cfg = read_project_channel_config(dir.path());
3628        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3629        assert!(cfg.discord_channel_id.is_none());
3630    }
3631
3632    #[test]
3633    fn read_project_channel_config_handles_malformed_yaml() {
3634        let dir = tempfile::tempdir().expect("tempdir");
3635        let orch_yml = dir.path().join(".orch.yml");
3636        fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3637        let cfg = read_project_channel_config(dir.path());
3638        // Should return defaults (not panic) and emit a warn! log with the parse error.
3639        assert!(cfg.telegram_topic_id.is_none());
3640        assert!(cfg.discord_channel_id.is_none());
3641    }
3642
3643    #[test]
3644    #[cfg(unix)]
3645    fn read_project_channel_config_handles_unreadable_file() {
3646        let dir = tempfile::tempdir().expect("tempdir");
3647        // A directory fails to read for every user, root included, where chmod 000 does not
3648        fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3649        let cfg = read_project_channel_config(dir.path());
3650        // Should return defaults (not panic) and emit a warn! log with the read error.
3651        assert!(cfg.telegram_topic_id.is_none());
3652        assert!(cfg.discord_channel_id.is_none());
3653    }
3654
3655    fn git(dir: &Path, args: &[&str]) {
3656        let mut cmd = process::Command::new("git");
3657        cmd.current_dir(dir);
3658        if args.first() == Some(&"commit") {
3659            cmd.args(["-c", "commit.gpgsign=false"]);
3660        }
3661        let out = cmd
3662            .args(args)
3663            .output()
3664            .unwrap_or_else(|e| panic!("git {args:?} failed to start: {e}"));
3665        assert!(
3666            out.status.success(),
3667            "git {:?} failed:\nstdout: {}\nstderr: {}",
3668            args,
3669            String::from_utf8_lossy(&out.stdout),
3670            String::from_utf8_lossy(&out.stderr)
3671        );
3672    }
3673
3674    fn restore_orch_home(prev: Option<String>) {
3675        match prev {
3676            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3677            Some(v) => unsafe { env::set_var("ORCH_HOME", v) },
3678            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3679            None => unsafe { env::remove_var("ORCH_HOME") },
3680        }
3681    }
3682
3683    struct StartupConflict {
3684        _tmp: tempfile::TempDir,
3685        _orch_home: tempfile::TempDir,
3686        prev_orch_home: Option<String>,
3687        project_dir: PathBuf,
3688        worktree_dir: PathBuf,
3689        branch: String,
3690        store: Arc<TaskStore>,
3691        store_id: i64,
3692        engine: ProjectEngine,
3693    }
3694
3695    impl Drop for StartupConflict {
3696        fn drop(&mut self) {
3697            restore_orch_home(self.prev_orch_home.take());
3698        }
3699    }
3700
3701    async fn startup_conflict_fixture(
3702        retries: i32,
3703        pr_number: Option<i32>,
3704        start_rebase: bool,
3705    ) -> StartupConflict {
3706        let orch_home = tempfile::tempdir().unwrap();
3707        let prev_orch_home = env::var("ORCH_HOME").ok();
3708        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3709        unsafe { env::set_var("ORCH_HOME", orch_home.path()) };
3710
3711        let tmp = tempfile::tempdir().unwrap();
3712        let origin = tmp.path().join("origin.git");
3713        fs::create_dir_all(&origin).unwrap();
3714        git(&origin, &["init", "--bare", "-b", "main"]);
3715        let project_dir = tmp.path().join("repo");
3716        git(
3717            tmp.path(),
3718            &[
3719                "clone",
3720                origin.to_str().unwrap(),
3721                project_dir.to_str().unwrap(),
3722            ],
3723        );
3724        git(&project_dir, &["config", "user.email", "t@t.com"]);
3725        git(&project_dir, &["config", "user.name", "T"]);
3726        fs::write(project_dir.join("f.txt"), "base\n").unwrap();
3727        git(&project_dir, &["add", "."]);
3728        git(&project_dir, &["commit", "-m", "base"]);
3729        git(&project_dir, &["push", "origin", "HEAD:main"]);
3730        git(&project_dir, &["remote", "set-head", "origin", "main"]);
3731
3732        let branch = "gh-issue-4087-conflict".to_string();
3733        git(&project_dir, &["checkout", "-b", &branch]);
3734        fs::write(project_dir.join("f.txt"), "agent\n").unwrap();
3735        git(&project_dir, &["commit", "-am", "agent"]);
3736        git(&project_dir, &["checkout", "main"]);
3737        fs::write(project_dir.join("f.txt"), "main\n").unwrap();
3738        git(&project_dir, &["commit", "-am", "main"]);
3739        git(&project_dir, &["push", "origin", "HEAD:main"]);
3740
3741        let wt_base = orch_home.path().join("worktrees").join("repo");
3742        fs::create_dir_all(&wt_base).unwrap();
3743        let worktree_dir = wt_base.join(&branch);
3744        git(
3745            &project_dir,
3746            &["worktree", "add", worktree_dir.to_str().unwrap(), &branch],
3747        );
3748        git(&worktree_dir, &["config", "user.email", "t@t.com"]);
3749        git(&worktree_dir, &["config", "user.name", "T"]);
3750        git(&worktree_dir, &["fetch", "origin"]);
3751        if start_rebase {
3752            let rebase = process::Command::new("git")
3753                .args(["rebase", "origin/main"])
3754                .current_dir(&worktree_dir)
3755                .output()
3756                .unwrap();
3757            assert!(!rebase.status.success(), "precondition: rebase conflicts");
3758        }
3759
3760        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3761        let store_id = store
3762            .upsert_external(&UpsertExternal {
3763                repo: "owner/repo",
3764                ext_id: "4087",
3765                title: "Conflict",
3766                body: "",
3767                author: "",
3768                url: "",
3769                labels: &[],
3770                origin: "github",
3771            })
3772            .await
3773            .unwrap();
3774        store
3775            .update_status(store_id, TaskStatus::Routed)
3776            .await
3777            .unwrap();
3778        let mut fields = vec![
3779            ("branch", serde_json::json!(branch)),
3780            (
3781                "worktree",
3782                serde_json::json!(worktree_dir.to_string_lossy().as_ref()),
3783            ),
3784            ("merge_conflict_retries", serde_json::json!(retries)),
3785        ];
3786        if let Some(pr) = pr_number {
3787            fields.push(("pr_number", serde_json::json!(pr)));
3788        }
3789        store.set_fields(store_id, &fields).await.unwrap();
3790
3791        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3792        let engine = ProjectEngine {
3793            repo: "owner/repo".to_string(),
3794            project_dir: project_dir.clone(),
3795            backend: Arc::clone(&backend),
3796            task_manager: Arc::new(TaskManager::with_store(
3797                backend,
3798                Arc::clone(&store),
3799                "owner/repo".to_string(),
3800            )),
3801            runner: Arc::new(
3802                runner::TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store)),
3803            ),
3804            store: Arc::clone(&store),
3805        };
3806
3807        StartupConflict {
3808            _tmp: tmp,
3809            _orch_home: orch_home,
3810            prev_orch_home,
3811            project_dir,
3812            worktree_dir,
3813            branch,
3814            store,
3815            store_id,
3816            engine,
3817        }
3818    }
3819
3820    #[tokio::test]
3821    #[serial_test::serial(orch_home)]
3822    async fn startup_keeps_a_content_conflict_when_the_task_retried_a_merge() {
3823        let fx = startup_conflict_fixture(2, Some(4043), false).await;
3824        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3825            .await
3826            .unwrap();
3827
3828        assert!(fx.worktree_dir.exists(), "worktree must stay");
3829        assert!(
3830            git_ops::rebase_in_progress(&fx.worktree_dir).await,
3831            "the content conflict must stay for the agent"
3832        );
3833        let task = fx.store.get(fx.store_id).await.unwrap();
3834        assert_eq!(task.status, TaskStatus::Routed);
3835        assert_eq!(task.branch, fx.branch);
3836        assert!(!task.worktree.is_empty());
3837    }
3838
3839    #[tokio::test]
3840    #[serial_test::serial(orch_home)]
3841    async fn startup_keeps_an_in_progress_rebase_when_the_task_has_a_pr() {
3842        let fx = startup_conflict_fixture(0, Some(12), true).await;
3843        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3844            .await
3845            .unwrap();
3846
3847        assert!(fx.worktree_dir.exists(), "worktree must stay");
3848        assert!(
3849            git_ops::rebase_in_progress(&fx.worktree_dir).await,
3850            "an in-progress rebase for a PR task must stay"
3851        );
3852        let task = fx.store.get(fx.store_id).await.unwrap();
3853        assert_eq!(task.status, TaskStatus::Routed);
3854        assert_eq!(task.branch, fx.branch);
3855    }
3856
3857    #[tokio::test]
3858    #[serial_test::serial(orch_home)]
3859    async fn startup_resets_a_conflict_without_retries_or_a_pr() {
3860        let fx = startup_conflict_fixture(0, None, false).await;
3861        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3862            .await
3863            .unwrap();
3864
3865        assert!(
3866            !fx.worktree_dir.exists(),
3867            "worktree is removed when there is nothing to keep"
3868        );
3869        let task = fx.store.get(fx.store_id).await.unwrap();
3870        assert_eq!(task.status, TaskStatus::New);
3871    }
3872
3873    #[tokio::test]
3874    #[serial_test::serial(orch_home)]
3875    async fn startup_reset_keeps_branch_when_rescue_cannot_push() {
3876        let fx = startup_conflict_fixture(0, None, false).await;
3877        git(
3878            &fx.project_dir,
3879            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3880        );
3881        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3882            .await
3883            .unwrap();
3884
3885        let task = fx.store.get(fx.store_id).await.unwrap();
3886        assert_eq!(task.status, TaskStatus::New);
3887        assert_eq!(
3888            task.branch, fx.branch,
3889            "the task still names the local branch cleanup kept"
3890        );
3891        assert!(
3892            fx.worktree_dir.exists(),
3893            "rescue failure keeps the worktree on disk"
3894        );
3895    }
3896}