Skip to main content

orch/engine/
mod.rs

1//! Engine — the core orchestration loop.
2//!
3//! The engine is the central coordinator. It owns the tick loop, backend
4//! connections, channel registry, transport layer, and tmux session manager.
5//!
6//! Submodules handle the heavy lifting:
7//! - [`tick`] — core tick phases (session polling, stuck recovery, routing, dispatch)
8//! - [`sync`] — periodic sync operations (cleanup, PR review, mentions, skills)
9//! - [`review`] — PR review pipeline (review agent, auto-merge, change handling)
10//! - [`cleanup`] — worktree cleanup and merged-PR detection
11//!
12//! This file contains struct definitions, project initialization, and the
13//! main event loop (`serve()`).
14
15pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40/// Standard Orch attribution footer for issue bodies, PR bodies, and comments.
41///
42/// Append to every user-visible string posted to GitHub so activity is
43/// clearly attributed to orch rather than appearing as a human post.
44use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{remove_worktree_and_branch, remove_worktree_and_branch_for};
58use crate::engine::router::{Router, RouterConfig};
59use crate::engine::runner::worktree::{
60    abort_worktree_rebase, list_project_worktrees, rebase_worktree_on_origin_main,
61    task_id_from_worktree_name, validate_worktree_gitdir,
62};
63use crate::engine::tasks::TaskManager;
64use crate::github::http::{GhHttp, rate_limit_metrics};
65use crate::repo_context::REPO_CONTEXT;
66use crate::store::{TaskStatus, TaskStore};
67use crate::tmux::TmuxManager;
68use runner::WeightSignal;
69// AtomicBool/Ordering removed — shutdown is now immediate (reset tasks + break)
70use crate::backends::Status;
71use crate::backends::github::GitHubBackend;
72use crate::channels::github;
73use crate::channels::github::is_transient_bind_error;
74use crate::channels::github::webhook_backoff_delay;
75use crate::channels::notification;
76use crate::channels::notification::TaskNotification;
77use crate::channels::transport;
78use crate::cluster;
79use crate::cluster::local;
80use crate::engine::cooldown as engine_cooldown;
81use crate::engine::review as engine_review;
82use crate::engine::runner::git_ops;
83use crate::engine::runner::worktree;
84use crate::engine::suspend as engine_suspend;
85use crate::engine::suspend::SuspendDetector;
86use crate::engine::tasks as engine_tasks;
87use crate::github::app;
88use crate::home;
89use crate::secret_ref;
90use crate::store;
91use crate::webhook_status::WebhookStatus;
92use futures::future;
93use std::collections::HashMap;
94use std::collections::HashSet;
95use std::fmt;
96use std::fmt::Debug;
97use std::fmt::Formatter;
98use std::fs;
99use std::path::Path;
100use std::path::PathBuf;
101use std::sync::Arc;
102use std::sync::atomic::AtomicBool;
103use std::sync::atomic::AtomicU64;
104use std::sync::atomic::Ordering;
105use std::time as std_time;
106use std::time::Duration;
107use std::time::Instant;
108use std::time::SystemTime;
109use tokio::process::Command;
110use tokio::signal;
111use tokio::signal::unix;
112use tokio::signal::unix::SignalKind;
113use tokio::sync::Mutex;
114use tokio::sync::broadcast::Sender;
115use tokio::sync::broadcast::error::RecvError;
116use tokio::sync::mpsc::Receiver;
117use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
118use tokio::task;
119use tokio::time;
120
121pub fn orch_footer() -> &'static str {
122    "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
123}
124
125/// Build a standard attribution footer for GitHub comments posted by orch bots.
126///
127/// `verb` is "Created", "Reviewed", or "Commented" depending on context.
128pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
129    match model {
130        Some(m) => format!(
131            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
132        ),
133        None => format!(
134            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
135        ),
136    }
137}
138
139/// Lightweight reference tuple for channel message handling.
140type EngineRef = (
141    String,
142    Arc<dyn ExternalBackend>,
143    Arc<TaskManager>,
144    Option<Arc<TaskStore>>,
145);
146
147/// Per-project engine state.
148///
149/// Each project has its own backend, task runner, and task manager,
150/// but they share the global tmux manager, transport, and semaphore.
151#[derive(Clone)]
152pub struct ProjectEngine {
153    pub repo: String,
154    pub project_dir: PathBuf,
155    pub backend: Arc<dyn ExternalBackend>,
156    pub task_manager: Arc<TaskManager>,
157    pub runner: Arc<runner::TaskRunner>,
158    pub store: Arc<TaskStore>,
159}
160
161impl Debug for ProjectEngine {
162    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
163        f.debug_struct("ProjectEngine")
164            .field("repo", &self.repo)
165            .field("project_dir", &self.project_dir)
166            .finish_non_exhaustive()
167    }
168}
169
170/// A restart ends every review session, so an `in_review` task with no session has no reviewer.
171/// Hand it back to `needs_review` at start, so it does not wait for the stuck-review recovery
172pub(crate) async fn reset_orphaned_reviews(
173    task_manager: &TaskManager,
174    store: &Arc<TaskStore>,
175    repo: &str,
176    tmux: &TmuxManager,
177) -> usize {
178    // The store holds external and internal tasks, the second list is the legacy internal table
179    let mut ids: Vec<ExternalId> = store
180        .list_by_status(repo, TaskStatus::InReview)
181        .await
182        .unwrap_or_default()
183        .iter()
184        .map(|t| engine_tasks::store_task_to_external(t).id)
185        .collect();
186    if let Ok(legacy) = task_manager
187        .list_internal_by_status(TaskStatus::InReview)
188        .await
189    {
190        ids.extend(legacy.into_iter().map(|t| t.id));
191    }
192    ids.sort_by(|a, b| a.0.cmp(&b.0));
193    ids.dedup_by(|a, b| a.0 == b.0);
194
195    let mut reset = 0;
196    for id in &ids {
197        let session = tmux.session_name(repo, &format!("{}-review", id.0));
198
199        if tmux.session_exists(&session).await {
200            continue;
201        }
202
203        if let Err(e) = task_manager
204            .update_task_status(id, Status::NeedsReview)
205            .await
206        {
207            tracing::warn!(
208                task_id = id.0,
209                ?e,
210                "failed to reset an orphaned review at start"
211            );
212            continue;
213        }
214
215        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
216            let reason = "startup: review session gone, reset to needs_review";
217
218            if let Err(e) = store
219                .finalize_incomplete_runs(store_id, "aborted", reason)
220                .await
221            {
222                tracing::warn!(
223                    task_id = id.0,
224                    ?e,
225                    "failed to finalize the orphaned review's runs"
226                );
227            }
228        }
229
230        reset += 1;
231    }
232
233    reset
234}
235
236/// Starts the relay client when `relay.url` is set. The token is read in the background, so a
237/// passbox prompt nobody answers never holds up the engine
238fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
239    // The global config only: a project file must never point a secret at its own endpoint
240    let url = config::get_global("relay.url").unwrap_or_default();
241    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
242        return;
243    };
244
245    if url.is_empty() {
246        return;
247    }
248
249    let token_ref = config::get_global("relay.token").unwrap_or_default();
250    let notify = Arc::clone(notify);
251    tokio::spawn(async move {
252        let mut warned = false;
253        let token = loop {
254            let reference = token_ref.clone();
255            let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
256                .await
257                .map_err(anyhow::Error::from)
258                .and_then(|r| r);
259
260            match read {
261                Ok(token) if !token.is_empty() => break token,
262                Ok(_) => {
263                    tracing::warn!("relay.token is empty, the relay client stays off");
264                    return;
265                }
266                Err(e) if !warned => {
267                    tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
268                    warned = true;
269                }
270                Err(_) => {}
271            }
272
273            time::sleep(Duration::from_secs(300)).await;
274        };
275
276        relay::run(url, token, store, notify).await;
277    });
278}
279
280/**
281 * Runs `sync_tick` for every project at once, then the tail that is the same for all of them,
282 * in a spawned task. Sync time is the slowest project instead of the sum of all of them
283 * (#3864). The caller must already hold `sync_in_progress`; this releases it when done
284 **/
285#[expect(
286    clippy::too_many_arguments,
287    reason = "the spawned task takes its own handle to each piece of engine state"
288)]
289fn spawn_sync_pass(
290    project_engines: &[ProjectEngine],
291    tmux: &Arc<TmuxManager>,
292    engine_config: &EngineConfig,
293    router: &Arc<RwLock<Router>>,
294    dispatching: &Arc<dashmap::DashMap<String, String>>,
295    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
296    sync_in_progress: &Arc<AtomicBool>,
297    trigger: &'static str,
298) {
299    let engines = project_engines.to_vec();
300    let tmux = Arc::clone(tmux);
301    let engine_config = engine_config.clone();
302    let router = Arc::clone(router);
303    let dispatching = Arc::clone(dispatching);
304    let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
305    let guard = Arc::clone(sync_in_progress);
306
307    tokio::spawn(async move {
308        let started = Instant::now();
309        let mut passes = Vec::with_capacity(engines.len());
310        for engine in &engines {
311            passes.push(REPO_CONTEXT.scope(
312                engine.repo.clone(),
313                sync_one_project(
314                    engine,
315                    &tmux,
316                    &engine_config,
317                    &router,
318                    &dispatching,
319                    &auto_merge_in_flight,
320                ),
321            ));
322        }
323        future::join_all(passes).await;
324
325        sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
326        tracing::info!(
327            trigger,
328            elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
329            "sync tick complete"
330        );
331        guard.store(false, Ordering::Release);
332    });
333}
334
335async fn sync_one_project(
336    engine: &ProjectEngine,
337    tmux: &Arc<TmuxManager>,
338    engine_config: &EngineConfig,
339    router: &Arc<RwLock<Router>>,
340    dispatching: &Arc<dashmap::DashMap<String, String>>,
341    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
342) {
343    if let Err(e) = Box::pin(sync::sync_tick(
344        &engine.backend,
345        tmux,
346        &engine.repo,
347        engine_config,
348        router,
349        &engine.task_manager,
350        &engine.store,
351        dispatching,
352        auto_merge_in_flight,
353    ))
354    .await
355    {
356        tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
357    }
358}
359
360/// Milliseconds a group of tick or sync phases may take before it is worth an operator's eye
361const SLOW_PHASE_GROUP_MS: u128 = 1_000;
362
363/**
364 * Logs how long each phase of a tick or sync group took, for one project. Tick and sync time
365 * grows with the project count, and the totals alone did not say which project or phase spent
366 * it (#3864). Info once a group crosses `SLOW_PHASE_GROUP_MS`, so a quiet install stays quiet
367 **/
368pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
369    let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
370    if total_ms >= SLOW_PHASE_GROUP_MS {
371        tracing::info!(group, repo, total_ms, ?phases, "phase timings");
372    } else {
373        tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
374    }
375}
376
377/// Return the configured agent list from top-level `agents` in config.yml,
378/// falling back to [`router::config::DEFAULT_AGENTS`] when not set.
379///
380/// This is the canonical source of truth for "which agents does orch know
381/// about" — use it instead of `DEFAULT_AGENTS` directly so that
382/// Claude-compatible agents added via config (e.g. `olm`) are recognized
383/// without code changes.
384///
385/// The key is top-level (`agents:`) rather than nested (`engine.agents:`)
386/// because `config::get` serializes nested YAML arrays via `serde_norway::to_string`
387/// which produces YAML block format, not JSON — breaking downstream parsing.
388pub fn configured_agents() -> Vec<String> {
389    if let Ok(agents_str) = config::get("agents")
390        && !agents_str.is_empty()
391        && agents_str != "[]"
392    {
393        // config::get returns serde_norway::to_string for arrays, which is
394        // YAML block format ("- claude\n- codex\n..."). Parse as YAML first.
395        if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
396            && !agents_arr.is_empty()
397        {
398            return agents_arr;
399        }
400        // Fallback: try JSON (inline arrays) or comma-separated
401        if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
402            && !agents_arr.is_empty()
403        {
404            return agents_arr;
405        }
406    }
407    router::config::DEFAULT_AGENTS
408        .iter()
409        .map(ToString::to_string)
410        .collect()
411}
412
413/// Seconds a live session may run past its own timeout before stuck recovery takes it. The
414/// `timeout` wrapper ends the agent at the limit, and the runner still collects the result
415pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
416
417/// The limit the `timeout` wrapper puts on one agent run, per complexity
418#[derive(Clone, Copy, Debug, PartialEq, Eq)]
419pub struct RunTimeouts {
420    pub base: u64,
421    pub medium: u64,
422    pub complex: u64,
423}
424
425impl Default for RunTimeouts {
426    fn default() -> Self {
427        Self {
428            base: 1800,
429            medium: 1800,
430            complex: 1800,
431        }
432    }
433}
434
435impl RunTimeouts {
436    /// `workflow.timeout_seconds` (at least 1800), raised by
437    /// `workflow.timeout_by_complexity.{medium,complex}` but never below the base
438    pub fn from_config() -> Self {
439        let read =
440            |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
441        let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
442        Self {
443            base,
444            medium: read("workflow.timeout_by_complexity.medium")
445                .unwrap_or(base)
446                .max(base),
447            complex: read("workflow.timeout_by_complexity.complex")
448                .unwrap_or(base)
449                .max(base),
450        }
451    }
452
453    pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
454        match complexity {
455            Some("complex") => self.complex,
456            Some("medium") => self.medium,
457            _ => self.base,
458        }
459    }
460}
461
462/// Engine configuration.
463#[derive(Clone, Debug)]
464pub struct EngineConfig {
465    /// Main tick interval
466    pub tick_interval: Duration,
467    /// GitHub sync interval (cleanup, PR review, mentions)
468    pub sync_interval: Duration,
469    /// Webhook health check interval (seconds)
470    pub webhook_health_check_interval: Option<Duration>,
471    /// Maximum parallel task executions
472    pub max_parallel: usize,
473    /// Stuck task timeout for tasks with an active tmux session (seconds)
474    pub stuck_timeout: u64,
475    /// Stuck task timeout for tasks with no active tmux session (seconds).
476    /// Shorter than `stuck_timeout` because no session means the agent has already exited.
477    pub no_session_stuck_timeout: u64,
478    /// Stuck task timeout for in_review tasks with no active tmux session (seconds).
479    /// Longer than `no_session_stuck_timeout` because a review agent ends its tmux session on
480    /// normal completion before its result reaches the engine. A shorter limit resets a task
481    /// whose finished review is still being delivered, and the result is dropped as stale.
482    /// `from_config` keeps it above the review agent's own timeout plus
483    /// `review::REVIEW_RESULT_DELIVERY_SECS`.
484    pub in_review_no_session_stuck_timeout: u64,
485    /// Per-run agent timeouts. A live session is not stuck before its own timeout runs out
486    pub run_timeouts: RunTimeouts,
487    /// Auto-create follow-up tasks when PR reviews request changes
488    pub auto_create_followup_on_changes: bool,
489    /// Graceful shutdown timeout — how long to wait for running agents before exiting.
490    pub graceful_shutdown_timeout: Duration,
491    /// Grace period before silence detection kicks in (seconds).
492    /// Allows time for agent startup, model download, API handshake.
493    pub silence_grace_period: u64,
494    /// Cooldown duration for a model detected as silent (seconds).
495    pub silence_cooldown: u64,
496    /// How often to check for a newer orch release and notify channels (seconds).
497    /// Set to 0 to disable. Default: 3600 (1 hour).
498    pub upgrade_check_interval: u64,
499    /// Seconds of continuous startup failure (GitHub unreachable / auth errors)
500    /// before the engine escalates from WARN to ERROR-level logging and sends a
501    /// one-time push notification to configured channels. Set to 0 to disable.
502    /// Default: 3600 (1 hour).
503    pub startup_failure_escalation_secs: u64,
504}
505
506impl Default for EngineConfig {
507    fn default() -> Self {
508        Self {
509            tick_interval: Duration::from_secs(10),
510            sync_interval: Duration::from_secs(45),
511            webhook_health_check_interval: Some(Duration::from_secs(60)),
512            max_parallel: 4,
513            stuck_timeout: 1800,
514            no_session_stuck_timeout: 600,
515            in_review_no_session_stuck_timeout: 1920,
516            run_timeouts: RunTimeouts::default(),
517            auto_create_followup_on_changes: true,
518            graceful_shutdown_timeout: Duration::from_secs(600),
519            silence_grace_period: 300,
520            silence_cooldown: 3600,
521            upgrade_check_interval: 3600,
522            startup_failure_escalation_secs: 3600,
523        }
524    }
525}
526
527impl EngineConfig {
528    /// Stuck limit for an in_progress task whose session is alive: `stuck_timeout`, raised to the
529    /// run's own timeout plus `RUN_TIMEOUT_GRACE_SECS`
530    pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
531        self.stuck_timeout.max(
532            self.run_timeouts
533                .for_complexity(complexity)
534                .saturating_add(RUN_TIMEOUT_GRACE_SECS),
535        )
536    }
537
538    /// Load engine configuration from config files, falling back to defaults.
539    pub fn from_config() -> Self {
540        let mut config = Self::default();
541
542        if let Ok(val) = config::get("engine.tick_interval") {
543            if let Ok(secs) = val.parse::<u64>() {
544                config.tick_interval = Duration::from_secs(secs);
545            } else {
546                tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
547            }
548        }
549
550        if let Ok(val) = config::get("engine.sync_interval") {
551            if let Ok(secs) = val.parse::<u64>() {
552                config.sync_interval = Duration::from_secs(secs);
553            } else {
554                tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
555            }
556        }
557
558        if let Ok(val) = config::get("engine.max_parallel") {
559            if let Ok(n) = val.parse::<usize>() {
560                config.max_parallel = n;
561            } else {
562                tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
563            }
564        }
565
566        if let Ok(val) = config::get("engine.stuck_timeout") {
567            if let Ok(secs) = val.parse::<u64>() {
568                config.stuck_timeout = secs;
569            } else {
570                tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
571            }
572        }
573
574        if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
575            if let Ok(secs) = val.parse::<u64>() {
576                config.no_session_stuck_timeout = secs;
577            } else {
578                tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
579            }
580        }
581
582        let mut in_review_from_config = false;
583        if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
584            if let Ok(secs) = val.parse::<u64>() {
585                config.in_review_no_session_stuck_timeout = secs;
586                in_review_from_config = true;
587            } else {
588                tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
589            }
590        }
591
592        // Keep the no-session review limit over the review agent's own timeout plus delivery
593        // time, whatever either is set to. Under that, recovery resets a task whose finished
594        // review is still on its way to the engine and the result is dropped as stale
595        let review_timeout = engine_review::review_timeout_seconds();
596        let floored = engine_review::no_session_review_floor(
597            config.in_review_no_session_stuck_timeout,
598            review_timeout,
599        );
600        if floored != config.in_review_no_session_stuck_timeout {
601            if in_review_from_config {
602                tracing::warn!(
603                    configured_secs = config.in_review_no_session_stuck_timeout,
604                    review_timeout_secs = review_timeout,
605                    raised_to_secs = floored,
606                    "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
607                );
608            } else {
609                tracing::debug!(
610                    review_timeout_secs = review_timeout,
611                    raised_to_secs = floored,
612                    "raised the default in_review no-session limit over a long review timeout"
613                );
614            }
615            config.in_review_no_session_stuck_timeout = floored;
616        }
617
618        config.run_timeouts = RunTimeouts::from_config();
619
620        if let Ok(val) = config::get("engine.webhook_health_check_interval") {
621            if let Ok(secs) = val.parse::<u64>() {
622                if secs > 0 {
623                    config.webhook_health_check_interval = Some(Duration::from_secs(secs));
624                } else {
625                    config.webhook_health_check_interval = None;
626                }
627            } else {
628                tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
629            }
630        }
631
632        if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
633            config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
634        }
635
636        if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
637            if let Ok(secs) = val.parse::<u64>() {
638                config.graceful_shutdown_timeout = Duration::from_secs(secs);
639            } else {
640                tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
641            }
642        }
643
644        if let Ok(val) = config::get("engine.silence_grace_period") {
645            if let Ok(secs) = val.parse::<u64>() {
646                config.silence_grace_period = secs;
647            } else {
648                tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
649            }
650        }
651
652        if let Ok(val) = config::get("engine.silence_cooldown") {
653            if let Ok(secs) = val.parse::<u64>() {
654                config.silence_cooldown = secs;
655            } else {
656                tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
657            }
658        }
659
660        if let Ok(val) = config::get("engine.upgrade_check_interval") {
661            if let Ok(secs) = val.parse::<u64>() {
662                config.upgrade_check_interval = secs;
663            } else {
664                tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
665            }
666        }
667
668        if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
669            if let Ok(secs) = val.parse::<u64>() {
670                config.startup_failure_escalation_secs = secs;
671            } else {
672                tracing::warn!(
673                    key = "engine.startup_failure_escalation_secs",
674                    value = %val,
675                    default_secs = config.startup_failure_escalation_secs,
676                    "invalid value for engine.startup_failure_escalation_secs, using default"
677                );
678            }
679        }
680
681        config
682    }
683}
684
685/// KV key holding the epoch second the GitHub core window resets, so `orch task status`
686/// can show the limit from a separate process.
687pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
688
689/// Format an epoch second as a local `HH:MM:SS` for operator-facing output.
690pub fn format_rate_limited_until(reset_at: i64) -> String {
691    chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
692        || reset_at.to_string(),
693        |t| {
694            t.with_timezone(&chrono::Local)
695                .format("%H:%M:%S %Z")
696                .to_string()
697        },
698    )
699}
700
701/// Announce a GitHub rate-limit window on every operator surface: the log (`orch log`),
702/// a KV flag (`orch task status`) and the channels (Telegram, Discord).
703async fn report_rate_limited(
704    reset_at: i64,
705    store: Option<&Arc<TaskStore>>,
706    transport: Option<&Arc<Transport>>,
707) {
708    let m = rate_limit_metrics();
709    let until = format_rate_limited_until(reset_at);
710    tracing::warn!(
711        reset_at,
712        rate_limit_hits = m.hits,
713        skipped_calls = m.skipped_calls,
714        five_xx_hits = m.five_xx_hits,
715        rest_remaining = ?m.rest_remaining,
716        graphql_remaining = ?m.graphql_remaining,
717        "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
718    );
719
720    if let Some(store) = store
721        && let Err(e) = store
722            .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
723            .await
724    {
725        tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
726    }
727
728    if let Some(transport) = transport {
729        transport.push_notification(TaskNotification {
730            task_id: "github".to_string(),
731            title: "GitHub rate limit".to_string(),
732            status: "blocked".to_string(),
733            agent: "orch".to_string(),
734            duration_seconds: 0.0,
735            summary: format!(
736                "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
737            ),
738            repo: None,
739            notify_target: None,
740            pr_number: None,
741        });
742    }
743}
744
745/// Grow or shrink the dispatch pool toward `target` without cancelling running work.
746///
747/// Growing adds permits at once. Shrinking can only take back the permits that
748/// are free right now, so the return value is the limit actually in force; call
749/// again as running tasks release theirs
750fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
751    if target > in_force {
752        semaphore.add_permits(target - in_force);
753        return target;
754    }
755
756    if target < in_force {
757        return in_force - semaphore.forget_permits(in_force - target);
758    }
759
760    in_force
761}
762
763/// Config subtrees the engine only reads at startup: the channel gateways hold
764/// long-lived websockets and the webhook listener binds its port once.
765const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
766
767/// Read the restart-only subtrees so a later reload can tell they changed.
768fn restart_only_snapshot() -> Vec<Option<String>> {
769    RESTART_ONLY_KEYS
770        .iter()
771        .map(|key| config::get(key).ok())
772        .collect()
773}
774
775/// Report the restart-only keys an edit changed. Only the key is logged, never
776/// the value, because `channels.*` holds bot tokens
777fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
778    let changed: Vec<&'static str> = RESTART_ONLY_KEYS
779        .iter()
780        .zip(old)
781        .zip(new)
782        .filter(|((_, old), new)| old != new)
783        .map(|((key, _), _)| *key)
784        .collect();
785
786    for key in &changed {
787        tracing::info!(key = key, "config changed, takes effect after restart");
788    }
789
790    changed
791}
792
793/// Log every engine setting whose value changed across a config reload, so
794/// `orch log` shows what the operator's edit actually did.
795fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
796    macro_rules! log_changed {
797        ($($key:literal => $field:ident),+ $(,)?) => {
798            $(if old.$field != new.$field {
799                tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
800            })+
801        };
802    }
803
804    log_changed!(
805        "engine.tick_interval" => tick_interval,
806        "engine.sync_interval" => sync_interval,
807        "engine.webhook_health_check_interval" => webhook_health_check_interval,
808        "engine.max_parallel" => max_parallel,
809        "engine.stuck_timeout" => stuck_timeout,
810        "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
811        "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
812        "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
813        "engine.silence_grace_period" => silence_grace_period,
814        "engine.silence_cooldown" => silence_cooldown,
815        "engine.upgrade_check_interval" => upgrade_check_interval,
816        "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
817        "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
818    );
819}
820
821fn should_run_upgrade_check(
822    last_upgrade_check: Option<Instant>,
823    upgrade_check_interval: u64,
824) -> bool {
825    if upgrade_check_interval == 0 {
826        return false;
827    }
828
829    last_upgrade_check
830        .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
831}
832
833/// Connect one project's backend and build its task manager and runner over
834/// the shared store.
835///
836/// `event_tx` wires status transitions into the event bus; pass `None` during
837/// startup, where the bus does not exist yet and task managers are rebuilt
838/// once it does
839async fn connect_project_engine(
840    repo: String,
841    project_dir: PathBuf,
842    store: Arc<TaskStore>,
843    event_tx: Option<Sender<events::TaskEvent>>,
844) -> anyhow::Result<ProjectEngine> {
845    let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
846
847    // A failed health check (rate limit, network, auth) must not drop the project: it would have
848    // no review subscriber until the next restart. Later ticks hit the API again and retry on
849    // their own, so build the engine and warn while the check keeps failing.
850    match backend.health_check().await {
851        Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
852        Err(e) => tracing::warn!(
853            repo = %repo,
854            error = %e,
855            "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
856        ),
857    }
858
859    let task_manager = Arc::new(match event_tx {
860        Some(tx) => {
861            TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
862        }
863        None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
864    });
865    let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
866
867    Ok(ProjectEngine {
868        repo,
869        project_dir,
870        backend,
871        task_manager,
872        runner,
873        store,
874    })
875}
876
877/**
878 * Split the configured project list into the ones not running yet and the names of the ones that
879 * went away. Kept separate from `sync_project_engines` so the comparison can be tested without a
880 * GitHub backend: getting it wrong either spawns a second set of subscribers for a project already
881 * running, or silently never picks a new one up
882 **/
883fn diff_configured_projects(
884    configured: &[(String, PathBuf)],
885    active_repos: &HashSet<String>,
886) -> (Vec<(String, PathBuf)>, Vec<String>) {
887    let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
888
889    let added = configured
890        .iter()
891        .filter(|(repo, _)| !active_repos.contains(repo))
892        .cloned()
893        .collect();
894    let removed = active_repos
895        .iter()
896        .filter(|repo| !configured_repos.contains(repo.as_str()))
897        .cloned()
898        .collect();
899
900    (added, removed)
901}
902
903/// Pick up projects added to `projects:` since startup, and report the ones
904/// that were removed.
905///
906/// A removed project keeps running until the next restart: its subscriber
907/// tasks and any agent it already dispatched outlive the config read
908#[expect(
909    clippy::too_many_arguments,
910    reason = "a new project engine needs every shared engine handle"
911)]
912async fn sync_project_engines(
913    project_engines: &mut Vec<ProjectEngine>,
914    active_repos: &mut HashSet<String>,
915    event_bus: &events::EventBus,
916    tmux: &Arc<TmuxManager>,
917    capture: &Arc<CaptureService>,
918    semaphore: &Arc<Semaphore>,
919    review_semaphore: &Arc<Semaphore>,
920    weight_tx: &mpsc::Sender<WeightSignal>,
921    router: &Arc<RwLock<Router>>,
922    dispatching: &Arc<dashmap::DashMap<String, String>>,
923) {
924    let configured = match config::get_projects_with_paths() {
925        Ok(p) if !p.is_empty() => p,
926        Ok(_) => return,
927        Err(e) => {
928            tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
929            return;
930        }
931    };
932    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
933        return;
934    };
935
936    let (added, removed) = diff_configured_projects(&configured, active_repos);
937    for repo in removed {
938        tracing::info!(
939            repo = %repo,
940            "project removed from config, takes effect after restart"
941        );
942    }
943
944    for (repo, project_dir) in added {
945        config::watch(&project_dir.join(".orch.yml"));
946        let engine = match connect_project_engine(
947            repo.clone(),
948            project_dir,
949            Arc::clone(&store),
950            Some(event_bus.sender()),
951        )
952        .await
953        {
954            Ok(engine) => engine,
955            Err(e) => {
956                // A failed health check no longer lands here, so this is a backend that could
957                // not be built at all, such as a repo slug that is not `owner/name`
958                tracing::warn!(
959                    repo = %repo,
960                    err = %e,
961                    "project added to config but its backend could not be built, retrying on the next config change"
962                );
963                continue;
964            }
965        };
966
967        subscribers::dispatch::spawn(
968            event_bus.subscribe(),
969            Arc::clone(&engine.backend),
970            Arc::clone(tmux),
971            Arc::clone(&engine.runner),
972            Arc::clone(capture),
973            Arc::clone(semaphore),
974            Arc::clone(&engine.task_manager),
975            weight_tx.clone(),
976            Arc::clone(router),
977            Arc::clone(dispatching),
978            Arc::clone(&engine.store),
979            engine.repo.clone(),
980        );
981        subscribers::review::spawn(
982            event_bus.subscribe(),
983            Arc::clone(&engine.backend),
984            Arc::clone(tmux),
985            Arc::clone(review_semaphore),
986            Arc::clone(&engine.task_manager),
987            Arc::clone(router),
988            Arc::clone(dispatching),
989            Arc::clone(&engine.store),
990            engine.repo.clone(),
991        );
992        subscribers::unblock::spawn(
993            event_bus.subscribe(),
994            Arc::clone(&engine.backend),
995            Arc::clone(&engine.task_manager),
996            Arc::clone(&engine.store),
997            engine.repo.clone(),
998        );
999
1000        active_repos.insert(repo.clone());
1001        project_engines.push(engine);
1002        tracing::info!(repo = %repo, "project added from config, now syncing");
1003    }
1004}
1005
1006/// Initialize all project engines from config.
1007///
1008/// Returns a vector of ProjectEngine, one for each configured project.
1009async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1010    let projects = config::get_projects_with_paths()?;
1011    let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1012    tracing::info!(repos = ?repos, "loading projects from config");
1013
1014    if projects.is_empty() {
1015        let config_path = home::config_path().map_or_else(
1016            |_| "~/.orch/config.yml".to_string(),
1017            |p| p.display().to_string(),
1018        );
1019        anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1020    }
1021
1022    let mut engines = Vec::new();
1023
1024    // Initialize unified task store once — all project engines share the same SQLite file.
1025    // Creating one pool here avoids N separate connection pools for the same database.
1026    let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1027    if cluster::enabled() {
1028        cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1029        local::serve((*store).clone()).await?;
1030    }
1031
1032    // Load persisted model cooldowns and register store for future writes.
1033    // Runs once here so the global cooldown store is not overwritten per project.
1034    engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1035
1036    for (repo, project_dir) in projects {
1037        tracing::info!(repo = %repo, "initializing project engine");
1038        config::watch(&project_dir.join(".orch.yml"));
1039
1040        engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1041    }
1042
1043    Ok(engines)
1044}
1045
1046async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1047    for engine in project_engines {
1048        let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1049        let repo_root_path = PathBuf::from(&repo_root);
1050        let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1051        let worktrees = list_project_worktrees(&repo_root_path).await?;
1052
1053        // Fetch refs once for all worktrees in this project, so rebase below
1054        // operates on up-to-date origin/* refs without N sequential fetches.
1055        if let Err(e) = git_ops::git()
1056            .await
1057            .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1058            .output_with_context()
1059            .await
1060        {
1061            tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1062        }
1063
1064        let mut orphans_removed: u32 = 0;
1065        let mut invalid_removed: u32 = 0;
1066        let mut cleaned: u32 = 0;
1067        let mut valid_kept: u32 = 0;
1068
1069        for worktree_dir in worktrees {
1070            let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1071                continue;
1072            };
1073
1074            // Validate gitdir link before doing anything else — a broken .git file
1075            // means git commands on this worktree will fail with "not a git repository".
1076            if !validate_worktree_gitdir(&worktree_dir).await {
1077                tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1078                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1079                    .await;
1080                invalid_removed += 1;
1081                continue;
1082            }
1083
1084            let Some(task_id) = task_id_from_worktree_name(name) else {
1085                tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1086                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1087                    .await;
1088                orphans_removed += 1;
1089                continue;
1090            };
1091
1092            let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1093                tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1094                remove_worktree_and_branch(
1095                    &task_id,
1096                    &worktree_dir,
1097                    Some(name),
1098                    &repo_root_path,
1099                    false,
1100                )
1101                .await;
1102                orphans_removed += 1;
1103                continue;
1104            };
1105
1106            let task = engine.store.get(store_id).await?;
1107            let branch_name = if task.branch.is_empty() {
1108                name
1109            } else {
1110                task.branch.as_str()
1111            };
1112
1113            match task.status {
1114                TaskStatus::New
1115                | TaskStatus::Routed
1116                | TaskStatus::InProgress
1117                | TaskStatus::NeedsReview
1118                | TaskStatus::InReview => {
1119                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1120                    match rebase_worktree_on_origin_main(&worktree_dir, &default_branch).await {
1121                        Err(e) => {
1122                            // If rebase failed we already made efforts to stash and
1123                            // restore changes safely. Log the error and reset the
1124                            // task as before. The worktree removal path is used
1125                            // to avoid leaving corrupted worktrees around.
1126                            tracing::warn!(repo = %engine.repo, task_id = %task_id, err = %e, "startup rebase failed, resetting task");
1127                            abort_worktree_rebase(&worktree_dir).await;
1128                            let keep_remote = task.pr_number.is_some();
1129                            remove_worktree_and_branch(
1130                                &task_id,
1131                                &worktree_dir,
1132                                Some(branch_name),
1133                                &repo_root_path,
1134                                keep_remote,
1135                            )
1136                            .await;
1137                            match engine.store.resolve_task_id(&engine.repo, &task_id).await {
1138                                Ok(Some(reset_id)) => {
1139                                    if let Err(e) = engine.store.reset_to_new(reset_id).await {
1140                                        tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1141                                    }
1142                                }
1143                                Ok(None) => {
1144                                    tracing::warn!(
1145                                        task_id,
1146                                        "task not found in store during reconciliation"
1147                                    );
1148                                }
1149                                Err(e) => {
1150                                    tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1151                                }
1152                            }
1153                            if let Err(e) = engine
1154                                .task_manager
1155                                .update_task_status(&ExternalId(task_id.clone()), Status::New)
1156                                .await
1157                            {
1158                                tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1159                            }
1160                            cleaned += 1;
1161                        }
1162                        _ => {
1163                            valid_kept += 1;
1164                        }
1165                    }
1166                }
1167                // A blocked or waiting task resumes from this work, and without a PR the branch
1168                // may hold the only copy of it. An approved task's PR is updated on GitHub
1169                TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1170                    valid_kept += 1;
1171                }
1172                TaskStatus::Done => {
1173                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1174                    remove_worktree_and_branch_for(
1175                        &task_id,
1176                        &worktree_dir,
1177                        Some(branch_name),
1178                        &repo_root_path,
1179                        false,
1180                        task.pr_number.is_some(),
1181                    )
1182                    .await;
1183                    cleaned += 1;
1184                }
1185            }
1186        }
1187
1188        tracing::info!(
1189            repo = %engine.repo,
1190            orphans_removed,
1191            invalid_removed,
1192            cleaned,
1193            valid_kept,
1194            "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1195        );
1196
1197        // Prune stale .git/worktrees/ entries whose directories no longer exist.
1198        // This handles entries left behind by crashes or manual directory removal.
1199        let prune = Command::new("git")
1200            .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1201            .output_with_context()
1202            .await;
1203        match prune {
1204            Ok(output) if output.status.success() => {
1205                tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1206            }
1207            Ok(output) => {
1208                let stderr = String::from_utf8_lossy(&output.stderr);
1209                tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1210            }
1211            Err(e) => {
1212                tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1213            }
1214        }
1215    }
1216
1217    Ok(())
1218}
1219
1220/// Sync Fibonacci estimates stored in SQLite to the GitHub Projects board for
1221/// all tasks that have a positive estimate.
1222///
1223/// Runs once on startup to catch tasks that were routed before the estimate
1224/// sync feature was available or before `project_estimate_field_id` was
1225/// configured. The underlying mutations are idempotent, so re-syncing an
1226/// already-correct estimate is harmless.
1227async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1228    for engine in project_engines {
1229        let tasks = match engine.store.list_external_tasks_with_estimates().await {
1230            Ok(t) => t,
1231            Err(e) => {
1232                tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1233                continue;
1234            }
1235        };
1236
1237        if tasks.is_empty() {
1238            continue;
1239        }
1240
1241        let mut synced: u32 = 0;
1242        let mut failed: u32 = 0;
1243        for (external_id, estimate) in &tasks {
1244            match engine
1245                .backend
1246                .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1247                .await
1248            {
1249                Ok(()) => synced += 1,
1250                Err(e) => {
1251                    tracing::debug!(
1252                        repo = %engine.repo,
1253                        task_id = %external_id,
1254                        estimate,
1255                        err = %e,
1256                        "startup estimate reconciliation: sync failed"
1257                    );
1258                    failed += 1;
1259                }
1260            }
1261        }
1262
1263        if synced > 0 || failed > 0 {
1264            tracing::info!(
1265                repo = %engine.repo,
1266                synced,
1267                failed,
1268                "startup estimate reconciliation complete"
1269            );
1270        }
1271    }
1272}
1273
1274/// Read per-project channel configuration from `.orch.yml`.
1275fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1276    let config_path = project_dir.join(".orch.yml");
1277    if !config_path.exists() {
1278        return ProjectChannelConfig::default();
1279    }
1280    let content = match fs::read_to_string(&config_path) {
1281        Ok(c) => c,
1282        Err(e) => {
1283            tracing::warn!(
1284                path = %config_path.display(),
1285                error = %e,
1286                "failed to read .orch.yml; channel config will use defaults"
1287            );
1288            return ProjectChannelConfig::default();
1289        }
1290    };
1291    let val: serde_norway::Value = match serde_norway::from_str(&content) {
1292        Ok(v) => v,
1293        Err(e) => {
1294            tracing::warn!(
1295                path = %config_path.display(),
1296                error = %e,
1297                "failed to parse .orch.yml; channel config will use defaults"
1298            );
1299            return ProjectChannelConfig::default();
1300        }
1301    };
1302    let Some(channels) = val.get("channels") else {
1303        return ProjectChannelConfig::default();
1304    };
1305    let get_str = |section: &str, key: &str| -> Option<String> {
1306        channels
1307            .get(section)
1308            .and_then(|s| s.get(key))
1309            .and_then(|v| v.as_str())
1310            .map(String::from)
1311    };
1312    ProjectChannelConfig {
1313        telegram_topic_id: get_str("telegram", "topic_id"),
1314        discord_channel_id: get_str("discord", "channel_id"),
1315    }
1316}
1317
1318/// Fetch the latest orch release tag from GitHub via the `gh` CLI (async).
1319///
1320/// Returns the version string (without the leading "v"), or `None` when the
1321/// check fails (no network, unauthenticated, `gh` not installed, etc.).
1322async fn fetch_latest_release_version() -> Option<String> {
1323    let output = Command::new("gh")
1324        .args([
1325            "api",
1326            "repos/gabrielkoerich/orch/releases/latest",
1327            "--jq",
1328            ".tag_name",
1329        ])
1330        .output()
1331        .await
1332        .ok()?;
1333    if !output.status.success() {
1334        return None;
1335    }
1336    let tag = String::from_utf8(output.stdout).ok()?;
1337    let version = tag.trim().trim_start_matches('v').to_string();
1338    if version.is_empty() {
1339        None
1340    } else {
1341        Some(version)
1342    }
1343}
1344
1345/// Check if a newer orch release is available and send channel notifications
1346/// asking the operator to upgrade manually. Upgrading is operator-only — the
1347/// service never runs `brew` itself.
1348///
1349/// Uses the store's KV to throttle checks (`upgrade:last_check_at`) and
1350/// deduplicate notifications (`upgrade:last_notified_version`). Each distinct
1351/// latest version is notified about once; the notification repeats only when
1352/// an even newer version is released.
1353async fn check_and_notify_upgrade(
1354    store: &Arc<TaskStore>,
1355    channels: &Arc<ChannelRegistry>,
1356    current_version: &str,
1357    check_interval: Duration,
1358) {
1359    let last_check_key = "upgrade:last_check_at";
1360    let last_notified_key = "upgrade:last_notified_version";
1361
1362    // Throttle: only check once per interval.
1363    let now = chrono::Utc::now().timestamp();
1364    if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1365        && let Ok(last_check) = last_check_str.parse::<i64>()
1366        && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1367    {
1368        return;
1369    }
1370
1371    // Record check timestamp (best-effort).
1372    let _ = store.kv_set(last_check_key, &now.to_string()).await;
1373
1374    let Some(latest) = fetch_latest_release_version().await else {
1375        tracing::debug!("upgrade check: failed to fetch latest release version");
1376        return;
1377    };
1378
1379    if latest == current_version {
1380        tracing::debug!(current = %current_version, "orch is up to date");
1381        return;
1382    }
1383
1384    tracing::warn!(
1385        current_version = %current_version,
1386        latest_version = %latest,
1387        "orch upgrade available"
1388    );
1389
1390    // Operator-only upgrade: send channel notifications so the operator can
1391    // upgrade on their own schedule. The service never runs brew itself.
1392
1393    // Deduplicate: don't re-notify for the same latest version.
1394    if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1395        && last_notified == latest
1396    {
1397        tracing::debug!(latest = %latest, "upgrade already notified");
1398        return;
1399    }
1400
1401    let msg_telegram = format!(
1402        "⚠️ <b>Orch Upgrade Available</b>\n\n\
1403         Service: <code>{current_version}</code>\n\
1404         Latest:  <code>{latest}</code>\n\n\
1405         Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1406    );
1407
1408    let msg_discord = format!(
1409        "⚠️ **Orch Upgrade Available**\n\n\
1410         Service: `{current_version}`\n\
1411         Latest:  `{latest}`\n\n\
1412         Run: `brew update && brew upgrade orch && brew services restart orch`"
1413    );
1414
1415    let msg_slack = format!(
1416        "⚠️ *Orch Upgrade Available*\n\n\
1417         Service: `{current_version}`\n\
1418         Latest:  `{latest}`\n\n\
1419         Run: `brew update && brew upgrade orch && brew services restart orch`"
1420    );
1421
1422    for channel in channels.iter() {
1423        let (body, metadata, topic_id) = match channel.name() {
1424            "telegram" => (
1425                msg_telegram.clone(),
1426                serde_json::json!({"preformatted_html": true}),
1427                None,
1428            ),
1429            "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1430            "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1431            _ => continue,
1432        };
1433
1434        let msg = OutgoingMessage {
1435            thread_id: "upgrade".to_string(),
1436            body,
1437            reply_to: None,
1438            metadata,
1439            topic_id,
1440        };
1441
1442        if let Err(e) = channel.send(&msg).await {
1443            tracing::warn!(
1444                channel = channel.name(),
1445                ?e,
1446                "failed to send upgrade notification"
1447            );
1448        }
1449    }
1450
1451    // Record that we notified for this latest version.
1452    let _ = store.kv_set(last_notified_key, &latest).await;
1453}
1454
1455/// Send a one-time escalation notification when the startup retry loop has been
1456/// spinning for longer than `startup_failure_escalation_secs`.
1457///
1458/// This is called *before* the channel registry is initialised (the retry loop
1459/// sits above it in `serve()`), so we construct channel objects directly from
1460/// config — mirroring the approach used during normal channel registration, but
1461/// without relying on `ChannelRegistry`.
1462async fn send_startup_escalation_notification(elapsed_secs: u64) {
1463    #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1464    let duration = notification::format_duration(elapsed_secs as f64);
1465
1466    let msg_telegram = format!(
1467        "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1468         The engine has been retrying backend connections for {duration}\n\
1469         without success. All task processing, routing, and scheduled\n\
1470         jobs are paused. Check service logs with `orch log -f`."
1471    );
1472
1473    let msg_discord = format!(
1474        "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1475         The engine has been retrying backend connections for {duration}\n\
1476         without success. All task processing, routing, and scheduled\n\
1477         jobs are paused. Check service logs with `orch log -f`."
1478    );
1479
1480    let msg_slack = format!(
1481        "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1482         The engine has been retrying backend connections for {duration}\n\
1483         without success. All task processing, routing, and scheduled\n\
1484         jobs are paused. Check service logs with `orch log -f`."
1485    );
1486
1487    // Telegram
1488    let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1489    let tg_chat_id = config::get("channels.telegram.chat_id");
1490    if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1491        && !token.is_empty()
1492        && !chat_id.is_empty()
1493        && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1494    {
1495        let msg = OutgoingMessage {
1496            thread_id: "startup_escalation".to_string(),
1497            body: msg_telegram.clone(),
1498            reply_to: None,
1499            metadata: serde_json::json!({ "preformatted_html": true }),
1500            topic_id: config::get("channels.telegram.general_topic_id").ok(),
1501        };
1502        match channel.send(&msg).await {
1503            Err(e) => {
1504                tracing::warn!(?e, "failed to send startup escalation via telegram");
1505            }
1506            _ => {
1507                tracing::info!("startup escalation notification sent to telegram");
1508            }
1509        }
1510    }
1511
1512    // Discord
1513    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1514        let channel_id = config::get("channels.discord.channel_id")
1515            .or_else(|_| config::get("channels.discord.general_channel_id"));
1516        if !token.is_empty()
1517            && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1518        {
1519            let shard_id = config::get("channels.discord.shard_id")
1520                .ok()
1521                .and_then(|v| v.parse::<u64>().ok())
1522                .unwrap_or(0);
1523            let shard_count = config::get("channels.discord.shard_count")
1524                .ok()
1525                .and_then(|v| v.parse::<u64>().ok())
1526                .unwrap_or(1);
1527            if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1528                let msg = OutgoingMessage {
1529                    thread_id: "startup_escalation".to_string(),
1530                    body: msg_discord.clone(),
1531                    reply_to: None,
1532                    metadata: serde_json::Value::Null,
1533                    topic_id: None,
1534                };
1535                match gateway.send(&msg).await {
1536                    Err(e) => {
1537                        tracing::warn!(?e, "failed to send startup escalation via discord");
1538                    }
1539                    _ => {
1540                        tracing::info!("startup escalation notification sent to discord");
1541                    }
1542                }
1543            }
1544        }
1545    }
1546
1547    // Slack
1548    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1549        let channel_id = config::get("channels.slack.channel_id");
1550        if !token.is_empty()
1551            && let Ok(cid) = channel_id
1552            && let Ok(channel) = SlackChannel::new(token, Some(cid))
1553        {
1554            let msg = OutgoingMessage {
1555                thread_id: "startup_escalation".to_string(),
1556                body: msg_slack.clone(),
1557                reply_to: None,
1558                metadata: serde_json::Value::Null,
1559                topic_id: None,
1560            };
1561            match channel.send(&msg).await {
1562                Err(e) => {
1563                    tracing::warn!(?e, "failed to send startup escalation via slack");
1564                }
1565                _ => {
1566                    tracing::info!("startup escalation notification sent to slack");
1567                }
1568            }
1569        }
1570    }
1571}
1572
1573/// Start the orch service.
1574///
1575/// This is the main entry point — called by `orch serve`.
1576pub async fn serve() -> anyhow::Result<()> {
1577    let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1578    if killed > 0 {
1579        tracing::info!(count = killed, "terminated stale orch serve processes");
1580    }
1581
1582    tracing::info!("orch engine starting");
1583
1584    // Pre-create standard directories with async I/O so subsequent synchronous
1585    // callers (state_dir, orch_home, etc.) find them already present and their
1586    // create_dir_all calls become fast no-ops.
1587    if let Err(e) = home::init_dirs().await {
1588        tracing::warn!("failed to initialize orch directories: {e}");
1589    }
1590
1591    // Pre-warm the config cache with async I/O so get() / get_list() calls in
1592    // hot async paths never block a Tokio thread on std::fs::read_to_string.
1593    if let Err(e) = config::warm_cache().await {
1594        tracing::warn!("failed to warm config cache: {e}");
1595    }
1596
1597    let mut config = EngineConfig::from_config();
1598
1599    tracing::info!("internal database ready");
1600
1601    // Initialize project engines — retry with backoff so a network outage at
1602    // startup doesn't cause a crash-loop (launchd KeepAlive would restart us
1603    // immediately, filling the error log with 600+ identical messages).
1604    //
1605    // IMPORTANT: errors from init_project_engines() must NEVER propagate past
1606    // this loop to main().  If they did, anyhow would write "Error: …" to stderr,
1607    // which brew routes to orch.error.log — polluting it even when projects ARE
1608    // configured correctly but GitHub is temporarily unreachable.
1609    let startup_escalation_secs = config.startup_failure_escalation_secs;
1610    let mut project_engines = {
1611        let mut delay_secs = 5u64;
1612        let mut attempt = 0u32;
1613        let mut escalated = false;
1614        let start = Instant::now();
1615        loop {
1616            attempt += 1;
1617            match init_project_engines().await {
1618                Ok(engines) => break engines,
1619                Err(e) => {
1620                    let elapsed = start.elapsed().as_secs();
1621
1622                    // Short blips (under the escalation threshold) stay at WARN
1623                    // so they don't flood the operator's attention. Once the
1624                    // outage crosses the threshold, escalate to ERROR-level
1625                    // logging and send a one-time push notification to configured
1626                    // channels — "transient" has clearly become "prolonged."
1627                    if startup_escalation_secs > 0
1628                        && !escalated
1629                        && elapsed >= startup_escalation_secs
1630                    {
1631                        escalated = true;
1632                        #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1633                        let outage = notification::format_duration(elapsed as f64);
1634                        tracing::error!(
1635                            delay_secs,
1636                            attempt,
1637                            elapsed_secs = elapsed,
1638                            threshold_secs = startup_escalation_secs,
1639                            "project backends unavailable for {outage} — escalating startup failure: {e}",
1640                        );
1641                        // Send one-time notification to configured channels.
1642                        // Channels aren't registered yet at this stage, so we
1643                        // construct them directly from config (mirrors orch notify).
1644                        send_startup_escalation_notification(elapsed).await;
1645                    } else {
1646                        tracing::warn!(
1647                            delay_secs,
1648                            attempt,
1649                            "project backends unavailable, retrying: {e}"
1650                        );
1651                    }
1652
1653                    time::sleep(Duration::from_secs(delay_secs)).await;
1654                    delay_secs = (delay_secs * 2).min(120);
1655                }
1656            }
1657        }
1658    };
1659
1660    tracing::info!(
1661        project_count = project_engines.len(),
1662        "initialized project engines"
1663    );
1664
1665    // Full set of currently-configured project repos, used by the per-tick global
1666    // sweeps (e.g. `dispatch_routed_tasks_global`) to distinguish "orphaned" tasks
1667    // (repo no longer in config) from tasks belonging to some other active project.
1668    let mut active_repos: HashSet<String> =
1669        project_engines.iter().map(|e| e.repo.clone()).collect();
1670
1671    // Create the event bus for task status transitions
1672    let event_bus = events::EventBus::new(256);
1673    if let Err(e) = event_bus.start_ws_server().await {
1674        tracing::warn!(
1675            ?e,
1676            "failed to start event websocket server, continuing without it"
1677        );
1678    }
1679
1680    // Check for a newer orch release in the background and log if the service is behind.
1681    // The periodic checker sends channel notifications after channels are initialized;
1682    // upgrading is operator-only and never runs automatically.
1683    {
1684        let current = env!("ORCH_VERSION").to_string();
1685        tokio::spawn(async move {
1686            if let Some(latest) = fetch_latest_release_version().await
1687                && latest != current
1688            {
1689                tracing::warn!(
1690                    current_version = %current,
1691                    latest_version = %latest,
1692                    "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1693                );
1694            }
1695        });
1696    }
1697
1698    // Re-create task managers with shared store and event bus
1699    for engine in &mut project_engines {
1700        engine.task_manager = Arc::new(TaskManager::with_events(
1701            Arc::clone(&engine.backend),
1702            Arc::clone(&engine.store),
1703            engine.repo.clone(),
1704            event_bus.sender(),
1705        ));
1706    }
1707
1708    // Startup reconciliation decides task status, so only the leader runs it
1709    let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1710    if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1711        tracing::warn!(err = %e, "startup worktree reconciliation failed");
1712    }
1713
1714    if leads {
1715        // Run estimate reconciliation in the background so it never blocks startup.
1716        // Each GraphQL mutation takes ~0.8 s; with hundreds of tasks this would
1717        // otherwise delay the engine reaching its main loop by several minutes.
1718        let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1719        tokio::spawn(async move {
1720            reconcile_startup_estimates(&engines_snapshot).await;
1721        });
1722    }
1723
1724    // Build ChannelRouter from global config + per-project configs
1725    let global_channel_config = GlobalChannelConfig {
1726        telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1727        discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1728        control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1729        control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1730    };
1731    let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1732        .iter()
1733        .map(|e| {
1734            let cfg = read_project_channel_config(&e.project_dir);
1735            (e.repo.clone(), cfg)
1736        })
1737        .collect();
1738    let channel_router = Arc::new(ChannelRouter::new(
1739        &global_channel_config,
1740        &project_channel_configs,
1741    ));
1742    tracing::info!(
1743        projects = ?channel_router.projects(),
1744        "channel router initialized"
1745    );
1746
1747    // Initialize tmux manager (shared across all projects)
1748    let tmux = Arc::new(TmuxManager::new());
1749    // Kill leftover sessions from a previous process. This is daemon start only,
1750    // never a tick, because nextest makes a once-flag in tick_all kill live agents
1751    tick::startup_cleanup(&tmux).await;
1752
1753    // Initialize transport
1754    let transport = Arc::new(Transport::new());
1755
1756    // Initialize capture service and start background loop
1757    let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1758    let capture_for_tick = Arc::clone(&capture);
1759    tokio::spawn(async move {
1760        capture.start().await;
1761    });
1762
1763    // Initialize channel registry
1764    let mut channel_registry = ChannelRegistry::new();
1765
1766    // Try to initialize Telegram channel
1767    if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1768        && !token.is_empty()
1769    {
1770        let chat_id = config::get("channels.telegram.chat_id").ok();
1771        match TelegramChannel::new(token, chat_id) {
1772            Ok(telegram) => {
1773                if let Err(e) = telegram.health_check().await {
1774                    tracing::warn!(?e, "telegram channel health check failed, skipping");
1775                } else {
1776                    channel_registry.register(Box::new(telegram));
1777                    tracing::info!("telegram channel registered");
1778                }
1779            }
1780            Err(e) => {
1781                tracing::warn!(?e, "failed to create telegram channel, skipping");
1782            }
1783        }
1784    }
1785
1786    // Try to initialize Discord Gateway channel (websocket)
1787    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1788        && !token.is_empty()
1789    {
1790        let channel_id = config::get("channels.discord.channel_id").ok();
1791        let shard_id = config::get("channels.discord.shard_id")
1792            .ok()
1793            .and_then(|v| v.parse::<u64>().ok())
1794            .unwrap_or(0);
1795        let shard_count = config::get("channels.discord.shard_count")
1796            .ok()
1797            .and_then(|v| v.parse::<u64>().ok())
1798            .unwrap_or(1);
1799        match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1800            Ok(discord) => {
1801                if let Err(e) = discord.health_check().await {
1802                    tracing::warn!(?e, "discord gateway health check failed, skipping");
1803                } else {
1804                    channel_registry.register(Box::new(discord));
1805                    tracing::info!(shard_id, shard_count, "discord gateway registered");
1806                }
1807            }
1808            Err(e) => {
1809                tracing::warn!(?e, "failed to create discord gateway, skipping");
1810            }
1811        }
1812    }
1813
1814    // Try to initialize Slack channel
1815    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1816        && !token.is_empty()
1817    {
1818        let channel_id = config::get("channels.slack.channel_id").ok();
1819        match SlackChannel::new(token, channel_id) {
1820            Ok(slack) => {
1821                if let Err(e) = slack.health_check().await {
1822                    tracing::warn!(?e, "slack channel health check failed, skipping");
1823                } else {
1824                    channel_registry.register(Box::new(slack));
1825                    tracing::info!("slack channel registered");
1826                }
1827            }
1828            Err(e) => {
1829                tracing::warn!(?e, "failed to create slack channel, skipping");
1830            }
1831        }
1832    }
1833
1834    // Initialize tmux channel with transport for output streaming
1835    let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1836    channel_registry.register(Box::new(tmux_channel));
1837    tracing::info!("tmux channel registered");
1838
1839    // Start all channels and collect their message receivers
1840    let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1841    for channel in channel_registry.iter() {
1842        match channel.start().await {
1843            Ok(rx) => {
1844                tracing::info!(channel = channel.name(), "channel started");
1845                channel_receivers.push(rx);
1846            }
1847            Err(e) => {
1848                tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1849            }
1850        }
1851    }
1852
1853    // Wrap channel registry in Arc for shared access (notification dispatcher needs it)
1854    let channel_registry = Arc::new(channel_registry);
1855
1856    // Spawn tasks to handle incoming channel messages (if any channels are active)
1857    let transport_for_messages = Arc::clone(&transport);
1858    let tmux_for_messages = Arc::clone(&tmux);
1859    let capture_for_messages = Arc::clone(&capture_for_tick);
1860    let channels_for_messages = Arc::clone(&channel_registry);
1861    // Lightweight engine references for command execution and task creation
1862    let engine_refs: Vec<EngineRef> = project_engines
1863        .iter()
1864        .map(|e| {
1865            (
1866                e.repo.clone(),
1867                Arc::clone(&e.backend),
1868                Arc::clone(&e.task_manager),
1869                Some(Arc::clone(&e.store)),
1870            )
1871        })
1872        .collect();
1873    // Shared in-memory map for pending project picks (General channel multi-project picker).
1874    let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1875    let router_for_messages = Arc::clone(&channel_router);
1876    for mut rx in channel_receivers {
1877        let transport = Arc::clone(&transport_for_messages);
1878        let tmux = Arc::clone(&tmux_for_messages);
1879        let capture = Arc::clone(&capture_for_messages);
1880        let channels = Arc::clone(&channels_for_messages);
1881        let engine_refs = engine_refs.clone();
1882        let ch_router = Arc::clone(&router_for_messages);
1883        let picks = Arc::clone(&pending_picks);
1884        tokio::spawn(async move {
1885            while let Some(msg) = rx.recv().await {
1886                tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1887                channel_handler::handle_channel_message(
1888                    msg,
1889                    &transport,
1890                    &tmux,
1891                    &capture,
1892                    &channels,
1893                    &engine_refs,
1894                    &ch_router,
1895                    &picks,
1896                )
1897                .await;
1898            }
1899        });
1900    }
1901
1902    // Post-restart confirmation: if the service was restarted via the /restart control command,
1903    // a KV marker was stored before shutdown. Send a confirmation to the originating channel now
1904    // that we're back online. Use a short delay to let the channel connections re-establish.
1905    {
1906        let restart_channels = Arc::clone(&channel_registry);
1907        let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1908        if let Some(store) = restart_store {
1909            tokio::spawn(async move {
1910                time::sleep(Duration::from_secs(5)).await;
1911                match store.kv_get("control:restart_pending").await {
1912                    Ok(Some(conv_key)) => {
1913                        if let Some((ch_name, thread_id, topic_id)) =
1914                            transport::parse_conversation_key(&conv_key)
1915                        {
1916                            for ch in restart_channels.iter() {
1917                                if ch.name() == ch_name {
1918                                    let msg = OutgoingMessage {
1919                                        thread_id: thread_id.to_string(),
1920                                        body: "orch restarted successfully.".to_string(),
1921                                        reply_to: None,
1922                                        metadata: serde_json::json!({}),
1923                                        topic_id: topic_id.map(ToString::to_string),
1924                                    };
1925                                    if let Err(e) = ch.send(&msg).await {
1926                                        tracing::warn!(
1927                                            channel = ch_name,
1928                                            error = %e,
1929                                            "failed to send restart confirmation"
1930                                        );
1931                                    }
1932                                    break;
1933                                }
1934                            }
1935                        }
1936                        if let Err(e) = store.kv_delete("control:restart_pending").await {
1937                            tracing::warn!(error = %e, "failed to clear control:restart_pending");
1938                        }
1939                    }
1940                    Ok(None) => {} // no pending restart, nothing to do
1941                    Err(e) => {
1942                        tracing::warn!(error = %e, "failed to read control:restart_pending");
1943                    }
1944                }
1945            });
1946        }
1947    }
1948
1949    // Spawn notification dispatcher — reads task completion notifications
1950    // from transport and broadcasts to configured channels.
1951    //
1952    // Routing priority:
1953    // 1. Dedicated channel: if the notification has a `repo` and the
1954    //    ChannelRouter maps it to a topic/channel, send there with `topic_id`.
1955    // 2. Subscribed channels: send to each (channel, thread_id) subscribed to
1956    //    the repo, using `format_with_project()` so recipients know the source.
1957    // 3. Fallback: if no repo or no dedicated channel, broadcast to all
1958    //    channels without a topic_id (legacy behaviour).
1959    {
1960        let mut notification_rx = transport.subscribe_notifications();
1961        let channels = Arc::clone(&channel_registry);
1962        let notif_router = Arc::clone(&channel_router);
1963        // Grab a store reference for subscription lookups.
1964        let notif_store: Option<Arc<TaskStore>> =
1965            project_engines.first().map(|e| Arc::clone(&e.store));
1966        tokio::spawn(async move {
1967            loop {
1968                match notification_rx.recv().await {
1969                    Ok(notification) => {
1970                        let level = NotificationLevel::from_config();
1971                        if !level.should_notify(&notification.status) {
1972                            tracing::debug!(
1973                                task_id = %notification.task_id,
1974                                status = %notification.status,
1975                                "notification suppressed by level={:?}",
1976                                level
1977                            );
1978                            continue;
1979                        }
1980
1981                        tracing::info!(
1982                            task_id = %notification.task_id,
1983                            status = %notification.status,
1984                            repo = ?notification.repo,
1985                            "routing notification to channels"
1986                        );
1987
1988                        // Track whether we sent to at least one dedicated/subscribed target.
1989                        let mut routed = false;
1990
1991                        // 0. Direct target: if notify_target is set, send straight to
1992                        //    that Telegram chat_id and skip all other routing.
1993                        if let Some(ref target) = notification.notify_target {
1994                            let telegram = channels.iter().find(|c| c.name() == "telegram");
1995                            if let Some(channel) = telegram {
1996                                let body = notification.format_telegram();
1997                                let metadata = serde_json::json!({
1998                                    "preformatted_html": true,
1999                                    "chat_id_override": target
2000                                });
2001                                let msg = OutgoingMessage {
2002                                    thread_id: notification.task_id.clone(),
2003                                    body,
2004                                    reply_to: None,
2005                                    metadata,
2006                                    topic_id: None,
2007                                };
2008                                match channel.send(&msg).await {
2009                                    Err(e) => {
2010                                        tracing::warn!(
2011                                            task_id = %notification.task_id,
2012                                            target,
2013                                            ?e,
2014                                            "failed to send to notify_target"
2015                                        );
2016                                    }
2017                                    _ => {
2018                                        routed = true;
2019                                    }
2020                                }
2021                            }
2022                        }
2023
2024                        if routed {
2025                            // notify_target handled — skip further routing
2026                        } else if let Some(repo) = notification.repo.as_deref() {
2027                            // Track destinations already sent to in this iteration so
2028                            // the dedicated (path 1) and subscribed (path 2) loops
2029                            // don't double-fire the same Telegram/Discord destination
2030                            // when a repo's configured topic and a `channel_subscriptions`
2031                            // row resolve to the same `(channel, topic_id)`.
2032                            let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2033
2034                            // 1. Dedicated channel targets for this repo.
2035                            for channel in channels.iter() {
2036                                let ch_name = channel.name();
2037                                if let Some(topic_id) =
2038                                    notif_router.target_for_project(repo, ch_name)
2039                                {
2040                                    let body = match ch_name {
2041                                        "telegram" => notification.format_telegram(),
2042                                        "discord" => notification.format_discord(),
2043                                        "slack" => notification.format_slack(),
2044                                        _ => continue,
2045                                    };
2046                                    let metadata = if ch_name == "telegram" {
2047                                        serde_json::json!({"preformatted_html": true})
2048                                    } else {
2049                                        serde_json::json!({})
2050                                    };
2051                                    let resolved_topic = Some(topic_id.to_string());
2052                                    let msg = OutgoingMessage {
2053                                        thread_id: notification.task_id.clone(),
2054                                        body,
2055                                        reply_to: None,
2056                                        metadata,
2057                                        topic_id: resolved_topic.clone(),
2058                                    };
2059                                    if let Err(e) = channel.send(&msg).await {
2060                                        tracing::warn!(
2061                                            channel = ch_name,
2062                                            task_id = %notification.task_id,
2063                                            ?e,
2064                                            "failed to send to dedicated channel"
2065                                        );
2066                                    } else {
2067                                        sent.insert((ch_name.to_string(), resolved_topic));
2068                                        routed = true;
2069                                    }
2070                                }
2071                            }
2072
2073                            // 2. Subscribed channels for this repo.
2074                            if let Some(store) = &notif_store {
2075                                match store.list_subscribers_for_repo(repo).await {
2076                                    Ok(subscribers) => {
2077                                        for (ch_name, thread_id, topic_id) in subscribers {
2078                                            let channel =
2079                                                channels.iter().find(|c| c.name() == ch_name);
2080                                            let Some(channel) = channel else {
2081                                                continue;
2082                                            };
2083                                            let resolved_topic = if topic_id.is_empty() {
2084                                                None
2085                                            } else {
2086                                                Some(topic_id.clone())
2087                                            };
2088                                            // Skip if a dedicated send to the same
2089                                            // (channel, topic_id) already happened.
2090                                            if sent.contains(&(
2091                                                ch_name.clone(),
2092                                                resolved_topic.clone(),
2093                                            )) {
2094                                                tracing::debug!(
2095                                                    channel = %ch_name,
2096                                                    task_id = %notification.task_id,
2097                                                    topic_id = ?resolved_topic,
2098                                                    "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2099                                                );
2100                                                continue;
2101                                            }
2102                                            let body = notification.format_with_project(&ch_name);
2103                                            let metadata = if ch_name == "telegram" {
2104                                                serde_json::json!({"preformatted_html": true})
2105                                            } else {
2106                                                serde_json::json!({})
2107                                            };
2108                                            let msg = OutgoingMessage {
2109                                                thread_id: thread_id.clone(),
2110                                                body,
2111                                                reply_to: None,
2112                                                metadata,
2113                                                topic_id: resolved_topic.clone(),
2114                                            };
2115                                            if let Err(e) = channel.send(&msg).await {
2116                                                tracing::warn!(
2117                                                    channel = %ch_name,
2118                                                    task_id = %notification.task_id,
2119                                                    ?e,
2120                                                    "failed to send to subscribed channel"
2121                                                );
2122                                            } else {
2123                                                sent.insert((ch_name.clone(), resolved_topic));
2124                                                routed = true;
2125                                            }
2126                                        }
2127                                    }
2128                                    Err(e) => {
2129                                        tracing::warn!(
2130                                            repo,
2131                                            ?e,
2132                                            "failed to list subscribers for repo"
2133                                        );
2134                                    }
2135                                }
2136                            }
2137                        }
2138
2139                        // 3. Fallback: broadcast to all channels when no routing happened.
2140                        if !routed {
2141                            for channel in channels.iter() {
2142                                let (body, should_send, is_telegram) = match channel.name() {
2143                                    "telegram" => (notification.format_telegram(), true, true),
2144                                    "discord" => (notification.format_discord(), true, false),
2145                                    "slack" => (notification.format_slack(), true, false),
2146                                    // GitHub is already handled by backend.post_comment()
2147                                    // tmux doesn't need task completion notifications
2148                                    _ => (String::new(), false, false),
2149                                };
2150
2151                                if !should_send {
2152                                    continue;
2153                                }
2154
2155                                let metadata = if is_telegram {
2156                                    serde_json::json!({"preformatted_html": true})
2157                                } else {
2158                                    serde_json::json!({})
2159                                };
2160                                let msg = OutgoingMessage {
2161                                    thread_id: notification.task_id.clone(),
2162                                    body,
2163                                    reply_to: None,
2164                                    metadata,
2165                                    topic_id: None,
2166                                };
2167
2168                                if let Err(e) = channel.send(&msg).await {
2169                                    tracing::warn!(
2170                                        channel = channel.name(),
2171                                        task_id = %notification.task_id,
2172                                        ?e,
2173                                        "failed to send notification"
2174                                    );
2175                                }
2176                            }
2177                        }
2178                    }
2179                    Err(RecvError::Lagged(n)) => {
2180                        tracing::warn!(missed = n, "notification receiver lagged");
2181                    }
2182                    Err(RecvError::Closed) => {
2183                        tracing::debug!("notification channel closed");
2184                        break;
2185                    }
2186                }
2187            }
2188        });
2189        tracing::info!("notification dispatcher started");
2190    }
2191
2192    // Notify used by webhook events to wake up the engine tick immediately
2193    let webhook_notify = Arc::new(Notify::new());
2194
2195    // `WebhookStatus::fallback_mode` is only shown by `orch webhook status`, the sync
2196    // tick always runs every `engine.sync_interval` whether or not webhooks work
2197    let webhook_port: Option<u16>;
2198    let mut webhook_healthy: bool;
2199    let mut last_webhook_health_check = Instant::now();
2200    // Start webhook server if configured
2201    let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2202    // No readable secret keeps the webhook server off, it never accepts unsigned requests
2203    let webhook_secret = if webhook_enabled {
2204        secret_ref::config_secret("webhook.secret")
2205            .await
2206            .ok()
2207            .filter(|s| !s.trim().is_empty())
2208    } else {
2209        None
2210    };
2211    if webhook_enabled && webhook_secret.is_none() {
2212        tracing::error!(
2213            "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2214        );
2215    }
2216    let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2217    let webhook_secret = webhook_secret.unwrap_or_default();
2218
2219    // Shared status updated by the server-spawn task and the health-check loop.
2220    let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2221
2222    if webhook_enabled {
2223        // Spawn the HTTP server with exponential-backoff retry on transient
2224        // bind errors (e.g. EADDRINUSE).  After MAX_ATTEMPTS the engine falls
2225        // back to polling and updates the shared status accordingly.
2226        const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2227        let port: u16 = config::get("webhook.port")
2228            .ok()
2229            .and_then(|v| v.parse().ok())
2230            .unwrap_or(8080);
2231        webhook_port = Some(port);
2232        let secret = webhook_secret.clone();
2233        let webhook_repo = project_engines
2234            .first()
2235            .map(|e| e.repo.clone())
2236            .unwrap_or_default();
2237
2238        let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2239
2240        // Initialise status: configured, not yet healthy.
2241        let to_save = {
2242            let mut s = webhook_status.lock().await;
2243            s.configured = true;
2244            s.port = Some(port);
2245            s.healthy = false;
2246            s.fallback_mode = false;
2247            s.clone()
2248        };
2249        to_save.save().await;
2250
2251        let status_for_spawn = Arc::clone(&webhook_status);
2252        tokio::spawn(async move {
2253            let mut attempt = 0u32;
2254            loop {
2255                attempt += 1;
2256                tracing::info!(attempt, port, "attempting to start webhook server");
2257                match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2258                    .await
2259                {
2260                    Ok(()) => {
2261                        tracing::info!(attempt, "webhook server exited cleanly");
2262                        break;
2263                    }
2264                    Err(e) => {
2265                        let reason = e.to_string();
2266                        tracing::error!(attempt, %reason, "webhook server failed");
2267                        if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2268                            let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2269                                "max attempts reached"
2270                            } else {
2271                                "non-transient error"
2272                            };
2273                            tracing::error!(
2274                                attempt,
2275                                %reason,
2276                                kind,
2277                                orch_webhook_in_fallback = true,
2278                                "webhook server giving up, switching to polling fallback"
2279                            );
2280                            let to_save = {
2281                                let mut s = status_for_spawn.lock().await;
2282                                s.fallback_mode = true;
2283                                s.healthy = false;
2284                                s.last_failure_reason = Some(reason);
2285                                s.startup_attempts = attempt;
2286                                s.clone()
2287                            };
2288                            to_save.save().await;
2289                            break;
2290                        }
2291                        let delay = webhook_backoff_delay(attempt);
2292                        tracing::warn!(
2293                            attempt,
2294                            delay_ms = delay.as_millis(),
2295                            %reason,
2296                            "webhook bind failed (transient), retrying with backoff"
2297                        );
2298                        {
2299                            let to_save = {
2300                                let mut s = status_for_spawn.lock().await;
2301                                s.startup_attempts = attempt;
2302                                s.last_failure_reason = Some(reason);
2303                                s.clone()
2304                            };
2305                            to_save.save().await;
2306                        }
2307                        time::sleep(delay).await;
2308                    }
2309                }
2310            }
2311        });
2312
2313        // Spawn the message forwarding task (reads from webhook channel)
2314        let notify = Arc::clone(&webhook_notify);
2315        let transport_for_webhook = Arc::clone(&transport);
2316        tokio::spawn(async move {
2317            while let Some(msg) = rx.recv().await {
2318                tracing::info!(
2319                    channel = %msg.channel,
2320                    thread = %msg.thread_id,
2321                    event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2322                    action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2323                    "webhook event received, triggering immediate tick"
2324                );
2325                let route = transport_for_webhook.route(&msg).await;
2326                tracing::info!(
2327                    channel = %msg.channel,
2328                    route = ?route,
2329                    "webhook event routed"
2330                );
2331                // Wake up the engine tick immediately instead of waiting up to 10s
2332                notify.notify_one();
2333            }
2334        });
2335
2336        webhook_healthy = true;
2337        tracing::info!(port, orch_webhook_up = true, "webhook server started");
2338    } else {
2339        webhook_port = None;
2340        webhook_healthy = false;
2341        {
2342            let to_save = {
2343                let mut s = webhook_status.lock().await;
2344                s.configured = false;
2345                s.fallback_mode = true;
2346                s.clone()
2347            };
2348            to_save.save().await;
2349        }
2350        tracing::info!(
2351            orch_webhook_in_fallback = true,
2352            "webhook server disabled, using polling fallback mode"
2353        );
2354    }
2355
2356    start_relay_client(&project_engines, &webhook_notify);
2357    app::start_loader();
2358
2359    // Agent router (selects agent + model per task) - shared across projects.
2360    // Router::from_config() may run `opencode models` when the config uses opencode.
2361    // spawn_blocking keeps the Tokio runtime thread free during that I/O
2362    let router = Arc::new(RwLock::new(
2363        match task::spawn_blocking(Router::from_config).await {
2364            Ok(r) => r,
2365            Err(e) => {
2366                tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2367                task::spawn_blocking(|| Router::new(RouterConfig::default()))
2368                    .await
2369                    .unwrap_or_else(|e2| {
2370                        panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2371                    })
2372            }
2373        },
2374    ));
2375    {
2376        let r = router.read().await;
2377        tracing::info!(
2378            mode = %r.config.mode,
2379            agents = ?r.available_agents,
2380            fallback = %r.config.fallback_executor,
2381            "router initialized"
2382        );
2383    }
2384
2385    // Concurrency limiter (shared across all projects). `parallel_in_force` is the
2386    // limit actually handed out, which lags `config.max_parallel` after a shrink
2387    // until the running tasks give their permits back.
2388    let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2389    let mut parallel_in_force = config.max_parallel;
2390    // Reviews get their own pool, one reviewer per worker, so a busy worker pool never
2391    // starves a review (#3971)
2392    let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2393    let mut review_in_force = config.max_parallel;
2394
2395    // In-memory dispatch guard: tracks task IDs currently being dispatched.
2396    // Guards against GitHub API eventual consistency — after update_status(InProgress),
2397    // the label-removal webhook can trigger an immediate tick where list_by_status(Routed)
2398    // still returns the task (search index propagation delay). The tmux session does not
2399    // exist until the runner completes worktree setup (~10s later), so session_exists
2400    // alone is insufficient. Keyed by "{repo}/{task_id}".
2401    let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2402
2403    // In-memory guard to prevent double-spawn of auto-merge background tasks.
2404    // Since review_open_prs runs every sync_tick (~10s) but CI polling can take
2405    // up to 10 minutes, we track which tasks already have an auto-merge in flight.
2406    let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2407
2408    // Subscribe to config file changes for hot reload
2409    let mut config_rx = config::subscribe();
2410
2411    // Track sync interval
2412    let mut last_sync = Instant::now();
2413
2414    // Track upgrade check interval
2415    let mut last_upgrade_check: Option<Instant> = None;
2416
2417    // Reset epoch of the GitHub rate-limit window already reported, so the log line,
2418    // the KV flag and the channel notification fire once per window instead of per tick
2419    let mut rate_limit_reported: Option<i64> = None;
2420
2421    // Channel for weight signals from task runners back to the router
2422    let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2423
2424    // A restart ends every review session, so reset each in_review task left without one
2425    for engine in project_engines.iter().filter(|_| leads) {
2426        let reset =
2427            reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2428
2429        if reset > 0 {
2430            tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2431        }
2432    }
2433
2434    // Spawn dispatch subscribers — react to Routed events immediately
2435    for engine in &project_engines {
2436        subscribers::dispatch::spawn(
2437            event_bus.subscribe(),
2438            Arc::clone(&engine.backend),
2439            Arc::clone(&tmux),
2440            Arc::clone(&engine.runner),
2441            Arc::clone(&capture_for_tick),
2442            Arc::clone(&semaphore),
2443            Arc::clone(&engine.task_manager),
2444            weight_tx.clone(),
2445            Arc::clone(&router),
2446            Arc::clone(&dispatching),
2447            Arc::clone(&engine.store),
2448            engine.repo.clone(),
2449        );
2450    }
2451    tracing::info!("dispatch subscriber started");
2452
2453    // Spawn review subscribers — react to NeedsReview events immediately
2454    for engine in &project_engines {
2455        subscribers::review::spawn(
2456            event_bus.subscribe(),
2457            Arc::clone(&engine.backend),
2458            Arc::clone(&tmux),
2459            Arc::clone(&review_semaphore),
2460            Arc::clone(&engine.task_manager),
2461            Arc::clone(&router),
2462            Arc::clone(&dispatching),
2463            Arc::clone(&engine.store),
2464            engine.repo.clone(),
2465        );
2466    }
2467    tracing::info!("review subscriber started");
2468
2469    // Spawn unblock subscribers — react to Done events, unblock parent tasks immediately
2470    for engine in &project_engines {
2471        subscribers::unblock::spawn(
2472            event_bus.subscribe(),
2473            Arc::clone(&engine.backend),
2474            Arc::clone(&engine.task_manager),
2475            Arc::clone(&engine.store),
2476            engine.repo.clone(),
2477        );
2478    }
2479    tracing::info!("unblock subscriber started");
2480
2481    // Spawn notify subscriber — reacts to ALL events, pushes to transport.
2482    // Spawned once (not per-project) since the transport handles all repos.
2483    subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2484    tracing::info!("notify subscriber started");
2485
2486    // Main loop
2487    tracing::info!(
2488        tick = ?config.tick_interval,
2489        sync = ?config.sync_interval,
2490        parallel = config.max_parallel,
2491        "entering main loop"
2492    );
2493    // Values of the settings that only take effect at startup, so a reload can
2494    // tell the operator their edit needs a restart.
2495    let mut restart_only = restart_only_snapshot();
2496
2497    let mut interval = time::interval(config.tick_interval);
2498
2499    // Signal handlers (launchd/systemd send SIGTERM to stop services, SIGHUP for reload/restart)
2500    let mut sigterm = unix::signal(SignalKind::terminate())?;
2501    let mut sighup = unix::signal(SignalKind::hangup())?;
2502
2503    // Guard to prevent concurrent sync_tick runs (sync is spawned as a background task).
2504    let sync_in_progress = Arc::new(AtomicBool::new(false));
2505
2506    // Tick watchdog: detects when the main loop hasn't completed a tick in > 60s.
2507    // Updates an atomic timestamp at the end of each tick iteration.
2508    let last_tick_epoch = Arc::new(AtomicU64::new(
2509        SystemTime::now()
2510            .duration_since(std_time::UNIX_EPOCH)
2511            .unwrap_or_default()
2512            .as_secs(),
2513    ));
2514    {
2515        let last_tick_epoch = Arc::clone(&last_tick_epoch);
2516        let tick_interval_secs = config.tick_interval.as_secs();
2517        tokio::spawn(async move {
2518            // Check every 30s whether the main loop is still ticking.
2519            let mut watchdog_interval = time::interval(Duration::from_secs(30));
2520            // Own suspend baseline, independent of the main loop's checkpoint(). The
2521            // watchdog's interval timer also freezes during a host suspend, so this
2522            // detects the gap from its own wake-up instead of racing the main loop's
2523            // checkpoint() for the same GAP_LOG entry (see #3544).
2524            let mut suspend_detector = SuspendDetector::new();
2525            loop {
2526                watchdog_interval.tick().await;
2527                let own_gap = suspend_detector.check();
2528                let last = last_tick_epoch.load(Ordering::Relaxed);
2529                let now = SystemTime::now()
2530                    .duration_since(std_time::UNIX_EPOCH)
2531                    .unwrap_or_default()
2532                    .as_secs();
2533                let stale_secs = now.saturating_sub(last);
2534                // Warn if the tick loop hasn't completed in > 6× the tick interval (at least 60s).
2535                let threshold = (tick_interval_secs * 6).max(60);
2536                if stale_secs > threshold {
2537                    // A host suspend/resume gap fully explains a stale window that
2538                    // isn't an actual stall — the process (and its timers) were
2539                    // simply not scheduled while the host was asleep. Prefer this
2540                    // watchdog's own detector (immune to the checkpoint race above);
2541                    // fall back to the shared log in case the main loop's checkpoint()
2542                    // already ran and recorded a larger gap.
2543                    let shared_gap =
2544                        engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2545                    let gap = match (own_gap, shared_gap) {
2546                        (Some(a), Some(b)) => Some(a.max(b)),
2547                        (Some(a), None) => Some(a),
2548                        (None, Some(b)) => Some(b),
2549                        (None, None) => None,
2550                    };
2551                    if let Some(gap) = gap {
2552                        let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2553                        if gap_secs >= stale_secs {
2554                            tracing::info!(
2555                                stale_secs,
2556                                threshold,
2557                                suspend_gap_secs = gap_secs,
2558                                "tick loop stale window explained by host suspend/resume, not a stall"
2559                            );
2560                        } else {
2561                            tracing::error!(
2562                                stale_secs,
2563                                threshold,
2564                                suspend_gap_secs = gap_secs,
2565                                "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2566                                stale_secs,
2567                                threshold,
2568                                gap_secs,
2569                            );
2570                        }
2571                    } else {
2572                        tracing::error!(
2573                            stale_secs,
2574                            threshold,
2575                            "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2576                            stale_secs,
2577                            threshold,
2578                        );
2579                    }
2580                }
2581            }
2582        });
2583    }
2584
2585    loop {
2586        tokio::select! {
2587            _ = interval.tick() => {
2588                let tick_start = Instant::now();
2589
2590                // Detect a host suspend/resume gap once per iteration, in one place.
2591                // The watchdog and stuck-task recovery both read from this instead of
2592                // re-deriving suspend information from their own wall-clock deltas.
2593                if let Some(gap) = engine_suspend::checkpoint() {
2594                    tracing::info!(
2595                        gap_secs = gap.num_seconds(),
2596                        "host suspend/resume gap detected, discounting from stuck-task age checks"
2597                    );
2598                }
2599
2600                // Finish a pending shrink of the dispatch pool as running tasks
2601                // release their permits. Only logs when a permit was actually
2602                // taken back, so a long-running task does not log every tick.
2603                if review_in_force != config.max_parallel {
2604                    review_in_force =
2605                        resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2606                }
2607                if parallel_in_force != config.max_parallel {
2608                    let resized =
2609                        resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2610                    if resized != parallel_in_force {
2611                        parallel_in_force = resized;
2612                        tracing::info!(
2613                            in_force = parallel_in_force,
2614                            target = config.max_parallel,
2615                            "dispatch pool resized"
2616                        );
2617                    }
2618                }
2619
2620                // Tick weight recovery for rate-limited agents
2621                {
2622                    let mut rw = router.write().await;
2623                    rw.tick_weight_recovery();
2624                }
2625
2626                // Report the GitHub rate limit once per window, then tick anyway. Calls
2627                // into GitHub now fail fast instead of sleeping until the reset, so the
2628                // local phases keep moving and the watchdog does not see a stalled loop.
2629                if let Some(remaining) = GhHttp::is_rate_limited() {
2630                    let reset_at = chrono::Utc::now()
2631                        .timestamp()
2632                        .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2633                    // Tolerate jitter in the header so one window reports once
2634                    if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2635                        rate_limit_reported = Some(reset_at);
2636                        report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2637                    }
2638                } else if let Some(prev) = rate_limit_reported.take() {
2639                    tracing::info!(
2640                        limited_until = prev,
2641                        "GitHub API rate limit cleared"
2642                    );
2643                    if let Some(engine) = project_engines.first() {
2644                        let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2645                    }
2646                }
2647
2648                // Core tick: poll tasks for all projects
2649                tick::tick_all(
2650                    &project_engines,
2651                    &tmux,
2652                    &capture_for_tick,
2653                    &semaphore,
2654                    &config,
2655                    &router,
2656                    &weight_tx,
2657                    &dispatching,
2658                    Some(&transport),
2659                    &active_repos,
2660                    &auto_merge_in_flight,
2661                ).await;
2662
2663                // Drain pending weight signals after tick processing so the router
2664                // learns from outcomes produced by this iteration's task dispatches.
2665                while let Ok(signal) = weight_rx.try_recv() {
2666                    let mut rw = router.write().await;
2667                    match signal {
2668                        WeightSignal::RateLimited { ref agent } => {
2669                            rw.record_rate_limit(agent);
2670                        }
2671                        WeightSignal::Success { ref agent } => {
2672                            rw.record_success(agent);
2673                        }
2674                        WeightSignal::Failed { .. }
2675                        | WeightSignal::Rerouted
2676                        | WeightSignal::Blocked
2677                        | WeightSignal::None => {}
2678                    }
2679                }
2680
2681                // Periodic sync (less frequent) — spawned as a background task so it
2682                // cannot block the main tick loop. A stalled sync (hung HTTP call, slow
2683                // DB query, lock contention) previously parked all tokio workers for 12+
2684                // minutes because it ran inline. See issue #2574.
2685                if last_sync.elapsed() >= config.sync_interval {
2686                    if sync_in_progress.compare_exchange(
2687                        false,
2688                        true,
2689                        Ordering::AcqRel,
2690                        Ordering::Relaxed,
2691                    ).is_ok() {
2692                        spawn_sync_pass(
2693                            &project_engines,
2694                            &tmux,
2695                            &config,
2696                            &router,
2697                            &dispatching,
2698                            &auto_merge_in_flight,
2699                            &sync_in_progress,
2700                            "periodic",
2701                        );
2702                    } else {
2703                        tracing::debug!("sync tick still in progress, skipping");
2704                    }
2705                    // Records sync *schedule* time (not completion). This prevents
2706                    // re-entering the sync branch every tick while one is in flight.
2707                    // The sync_in_progress guard handles actual concurrency control.
2708                    last_sync = Instant::now();
2709                }
2710
2711                // Periodic webhook health check
2712                if webhook_enabled {
2713                    let health_check_interval = config.webhook_health_check_interval
2714                        .unwrap_or(Duration::from_secs(60));
2715
2716                    if last_webhook_health_check.elapsed() >= health_check_interval {
2717                        if let Some(port) = webhook_port {
2718                            let (health, failure_reason) =
2719                                github::check_webhook_health(port).await;
2720                            if health != webhook_healthy {
2721                                webhook_healthy = health;
2722                                if webhook_healthy {
2723                                    tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2724                                } else {
2725                                    tracing::warn!(
2726                                        port,
2727                                        orch_webhook_up = false,
2728                                        reason = failure_reason.as_deref().unwrap_or("unknown"),
2729                                        "webhook health check failed"
2730                                    );
2731                                }
2732                            }
2733                            // Persist updated status.
2734                            {
2735                                let to_save = {
2736                                    let mut s = webhook_status.lock().await;
2737                                    s.healthy = health;
2738                                    s.last_check_utc = Some(chrono::Utc::now());
2739                                    if health {
2740                                        s.last_failure_reason = None;
2741                                    } else if failure_reason.is_some() {
2742                                        s.last_failure_reason = failure_reason;
2743                                    }
2744                                    s.clone()
2745                                };
2746                                to_save.save().await;
2747                            }
2748                        }
2749                        last_webhook_health_check = Instant::now();
2750                    }
2751                }
2752
2753                // Periodic upgrade check — notify channels when a newer release is available.
2754                if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2755                    if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2756                        let channels = Arc::clone(&channel_registry);
2757                        let current = env!("ORCH_VERSION").to_string();
2758                        let check_interval =
2759                            Duration::from_secs(config.upgrade_check_interval);
2760                        tokio::spawn(async move {
2761                            check_and_notify_upgrade(&store, &channels, &current, check_interval)
2762                                .await;
2763                        });
2764                    }
2765                    last_upgrade_check = Some(Instant::now());
2766                }
2767
2768                // Update watchdog timestamp + log tick duration.
2769                let tick_elapsed = tick_start.elapsed();
2770                if tick_elapsed.as_secs() > 30 {
2771                    tracing::warn!(
2772                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2773                        "slow tick"
2774                    );
2775                } else {
2776                    tracing::debug!(
2777                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2778                        "tick complete"
2779                    );
2780                }
2781                last_tick_epoch.store(
2782                    SystemTime::now()
2783                        .duration_since(std_time::UNIX_EPOCH)
2784                        .unwrap_or_default()
2785                        .as_secs(),
2786                    Ordering::Relaxed,
2787                );
2788            }
2789            // Webhook events trigger an immediate tick (bypass polling interval)
2790            () = webhook_notify.notified() => {
2791                if let Some(remaining) = GhHttp::is_rate_limited() {
2792                    tracing::warn!(
2793                        remaining_secs = remaining.as_secs(),
2794                        "GitHub API rate-limited, skipping webhook-triggered tick"
2795                    );
2796                } else {
2797                    tracing::info!("webhook event triggered immediate tick");
2798
2799                    tick::tick_all(
2800                        &project_engines,
2801                        &tmux,
2802                        &capture_for_tick,
2803                        &semaphore,
2804                        &config,
2805                        &router,
2806                        &weight_tx,
2807                        &dispatching,
2808                        Some(&transport),
2809                        &active_repos,
2810                        &auto_merge_in_flight,
2811                    ).await;
2812
2813                // Drain pending weight signals after webhook-triggered tick processing.
2814                while let Ok(signal) = weight_rx.try_recv() {
2815                    let mut rw = router.write().await;
2816                    match signal {
2817                        WeightSignal::RateLimited { ref agent } => {
2818                            rw.record_rate_limit(agent);
2819                        }
2820                        WeightSignal::Success { ref agent } => {
2821                            rw.record_success(agent);
2822                        }
2823                        WeightSignal::Failed { .. }
2824                        | WeightSignal::Rerouted
2825                        | WeightSignal::Blocked
2826                        | WeightSignal::None => {}
2827                    }
2828                }
2829
2830                // Also attempt to run an immediate sync (ingest/cleanup) in the
2831                // background so webhook-driven events (e.g. new issues) are
2832                // picked up without waiting for the periodic sync interval.
2833                // Reuse the same sync_in_progress guard used by the periodic
2834                // branch to avoid concurrent syncs.
2835                if sync_in_progress.compare_exchange(
2836                    false,
2837                    true,
2838                    Ordering::AcqRel,
2839                    Ordering::Relaxed,
2840                ).is_ok() {
2841                    spawn_sync_pass(
2842                        &project_engines,
2843                        &tmux,
2844                        &config,
2845                        &router,
2846                        &dispatching,
2847                        &auto_merge_in_flight,
2848                        &sync_in_progress,
2849                        "webhook",
2850                    );
2851                    // Record schedule time so periodic sync doesn't immediately re-run.
2852                    last_sync = Instant::now();
2853                } else {
2854                    tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2855                }
2856                }
2857
2858                // Also reset the interval so we don't get a redundant tick right after
2859                interval.reset();
2860            }
2861            result = config_rx.recv() => {
2862                // A notification only arrives for a file that parsed, so there is no
2863                // broken-config case to guard against here (see `config::reload_file`).
2864                let reload = match result {
2865                    Ok(path) => {
2866                        tracing::info!(path = %path.display(), "config file changed, reloading");
2867                        true
2868                    }
2869                    Err(RecvError::Lagged(n)) => {
2870                        tracing::warn!(missed = n, "config change receiver lagged, reloading");
2871                        true
2872                    }
2873                    Err(RecvError::Closed) => {
2874                        tracing::warn!("config change channel closed");
2875                        false
2876                    }
2877                };
2878
2879                if reload {
2880                    // Reload engine config
2881                    let new_config = EngineConfig::from_config();
2882                    log_engine_config_changes(&config, &new_config);
2883
2884                    let new_restart_only = restart_only_snapshot();
2885                    log_restart_only_changes(&restart_only, &new_restart_only);
2886                    restart_only = new_restart_only;
2887
2888                    let tick_changed = new_config.tick_interval != config.tick_interval;
2889                    config = new_config;
2890
2891                    // Reset tick interval if it changed
2892                    if tick_changed {
2893                        interval = time::interval(config.tick_interval);
2894                        tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2895                    }
2896
2897                    // Resize the dispatch pool. A shrink only takes the permits that are
2898                    // free right now; the rest are taken as running tasks finish.
2899                    if review_in_force != config.max_parallel {
2900                        review_in_force = resize_dispatch_pool(
2901                            &review_semaphore,
2902                            review_in_force,
2903                            config.max_parallel,
2904                        );
2905                    }
2906                    if parallel_in_force != config.max_parallel {
2907                        parallel_in_force = resize_dispatch_pool(
2908                            &semaphore,
2909                            parallel_in_force,
2910                            config.max_parallel,
2911                        );
2912                        tracing::info!(
2913                            in_force = parallel_in_force,
2914                            target = config.max_parallel,
2915                            "dispatch pool resized"
2916                        );
2917                    }
2918
2919                    // Reload router config (async-safe: use reload_async to avoid blocking the runtime)
2920                    {
2921                        let mut router_guard = router.write().await;
2922                        router_guard.reload_async().await;
2923                    }
2924
2925                    // Pick up projects added to `projects:`
2926                    sync_project_engines(
2927                        &mut project_engines,
2928                        &mut active_repos,
2929                        &event_bus,
2930                        &tmux,
2931                        &capture_for_tick,
2932                        &semaphore,
2933                        &review_semaphore,
2934                        &weight_tx,
2935                        &router,
2936                        &dispatching,
2937                    ).await;
2938
2939                    tracing::info!(
2940                        tick = ?config.tick_interval,
2941                        sync = ?config.sync_interval,
2942                        parallel = config.max_parallel,
2943                        projects = project_engines.len(),
2944                        "config reloaded"
2945                    );
2946                }
2947            }
2948            signal_name = async {
2949                tokio::select! {
2950                    _ = signal::ctrl_c() => "SIGINT",
2951                    _ = sigterm.recv() => "SIGTERM",
2952                    _ = sighup.recv() => "SIGHUP",
2953                }
2954            } => {
2955                tracing::info!(signal = signal_name, "beginning graceful shutdown");
2956
2957                // Wrap the entire shutdown sequence in a timeout to enforce the configured limit.
2958                // This prevents indefinite blocking if status updates or tmux operations hang.
2959                let shutdown_result = time::timeout(
2960                    config.graceful_shutdown_timeout,
2961                    async {
2962                        // Reset in_progress tasks to routed so they re-dispatch after restart.
2963                        // Also reset in_review tasks to needs_review — their review agent
2964                        // tmux sessions will be killed when the process exits.
2965                        let mut reset_count = 0u32;
2966                        let mut review_reset_count = 0u32;
2967                        for engine in &project_engines {
2968                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
2969                                for task in &tasks {
2970                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
2971                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
2972                                    } else {
2973                                        if let Ok(Some(store_id)) =
2974                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
2975                                            && let Err(e) = engine
2976                                                .store
2977                                                .finalize_incomplete_runs(
2978                                                    store_id,
2979                                                    "aborted",
2980                                                    "graceful shutdown: reset in_progress task to routed",
2981                                                )
2982                                                .await
2983                                            {
2984                                                tracing::warn!(
2985                                                    task_id = task.id.0,
2986                                                    ?e,
2987                                                    "failed to finalize incomplete runs on shutdown"
2988                                                );
2989                                            }
2990                                        reset_count += 1;
2991                                    }
2992                                }
2993                            }
2994                            // Also reset internal in_progress tasks
2995                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
2996                                for task in &tasks {
2997                                    let task_id = format!("internal:{}", task.id);
2998                                    if let Err(e) = engine.task_manager.update_task_status(
2999                                        &ExternalId(task_id.clone()),
3000                                        Status::Routed,
3001                                    ).await {
3002                                        tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3003                                    } else {
3004                                        if let Err(e) = engine
3005                                            .store
3006                                            .finalize_incomplete_runs(
3007                                                task.id,
3008                                                "aborted",
3009                                                "graceful shutdown: reset in_progress task to routed",
3010                                            )
3011                                            .await
3012                                        {
3013                                            tracing::warn!(
3014                                                task_id,
3015                                                ?e,
3016                                                "failed to finalize incomplete internal runs on shutdown"
3017                                            );
3018                                        }
3019                                        reset_count += 1;
3020                                    }
3021                                }
3022                            }
3023                            // Reset in_review tasks — review agent sessions die on shutdown
3024                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3025                                for task in &tasks {
3026                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3027                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3028                                    } else {
3029                                        if let Ok(Some(store_id)) =
3030                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3031                                            && let Err(e) = engine
3032                                                .store
3033                                                .finalize_incomplete_runs(
3034                                                    store_id,
3035                                                    "aborted",
3036                                                    "graceful shutdown: reset in_review task to needs_review",
3037                                                )
3038                                                .await
3039                                            {
3040                                                tracing::warn!(
3041                                                    task_id = task.id.0,
3042                                                    ?e,
3043                                                    "failed to finalize incomplete review runs on shutdown"
3044                                                );
3045                                            }
3046                                        review_reset_count += 1;
3047                                    }
3048                                }
3049                            }
3050                            // Also reset internal in_review tasks
3051                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3052                                for task in &tasks {
3053                                    let task_id = format!("internal:{}", task.id);
3054                                    if let Err(e) = engine.task_manager.update_task_status(
3055                                        &ExternalId(task_id.clone()),
3056                                        Status::NeedsReview,
3057                                    ).await {
3058                                        tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3059                                    } else {
3060                                        if let Err(e) = engine
3061                                            .store
3062                                            .finalize_incomplete_runs(
3063                                                task.id,
3064                                                "aborted",
3065                                                "graceful shutdown: reset in_review task to needs_review",
3066                                            )
3067                                            .await
3068                                        {
3069                                            tracing::warn!(
3070                                                task_id,
3071                                                ?e,
3072                                                "failed to finalize incomplete internal review runs on shutdown"
3073                                            );
3074                                        }
3075                                        review_reset_count += 1;
3076                                    }
3077                                }
3078                            }
3079                        }
3080                        if reset_count > 0 {
3081                            tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3082                        }
3083                        if review_reset_count > 0 {
3084                            tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3085                        }
3086
3087                        // Kill all orch-managed tmux sessions so stale sessions don't
3088                        // block dispatch after restart (session_exists check).
3089                        let mut killed_sessions = Vec::new();
3090                        if let Ok(sessions) = tmux.list_sessions().await {
3091                            for session in &sessions {
3092                                if session.name.starts_with("orch-") {
3093                                    match tmux.kill_session(&session.name).await { Err(e) => {
3094                                        tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3095                                    } _ => {
3096                                        killed_sessions.push(session.name.clone());
3097                                    }}
3098                                }
3099                            }
3100                            if !killed_sessions.is_empty() {
3101                                tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3102                            }
3103                        }
3104
3105                        // Wait for sessions to actually die before exiting.
3106                        // Prevents race where process exits before tmux finishes cleanup.
3107                        if !killed_sessions.is_empty() {
3108                            let still_alive = tmux
3109                                .wait_for_sessions_dead(
3110                                    &killed_sessions,
3111                                    Duration::from_millis(100),
3112                                    Duration::from_secs(5),
3113                                )
3114                                .await;
3115                            if still_alive > 0 {
3116                                tracing::warn!(
3117                                    still_alive,
3118                                    "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3119                                );
3120                            }
3121                            tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3122                        }
3123                    },
3124                ).await;
3125
3126                if shutdown_result.is_err() {
3127                    tracing::error!(
3128                        timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3129                        "graceful shutdown timed out, forcing exit"
3130                    );
3131                }
3132
3133                break;
3134            }
3135        }
3136    }
3137
3138    // Clean up event bus port file and service version file
3139    events::cleanup_port_file().await;
3140    local::cleanup().await;
3141    events::cleanup_version_file();
3142
3143    // transport and channels drop here at end of scope
3144    let _ = transport;
3145    tracing::info!("orch engine stopped");
3146    Ok(())
3147}
3148
3149#[cfg(test)]
3150mod tests {
3151    use std::sync::Arc;
3152
3153    use super::*;
3154    use crate::backends::test_helpers::NoopBackend;
3155    use crate::channels::capture::CaptureService;
3156    use crate::channels::transport::Transport;
3157    use crate::tmux::SessionGuard;
3158    use crate::tmux::TmuxManager;
3159
3160    async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3161        let id = store
3162            .upsert_external(&crate::store::UpsertExternal {
3163                repo: "owner/repo",
3164                ext_id: number,
3165                title: "Under review",
3166                body: "",
3167                author: "",
3168                url: "",
3169                labels: &[],
3170                origin: "github",
3171            })
3172            .await
3173            .unwrap();
3174        store.update_status(id, TaskStatus::InReview).await.unwrap();
3175        id
3176    }
3177
3178    #[tokio::test]
3179    async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3180        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3181        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3182        let task_manager =
3183            TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3184        let tmux = TmuxManager::isolated();
3185        let orphaned = in_review_task(&store, "11").await;
3186        let live = in_review_task(&store, "12").await;
3187        let session = tmux
3188            .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3189            .await
3190            .unwrap();
3191        let _guard = SessionGuard(session);
3192
3193        let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3194
3195        assert_eq!(reset, 1);
3196        assert_eq!(
3197            store.get(orphaned).await.unwrap().status,
3198            TaskStatus::NeedsReview
3199        );
3200        assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3201    }
3202
3203    #[test]
3204    fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3205        let semaphore = Arc::new(Semaphore::new(4));
3206        let held: Vec<_> = (0..4)
3207            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3208            .collect();
3209        assert!(
3210            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3211            "pool is full"
3212        );
3213
3214        let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3215        assert_eq!(in_force, 8);
3216
3217        let grown: Vec<_> = (0..4)
3218            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3219            .collect();
3220        assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3221        assert!(
3222            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3223            "pool is full"
3224        );
3225    }
3226
3227    #[test]
3228    fn shrinking_max_parallel_never_cancels_a_running_task() {
3229        let semaphore = Arc::new(Semaphore::new(8));
3230        let mut held: Vec<_> = (0..8)
3231            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3232            .collect();
3233
3234        // Nothing is free, so all 8 tasks keep their permits and keep running.
3235        let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3236        assert_eq!(in_force, 8);
3237        assert_eq!(held.len(), 8);
3238
3239        // Three finish. Their permits are taken back instead of handed out again.
3240        held.truncate(5);
3241        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3242        assert_eq!(in_force, 5);
3243        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3244
3245        // One more finishes and the pool settles at the new limit.
3246        held.truncate(4);
3247        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3248        assert_eq!(in_force, 4);
3249        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3250
3251        // A slot freed after that is handed out, since 4 is now the real limit.
3252        held.truncate(3);
3253        assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3254    }
3255
3256    #[test]
3257    fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3258        let configured = vec![
3259            ("owner/one".to_string(), PathBuf::from("/srv/one")),
3260            ("owner/two".to_string(), PathBuf::from("/srv/two")),
3261        ];
3262        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3263
3264        let (added, removed) = diff_configured_projects(&configured, &active);
3265        assert_eq!(
3266            added,
3267            vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3268            "only the new project is started, so owner/one keeps its single set of subscribers"
3269        );
3270        assert!(removed.is_empty(), "{removed:?}");
3271    }
3272
3273    #[test]
3274    fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3275        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3276        let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3277            .into_iter()
3278            .collect();
3279
3280        let (added, removed) = diff_configured_projects(&configured, &active);
3281        assert!(added.is_empty(), "nothing new to start");
3282        assert_eq!(removed, vec!["owner/gone".to_string()]);
3283    }
3284
3285    #[test]
3286    fn an_unchanged_project_list_starts_nothing() {
3287        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3288        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3289
3290        assert_eq!(
3291            diff_configured_projects(&configured, &active),
3292            (vec![], vec![])
3293        );
3294    }
3295
3296    #[test]
3297    fn restart_only_keys_are_the_settings_read_once_at_startup() {
3298        // Guards the docs table: every key listed here is documented as needing
3299        // `orch service restart`, and nothing reloadable may be added to it.
3300        assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3301    }
3302
3303    #[test]
3304    fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3305        let before = vec![Some("bot_token: a\n".to_string()), None];
3306
3307        assert!(
3308            log_restart_only_changes(&before, &before.clone()).is_empty(),
3309            "a reload that did not touch these keys must stay quiet"
3310        );
3311
3312        // An edited token and a newly added webhook block are both changes
3313        let after = vec![
3314            Some("bot_token: b\n".to_string()),
3315            Some("port: 8080\n".to_string()),
3316        ];
3317        assert_eq!(
3318            log_restart_only_changes(&before, &after),
3319            ["channels", "webhook"]
3320        );
3321
3322        // Only the edited key is reported when the other is untouched
3323        let token_only = vec![Some("bot_token: b\n".to_string()), None];
3324        assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3325    }
3326
3327    #[test]
3328    fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3329        let semaphore = Arc::new(Semaphore::new(4));
3330        assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3331        assert_eq!(semaphore.available_permits(), 4);
3332    }
3333
3334    #[test]
3335    fn live_session_stuck_limit_follows_the_run_timeout() {
3336        let config = EngineConfig {
3337            stuck_timeout: 1800,
3338            run_timeouts: RunTimeouts {
3339                base: 1800,
3340                medium: 2700,
3341                complex: 3600,
3342            },
3343            ..EngineConfig::default()
3344        };
3345        assert_eq!(
3346            config.live_session_stuck_limit(Some("simple")),
3347            1800 + RUN_TIMEOUT_GRACE_SECS
3348        );
3349        assert_eq!(
3350            config.live_session_stuck_limit(Some("medium")),
3351            2700 + RUN_TIMEOUT_GRACE_SECS
3352        );
3353        assert_eq!(
3354            config.live_session_stuck_limit(Some("complex")),
3355            3600 + RUN_TIMEOUT_GRACE_SECS
3356        );
3357
3358        let long_stuck = EngineConfig {
3359            stuck_timeout: 7200,
3360            ..config
3361        };
3362        assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3363    }
3364
3365    #[test]
3366    fn engine_config_defaults() {
3367        let config = EngineConfig::default();
3368        assert_eq!(config.tick_interval, Duration::from_secs(10));
3369        assert_eq!(config.sync_interval, Duration::from_secs(45));
3370        assert_eq!(config.max_parallel, 4);
3371        assert_eq!(config.stuck_timeout, 1800);
3372        assert_eq!(config.no_session_stuck_timeout, 600);
3373        assert!(
3374            config.no_session_stuck_timeout < config.stuck_timeout,
3375            "no_session_stuck_timeout must be shorter than stuck_timeout"
3376        );
3377        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3378        assert_eq!(config.upgrade_check_interval, 3600);
3379    }
3380
3381    #[test]
3382    fn engine_config_from_config_uses_defaults_when_no_config() {
3383        let config = EngineConfig::from_config();
3384        assert_eq!(config.tick_interval, Duration::from_secs(10));
3385        assert_eq!(config.sync_interval, Duration::from_secs(45));
3386        // max_parallel may be overridden by user config; just check it's reasonable
3387        assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3388        assert_eq!(config.stuck_timeout, 1800);
3389        assert_eq!(config.no_session_stuck_timeout, 600);
3390        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3391        assert_eq!(config.upgrade_check_interval, 3600);
3392    }
3393
3394    #[test]
3395    fn engine_config_no_session_timeout_is_under_15_minutes() {
3396        let config = EngineConfig::default();
3397        assert!(
3398            config.no_session_stuck_timeout <= 900,
3399            "no_session_stuck_timeout ({}) exceeds 15 minutes",
3400            config.no_session_stuck_timeout
3401        );
3402    }
3403
3404    #[test]
3405    fn upgrade_check_runs_immediately_when_never_checked() {
3406        assert!(should_run_upgrade_check(None, 3600));
3407    }
3408
3409    #[test]
3410    fn upgrade_check_waits_for_interval_after_first_run() {
3411        assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3412    }
3413
3414    #[test]
3415    fn upgrade_check_can_be_disabled() {
3416        assert!(!should_run_upgrade_check(None, 0));
3417    }
3418
3419    #[test]
3420    fn configured_agents_parses_yaml_block_array() {
3421        // config::get returns serde_norway::to_string for arrays, which is YAML block format
3422        let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3423        let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3424        assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3425    }
3426
3427    #[test]
3428    fn configured_agents_parses_json_inline_array() {
3429        let json_str = r#"["claude","codex","olm"]"#;
3430        let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3431        assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3432    }
3433
3434    #[test]
3435    fn configured_agents_fallback_returns_default_agents() {
3436        // When no config is present, configured_agents() returns DEFAULT_AGENTS
3437        let defaults = configured_agents();
3438        assert!(defaults.contains(&"claude".to_string()));
3439        assert!(defaults.contains(&"codex".to_string()));
3440        assert!(defaults.contains(&"opencode".to_string()));
3441    }
3442
3443    /// Integration test that requires a fully working tmux environment with
3444    /// reliable capture-pane timing. Runs locally but not in CI (GitHub Actions
3445    /// has tmux installed but capture-pane timing is unreliable).
3446    ///
3447    /// DO NOT REMOVE #[ignore] — this test was un-ignored in PR #608 which
3448    /// broke CI. The `tmux -V` skip check is insufficient because tmux CAN
3449    /// create sessions on CI runners, but capture-pane output is empty or
3450    /// delayed, causing the assertion to fail after a 10s timeout.
3451    #[tokio::test]
3452    #[ignore = "capture-pane timing is unreliable on CI runners"]
3453    async fn integration_channel_to_tmux_to_capture() {
3454        // Create transport and capture service
3455        let transport = Arc::new(Transport::new());
3456        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3457        let tmux = TmuxManager::new();
3458
3459        let task_id = "testtask1".to_string();
3460        let session_name = format!("orch-test-{task_id}");
3461
3462        // Start a detached tmux session
3463        let _ = tmux
3464            .command()
3465            .args(["new-session", "-d", "-s", &session_name])
3466            .output()
3467            .await
3468            .expect("failed to start tmux session");
3469
3470        // Guard ensures session cleanup even on panic
3471        let _guard = SessionGuard(session_name.clone());
3472
3473        // Register session with capture service and transport binding
3474        capture
3475            .register_session("owner/repo", &task_id, &session_name)
3476            .await;
3477        transport
3478            .bind(
3479                "owner/repo",
3480                &task_id,
3481                &session_name,
3482                "telegram",
3483                "12345",
3484                None,
3485            )
3486            .await;
3487
3488        // Spawn capture.run() which exits when session is unregistered
3489        let capture_clone = Arc::clone(&capture);
3490        let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3491
3492        // Subscribe to transport output for this task
3493        let mut rx = transport
3494            .subscribe("owner/repo", &task_id)
3495            .await
3496            .expect("no subscription");
3497
3498        // Send a message into the tmux session via tmux send-keys (simulate channel input)
3499        let send = tmux
3500            .command()
3501            .args([
3502                "send-keys",
3503                "-t",
3504                &session_name,
3505                "echo from-channel",
3506                "Enter",
3507            ])
3508            .output()
3509            .await
3510            .expect("failed to send keys");
3511        assert!(send.status.success());
3512
3513        // Wait for an output chunk from capture -> transport
3514        let mut got = false;
3515        for _ in 0..20 {
3516            if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3517                && c.content.contains("from-channel")
3518            {
3519                got = true;
3520                break;
3521            }
3522        }
3523
3524        // Clean up: unregister session (tmux kill handled by SessionGuard on drop)
3525        capture.unregister_session("owner/repo", &task_id).await;
3526
3527        // Wait for capture.run to finish
3528        let _ = capture_handle.await;
3529
3530        assert!(got, "did not observe tmux output via capture/transport");
3531    }
3532
3533    // ── read_project_channel_config tests ─────────────────────────────────
3534
3535    #[test]
3536    fn read_project_channel_config_returns_defaults_when_no_file() {
3537        let dir = tempfile::tempdir().expect("tempdir");
3538        let cfg = read_project_channel_config(dir.path());
3539        assert!(cfg.telegram_topic_id.is_none());
3540        assert!(cfg.discord_channel_id.is_none());
3541    }
3542
3543    #[test]
3544    fn read_project_channel_config_parses_telegram_and_discord() {
3545        let dir = tempfile::tempdir().expect("tempdir");
3546        let orch_yml = dir.path().join(".orch.yml");
3547        fs::write(
3548            &orch_yml,
3549            "channels:\n  telegram:\n    topic_id: \"42\"\n  discord:\n    channel_id: \"9999\"\n",
3550        )
3551        .expect("write .orch.yml");
3552        let cfg = read_project_channel_config(dir.path());
3553        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3554        assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3555    }
3556
3557    #[test]
3558    fn read_project_channel_config_handles_partial_channels() {
3559        let dir = tempfile::tempdir().expect("tempdir");
3560        let orch_yml = dir.path().join(".orch.yml");
3561        fs::write(&orch_yml, "channels:\n  telegram:\n    topic_id: \"7\"\n")
3562            .expect("write .orch.yml");
3563        let cfg = read_project_channel_config(dir.path());
3564        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3565        assert!(cfg.discord_channel_id.is_none());
3566    }
3567
3568    #[test]
3569    fn read_project_channel_config_handles_malformed_yaml() {
3570        let dir = tempfile::tempdir().expect("tempdir");
3571        let orch_yml = dir.path().join(".orch.yml");
3572        fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3573        let cfg = read_project_channel_config(dir.path());
3574        // Should return defaults (not panic) and emit a warn! log with the parse error.
3575        assert!(cfg.telegram_topic_id.is_none());
3576        assert!(cfg.discord_channel_id.is_none());
3577    }
3578
3579    #[test]
3580    #[cfg(unix)]
3581    fn read_project_channel_config_handles_unreadable_file() {
3582        let dir = tempfile::tempdir().expect("tempdir");
3583        // A directory fails to read for every user, root included, where chmod 000 does not
3584        fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3585        let cfg = read_project_channel_config(dir.path());
3586        // Should return defaults (not panic) and emit a warn! log with the read error.
3587        assert!(cfg.telegram_topic_id.is_none());
3588        assert!(cfg.discord_channel_id.is_none());
3589    }
3590}