Skip to main content

orch/engine/
mod.rs

1//! Engine — the core orchestration loop.
2//!
3//! The engine is the central coordinator. It owns the tick loop, backend
4//! connections, channel registry, transport layer, and tmux session manager.
5//!
6//! Submodules handle the heavy lifting:
7//! - [`tick`] — core tick phases (session polling, stuck recovery, routing, dispatch)
8//! - [`sync`] — periodic sync operations (cleanup, PR review, mentions, skills)
9//! - [`review`] — PR review pipeline (review agent, auto-merge, change handling)
10//! - [`cleanup`] — worktree cleanup and merged-PR detection
11//!
12//! This file contains struct definitions, project initialization, and the
13//! main event loop (`serve()`).
14
15pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod relay;
27pub mod review;
28pub mod review_gate;
29pub mod review_poll;
30pub mod review_shipcheck;
31pub mod router;
32pub mod runner;
33pub mod subscribers;
34pub mod suspend;
35pub mod sync;
36pub mod tasks;
37pub mod tick;
38
39/// Standard Orch attribution footer for issue bodies, PR bodies, and comments.
40///
41/// Append to every user-visible string posted to GitHub so activity is
42/// clearly attributed to orch rather than appearing as a human post.
43use crate::backends::{ExternalBackend, ExternalId};
44use crate::channels::capture::CaptureService;
45use crate::channels::discord_ws::DiscordGateway;
46use crate::channels::github::start_webhook_server;
47use crate::channels::notification::NotificationLevel;
48use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
49use crate::channels::slack::SlackChannel;
50use crate::channels::telegram::TelegramChannel;
51use crate::channels::tmux::TmuxChannel;
52use crate::channels::transport::Transport;
53use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
54use crate::cmd::CommandErrorContext;
55use crate::config;
56use crate::engine::cleanup::{remove_worktree_and_branch, remove_worktree_and_branch_for};
57use crate::engine::router::{Router, RouterConfig};
58use crate::engine::runner::worktree::{
59    abort_worktree_rebase, list_project_worktrees, rebase_worktree_on_origin_main,
60    task_id_from_worktree_name, validate_worktree_gitdir,
61};
62use crate::engine::tasks::TaskManager;
63use crate::github::http::{GhHttp, rate_limit_metrics};
64use crate::repo_context::REPO_CONTEXT;
65use crate::store::{TaskStatus, TaskStore};
66use crate::tmux::TmuxManager;
67use runner::WeightSignal;
68// AtomicBool/Ordering removed — shutdown is now immediate (reset tasks + break)
69use crate::backends::Status;
70use crate::backends::github::GitHubBackend;
71use crate::channels::github;
72use crate::channels::github::is_transient_bind_error;
73use crate::channels::github::webhook_backoff_delay;
74use crate::channels::notification;
75use crate::channels::notification::TaskNotification;
76use crate::channels::transport;
77use crate::cluster;
78use crate::cluster::local;
79use crate::engine::cooldown as engine_cooldown;
80use crate::engine::review as engine_review;
81use crate::engine::runner::git_ops;
82use crate::engine::runner::worktree;
83use crate::engine::suspend as engine_suspend;
84use crate::engine::suspend::SuspendDetector;
85use crate::engine::tasks as engine_tasks;
86use crate::github::app;
87use crate::home;
88use crate::secret_ref;
89use crate::store;
90use crate::webhook_status::WebhookStatus;
91use futures::future;
92use std::collections::HashMap;
93use std::collections::HashSet;
94use std::fmt;
95use std::fmt::Debug;
96use std::fmt::Formatter;
97use std::fs;
98use std::path::Path;
99use std::path::PathBuf;
100use std::sync::Arc;
101use std::sync::atomic::AtomicBool;
102use std::sync::atomic::AtomicU64;
103use std::sync::atomic::Ordering;
104use std::time as std_time;
105use std::time::Duration;
106use std::time::Instant;
107use std::time::SystemTime;
108use tokio::process::Command;
109use tokio::signal;
110use tokio::signal::unix;
111use tokio::signal::unix::SignalKind;
112use tokio::sync::Mutex;
113use tokio::sync::broadcast::Sender;
114use tokio::sync::broadcast::error::RecvError;
115use tokio::sync::mpsc::Receiver;
116use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
117use tokio::task;
118use tokio::time;
119
120pub fn orch_footer() -> &'static str {
121    "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
122}
123
124/// Build a standard attribution footer for GitHub comments posted by orch bots.
125///
126/// `verb` is "Created", "Reviewed", or "Commented" depending on context.
127pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
128    match model {
129        Some(m) => format!(
130            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
131        ),
132        None => format!(
133            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
134        ),
135    }
136}
137
138/// Lightweight reference tuple for channel message handling.
139type EngineRef = (
140    String,
141    Arc<dyn ExternalBackend>,
142    Arc<TaskManager>,
143    Option<Arc<TaskStore>>,
144);
145
146/// Per-project engine state.
147///
148/// Each project has its own backend, task runner, and task manager,
149/// but they share the global tmux manager, transport, and semaphore.
150#[derive(Clone)]
151pub struct ProjectEngine {
152    pub repo: String,
153    pub project_dir: PathBuf,
154    pub backend: Arc<dyn ExternalBackend>,
155    pub task_manager: Arc<TaskManager>,
156    pub runner: Arc<runner::TaskRunner>,
157    pub store: Arc<TaskStore>,
158}
159
160impl Debug for ProjectEngine {
161    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
162        f.debug_struct("ProjectEngine")
163            .field("repo", &self.repo)
164            .field("project_dir", &self.project_dir)
165            .finish_non_exhaustive()
166    }
167}
168
169/// A restart ends every review session, so an `in_review` task with no session has no reviewer.
170/// Hand it back to `needs_review` at start, so it does not wait for the stuck-review recovery
171pub(crate) async fn reset_orphaned_reviews(
172    task_manager: &TaskManager,
173    store: &Arc<TaskStore>,
174    repo: &str,
175    tmux: &TmuxManager,
176) -> usize {
177    // The store holds external and internal tasks, the second list is the legacy internal table
178    let mut ids: Vec<ExternalId> = store
179        .list_by_status(repo, TaskStatus::InReview)
180        .await
181        .unwrap_or_default()
182        .iter()
183        .map(|t| engine_tasks::store_task_to_external(t).id)
184        .collect();
185    if let Ok(legacy) = task_manager
186        .list_internal_by_status(TaskStatus::InReview)
187        .await
188    {
189        ids.extend(legacy.into_iter().map(|t| t.id));
190    }
191    ids.sort_by(|a, b| a.0.cmp(&b.0));
192    ids.dedup_by(|a, b| a.0 == b.0);
193
194    let mut reset = 0;
195    for id in &ids {
196        let session = tmux.session_name(repo, &format!("{}-review", id.0));
197
198        if tmux.session_exists(&session).await {
199            continue;
200        }
201
202        if let Err(e) = task_manager
203            .update_task_status(id, Status::NeedsReview)
204            .await
205        {
206            tracing::warn!(
207                task_id = id.0,
208                ?e,
209                "failed to reset an orphaned review at start"
210            );
211            continue;
212        }
213
214        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
215            let reason = "startup: review session gone, reset to needs_review";
216
217            if let Err(e) = store
218                .finalize_incomplete_runs(store_id, "aborted", reason)
219                .await
220            {
221                tracing::warn!(
222                    task_id = id.0,
223                    ?e,
224                    "failed to finalize the orphaned review's runs"
225                );
226            }
227        }
228
229        reset += 1;
230    }
231
232    reset
233}
234
235/// Starts the relay client when `relay.url` is set. The token is read in the background, so a
236/// passbox prompt nobody answers never holds up the engine
237fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
238    // The global config only: a project file must never point a secret at its own endpoint
239    let url = config::get_global("relay.url").unwrap_or_default();
240    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
241        return;
242    };
243
244    if url.is_empty() {
245        return;
246    }
247
248    let token_ref = config::get_global("relay.token").unwrap_or_default();
249    let notify = Arc::clone(notify);
250    tokio::spawn(async move {
251        let mut warned = false;
252        let token = loop {
253            let reference = token_ref.clone();
254            let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
255                .await
256                .map_err(anyhow::Error::from)
257                .and_then(|r| r);
258
259            match read {
260                Ok(token) if !token.is_empty() => break token,
261                Ok(_) => {
262                    tracing::warn!("relay.token is empty, the relay client stays off");
263                    return;
264                }
265                Err(e) if !warned => {
266                    tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
267                    warned = true;
268                }
269                Err(_) => {}
270            }
271
272            time::sleep(Duration::from_secs(300)).await;
273        };
274
275        relay::run(url, token, store, notify).await;
276    });
277}
278
279/**
280 * Runs `sync_tick` for every project at once, then the tail that is the same for all of them,
281 * in a spawned task. Sync time is the slowest project instead of the sum of all of them
282 * (#3864). The caller must already hold `sync_in_progress`; this releases it when done
283 **/
284#[expect(
285    clippy::too_many_arguments,
286    reason = "the spawned task takes its own handle to each piece of engine state"
287)]
288fn spawn_sync_pass(
289    project_engines: &[ProjectEngine],
290    tmux: &Arc<TmuxManager>,
291    engine_config: &EngineConfig,
292    router: &Arc<RwLock<Router>>,
293    dispatching: &Arc<dashmap::DashMap<String, String>>,
294    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
295    sync_in_progress: &Arc<AtomicBool>,
296    trigger: &'static str,
297) {
298    let engines = project_engines.to_vec();
299    let tmux = Arc::clone(tmux);
300    let engine_config = engine_config.clone();
301    let router = Arc::clone(router);
302    let dispatching = Arc::clone(dispatching);
303    let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
304    let guard = Arc::clone(sync_in_progress);
305
306    tokio::spawn(async move {
307        let started = Instant::now();
308        let mut passes = Vec::with_capacity(engines.len());
309        for engine in &engines {
310            passes.push(REPO_CONTEXT.scope(
311                engine.repo.clone(),
312                sync_one_project(
313                    engine,
314                    &tmux,
315                    &engine_config,
316                    &router,
317                    &dispatching,
318                    &auto_merge_in_flight,
319                ),
320            ));
321        }
322        future::join_all(passes).await;
323
324        sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
325        tracing::info!(
326            trigger,
327            elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
328            "sync tick complete"
329        );
330        guard.store(false, Ordering::Release);
331    });
332}
333
334async fn sync_one_project(
335    engine: &ProjectEngine,
336    tmux: &Arc<TmuxManager>,
337    engine_config: &EngineConfig,
338    router: &Arc<RwLock<Router>>,
339    dispatching: &Arc<dashmap::DashMap<String, String>>,
340    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
341) {
342    if let Err(e) = Box::pin(sync::sync_tick(
343        &engine.backend,
344        tmux,
345        &engine.repo,
346        engine_config,
347        router,
348        &engine.task_manager,
349        &engine.store,
350        dispatching,
351        auto_merge_in_flight,
352    ))
353    .await
354    {
355        tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
356    }
357}
358
359/// Milliseconds a group of tick or sync phases may take before it is worth an operator's eye
360const SLOW_PHASE_GROUP_MS: u128 = 1_000;
361
362/**
363 * Logs how long each phase of a tick or sync group took, for one project. Tick and sync time
364 * grows with the project count, and the totals alone did not say which project or phase spent
365 * it (#3864). Info once a group crosses `SLOW_PHASE_GROUP_MS`, so a quiet install stays quiet
366 **/
367pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
368    let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
369    if total_ms >= SLOW_PHASE_GROUP_MS {
370        tracing::info!(group, repo, total_ms, ?phases, "phase timings");
371    } else {
372        tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
373    }
374}
375
376/// Return the configured agent list from top-level `agents` in config.yml,
377/// falling back to [`router::config::DEFAULT_AGENTS`] when not set.
378///
379/// This is the canonical source of truth for "which agents does orch know
380/// about" — use it instead of `DEFAULT_AGENTS` directly so that
381/// Claude-compatible agents added via config (e.g. `olm`) are recognized
382/// without code changes.
383///
384/// The key is top-level (`agents:`) rather than nested (`engine.agents:`)
385/// because `config::get` serializes nested YAML arrays via `serde_norway::to_string`
386/// which produces YAML block format, not JSON — breaking downstream parsing.
387pub fn configured_agents() -> Vec<String> {
388    if let Ok(agents_str) = config::get("agents")
389        && !agents_str.is_empty()
390        && agents_str != "[]"
391    {
392        // config::get returns serde_norway::to_string for arrays, which is
393        // YAML block format ("- claude\n- codex\n..."). Parse as YAML first.
394        if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
395            && !agents_arr.is_empty()
396        {
397            return agents_arr;
398        }
399        // Fallback: try JSON (inline arrays) or comma-separated
400        if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
401            && !agents_arr.is_empty()
402        {
403            return agents_arr;
404        }
405    }
406    router::config::DEFAULT_AGENTS
407        .iter()
408        .map(ToString::to_string)
409        .collect()
410}
411
412/// Seconds a live session may run past its own timeout before stuck recovery takes it. The
413/// `timeout` wrapper ends the agent at the limit, and the runner still collects the result
414pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
415
416/// The limit the `timeout` wrapper puts on one agent run, per complexity
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418pub struct RunTimeouts {
419    pub base: u64,
420    pub medium: u64,
421    pub complex: u64,
422}
423
424impl Default for RunTimeouts {
425    fn default() -> Self {
426        Self {
427            base: 1800,
428            medium: 1800,
429            complex: 1800,
430        }
431    }
432}
433
434impl RunTimeouts {
435    /// `workflow.timeout_seconds` (at least 1800), raised by
436    /// `workflow.timeout_by_complexity.{medium,complex}` but never below the base
437    pub fn from_config() -> Self {
438        let read =
439            |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
440        let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
441        Self {
442            base,
443            medium: read("workflow.timeout_by_complexity.medium")
444                .unwrap_or(base)
445                .max(base),
446            complex: read("workflow.timeout_by_complexity.complex")
447                .unwrap_or(base)
448                .max(base),
449        }
450    }
451
452    pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
453        match complexity {
454            Some("complex") => self.complex,
455            Some("medium") => self.medium,
456            _ => self.base,
457        }
458    }
459}
460
461/// Engine configuration.
462#[derive(Clone, Debug)]
463pub struct EngineConfig {
464    /// Main tick interval
465    pub tick_interval: Duration,
466    /// GitHub sync interval (cleanup, PR review, mentions)
467    pub sync_interval: Duration,
468    /// Webhook health check interval (seconds)
469    pub webhook_health_check_interval: Option<Duration>,
470    /// Maximum parallel task executions
471    pub max_parallel: usize,
472    /// Stuck task timeout for tasks with an active tmux session (seconds)
473    pub stuck_timeout: u64,
474    /// Stuck task timeout for tasks with no active tmux session (seconds).
475    /// Shorter than `stuck_timeout` because no session means the agent has already exited.
476    pub no_session_stuck_timeout: u64,
477    /// Stuck task timeout for in_review tasks with no active tmux session (seconds).
478    /// Longer than `no_session_stuck_timeout` because a review agent ends its tmux session on
479    /// normal completion before its result reaches the engine. A shorter limit resets a task
480    /// whose finished review is still being delivered, and the result is dropped as stale.
481    /// `from_config` keeps it above the review agent's own timeout plus
482    /// `review::REVIEW_RESULT_DELIVERY_SECS`.
483    pub in_review_no_session_stuck_timeout: u64,
484    /// Per-run agent timeouts. A live session is not stuck before its own timeout runs out
485    pub run_timeouts: RunTimeouts,
486    /// Auto-create follow-up tasks when PR reviews request changes
487    pub auto_create_followup_on_changes: bool,
488    /// Graceful shutdown timeout — how long to wait for running agents before exiting.
489    pub graceful_shutdown_timeout: Duration,
490    /// Grace period before silence detection kicks in (seconds).
491    /// Allows time for agent startup, model download, API handshake.
492    pub silence_grace_period: u64,
493    /// Cooldown duration for a model detected as silent (seconds).
494    pub silence_cooldown: u64,
495    /// How often to check for a newer orch release and notify channels (seconds).
496    /// Set to 0 to disable. Default: 3600 (1 hour).
497    pub upgrade_check_interval: u64,
498    /// Seconds of continuous startup failure (GitHub unreachable / auth errors)
499    /// before the engine escalates from WARN to ERROR-level logging and sends a
500    /// one-time push notification to configured channels. Set to 0 to disable.
501    /// Default: 3600 (1 hour).
502    pub startup_failure_escalation_secs: u64,
503}
504
505impl Default for EngineConfig {
506    fn default() -> Self {
507        Self {
508            tick_interval: Duration::from_secs(10),
509            sync_interval: Duration::from_secs(45),
510            webhook_health_check_interval: Some(Duration::from_secs(60)),
511            max_parallel: 4,
512            stuck_timeout: 1800,
513            no_session_stuck_timeout: 600,
514            in_review_no_session_stuck_timeout: 1920,
515            run_timeouts: RunTimeouts::default(),
516            auto_create_followup_on_changes: true,
517            graceful_shutdown_timeout: Duration::from_secs(600),
518            silence_grace_period: 300,
519            silence_cooldown: 3600,
520            upgrade_check_interval: 3600,
521            startup_failure_escalation_secs: 3600,
522        }
523    }
524}
525
526impl EngineConfig {
527    /// Stuck limit for an in_progress task whose session is alive: `stuck_timeout`, raised to the
528    /// run's own timeout plus `RUN_TIMEOUT_GRACE_SECS`
529    pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
530        self.stuck_timeout.max(
531            self.run_timeouts
532                .for_complexity(complexity)
533                .saturating_add(RUN_TIMEOUT_GRACE_SECS),
534        )
535    }
536
537    /// Load engine configuration from config files, falling back to defaults.
538    pub fn from_config() -> Self {
539        let mut config = Self::default();
540
541        if let Ok(val) = config::get("engine.tick_interval") {
542            if let Ok(secs) = val.parse::<u64>() {
543                config.tick_interval = Duration::from_secs(secs);
544            } else {
545                tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
546            }
547        }
548
549        if let Ok(val) = config::get("engine.sync_interval") {
550            if let Ok(secs) = val.parse::<u64>() {
551                config.sync_interval = Duration::from_secs(secs);
552            } else {
553                tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
554            }
555        }
556
557        if let Ok(val) = config::get("engine.max_parallel") {
558            if let Ok(n) = val.parse::<usize>() {
559                config.max_parallel = n;
560            } else {
561                tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
562            }
563        }
564
565        if let Ok(val) = config::get("engine.stuck_timeout") {
566            if let Ok(secs) = val.parse::<u64>() {
567                config.stuck_timeout = secs;
568            } else {
569                tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
570            }
571        }
572
573        if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
574            if let Ok(secs) = val.parse::<u64>() {
575                config.no_session_stuck_timeout = secs;
576            } else {
577                tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
578            }
579        }
580
581        let mut in_review_from_config = false;
582        if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
583            if let Ok(secs) = val.parse::<u64>() {
584                config.in_review_no_session_stuck_timeout = secs;
585                in_review_from_config = true;
586            } else {
587                tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
588            }
589        }
590
591        // Keep the no-session review limit over the review agent's own timeout plus delivery
592        // time, whatever either is set to. Under that, recovery resets a task whose finished
593        // review is still on its way to the engine and the result is dropped as stale
594        let review_timeout = engine_review::review_timeout_seconds();
595        let floored = engine_review::no_session_review_floor(
596            config.in_review_no_session_stuck_timeout,
597            review_timeout,
598        );
599        if floored != config.in_review_no_session_stuck_timeout {
600            if in_review_from_config {
601                tracing::warn!(
602                    configured_secs = config.in_review_no_session_stuck_timeout,
603                    review_timeout_secs = review_timeout,
604                    raised_to_secs = floored,
605                    "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
606                );
607            } else {
608                tracing::debug!(
609                    review_timeout_secs = review_timeout,
610                    raised_to_secs = floored,
611                    "raised the default in_review no-session limit over a long review timeout"
612                );
613            }
614            config.in_review_no_session_stuck_timeout = floored;
615        }
616
617        config.run_timeouts = RunTimeouts::from_config();
618
619        if let Ok(val) = config::get("engine.webhook_health_check_interval") {
620            if let Ok(secs) = val.parse::<u64>() {
621                if secs > 0 {
622                    config.webhook_health_check_interval = Some(Duration::from_secs(secs));
623                } else {
624                    config.webhook_health_check_interval = None;
625                }
626            } else {
627                tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
628            }
629        }
630
631        if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
632            config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
633        }
634
635        if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
636            if let Ok(secs) = val.parse::<u64>() {
637                config.graceful_shutdown_timeout = Duration::from_secs(secs);
638            } else {
639                tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
640            }
641        }
642
643        if let Ok(val) = config::get("engine.silence_grace_period") {
644            if let Ok(secs) = val.parse::<u64>() {
645                config.silence_grace_period = secs;
646            } else {
647                tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
648            }
649        }
650
651        if let Ok(val) = config::get("engine.silence_cooldown") {
652            if let Ok(secs) = val.parse::<u64>() {
653                config.silence_cooldown = secs;
654            } else {
655                tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
656            }
657        }
658
659        if let Ok(val) = config::get("engine.upgrade_check_interval") {
660            if let Ok(secs) = val.parse::<u64>() {
661                config.upgrade_check_interval = secs;
662            } else {
663                tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
664            }
665        }
666
667        if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
668            if let Ok(secs) = val.parse::<u64>() {
669                config.startup_failure_escalation_secs = secs;
670            } else {
671                tracing::warn!(
672                    key = "engine.startup_failure_escalation_secs",
673                    value = %val,
674                    default_secs = config.startup_failure_escalation_secs,
675                    "invalid value for engine.startup_failure_escalation_secs, using default"
676                );
677            }
678        }
679
680        config
681    }
682}
683
684/// KV key holding the epoch second the GitHub core window resets, so `orch task status`
685/// can show the limit from a separate process.
686pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
687
688/// Format an epoch second as a local `HH:MM:SS` for operator-facing output.
689pub fn format_rate_limited_until(reset_at: i64) -> String {
690    chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
691        || reset_at.to_string(),
692        |t| {
693            t.with_timezone(&chrono::Local)
694                .format("%H:%M:%S %Z")
695                .to_string()
696        },
697    )
698}
699
700/// Announce a GitHub rate-limit window on every operator surface: the log (`orch log`),
701/// a KV flag (`orch task status`) and the channels (Telegram, Discord).
702async fn report_rate_limited(
703    reset_at: i64,
704    store: Option<&Arc<TaskStore>>,
705    transport: Option<&Arc<Transport>>,
706) {
707    let m = rate_limit_metrics();
708    let until = format_rate_limited_until(reset_at);
709    tracing::warn!(
710        reset_at,
711        rate_limit_hits = m.hits,
712        skipped_calls = m.skipped_calls,
713        five_xx_hits = m.five_xx_hits,
714        rest_remaining = ?m.rest_remaining,
715        graphql_remaining = ?m.graphql_remaining,
716        "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
717    );
718
719    if let Some(store) = store
720        && let Err(e) = store
721            .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
722            .await
723    {
724        tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
725    }
726
727    if let Some(transport) = transport {
728        transport.push_notification(TaskNotification {
729            task_id: "github".to_string(),
730            title: "GitHub rate limit".to_string(),
731            status: "blocked".to_string(),
732            agent: "orch".to_string(),
733            duration_seconds: 0.0,
734            summary: format!(
735                "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
736            ),
737            repo: None,
738            notify_target: None,
739            pr_number: None,
740        });
741    }
742}
743
744/// Grow or shrink the dispatch pool toward `target` without cancelling running work.
745///
746/// Growing adds permits at once. Shrinking can only take back the permits that
747/// are free right now, so the return value is the limit actually in force; call
748/// again as running tasks release theirs
749fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
750    if target > in_force {
751        semaphore.add_permits(target - in_force);
752        return target;
753    }
754
755    if target < in_force {
756        return in_force - semaphore.forget_permits(in_force - target);
757    }
758
759    in_force
760}
761
762/// Config subtrees the engine only reads at startup: the channel gateways hold
763/// long-lived websockets and the webhook listener binds its port once.
764const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
765
766/// Read the restart-only subtrees so a later reload can tell they changed.
767fn restart_only_snapshot() -> Vec<Option<String>> {
768    RESTART_ONLY_KEYS
769        .iter()
770        .map(|key| config::get(key).ok())
771        .collect()
772}
773
774/// Report the restart-only keys an edit changed. Only the key is logged, never
775/// the value, because `channels.*` holds bot tokens
776fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
777    let changed: Vec<&'static str> = RESTART_ONLY_KEYS
778        .iter()
779        .zip(old)
780        .zip(new)
781        .filter(|((_, old), new)| old != new)
782        .map(|((key, _), _)| *key)
783        .collect();
784
785    for key in &changed {
786        tracing::info!(key = key, "config changed, takes effect after restart");
787    }
788
789    changed
790}
791
792/// Log every engine setting whose value changed across a config reload, so
793/// `orch log` shows what the operator's edit actually did.
794fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
795    macro_rules! log_changed {
796        ($($key:literal => $field:ident),+ $(,)?) => {
797            $(if old.$field != new.$field {
798                tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
799            })+
800        };
801    }
802
803    log_changed!(
804        "engine.tick_interval" => tick_interval,
805        "engine.sync_interval" => sync_interval,
806        "engine.webhook_health_check_interval" => webhook_health_check_interval,
807        "engine.max_parallel" => max_parallel,
808        "engine.stuck_timeout" => stuck_timeout,
809        "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
810        "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
811        "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
812        "engine.silence_grace_period" => silence_grace_period,
813        "engine.silence_cooldown" => silence_cooldown,
814        "engine.upgrade_check_interval" => upgrade_check_interval,
815        "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
816        "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
817    );
818}
819
820fn should_run_upgrade_check(
821    last_upgrade_check: Option<Instant>,
822    upgrade_check_interval: u64,
823) -> bool {
824    if upgrade_check_interval == 0 {
825        return false;
826    }
827
828    last_upgrade_check
829        .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
830}
831
832/// Connect one project's backend and build its task manager and runner over
833/// the shared store.
834///
835/// `event_tx` wires status transitions into the event bus; pass `None` during
836/// startup, where the bus does not exist yet and task managers are rebuilt
837/// once it does
838async fn connect_project_engine(
839    repo: String,
840    project_dir: PathBuf,
841    store: Arc<TaskStore>,
842    event_tx: Option<Sender<events::TaskEvent>>,
843) -> anyhow::Result<ProjectEngine> {
844    let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
845
846    // A failed health check (rate limit, network, auth) must not drop the project: it would have
847    // no review subscriber until the next restart. Later ticks hit the API again and retry on
848    // their own, so build the engine and warn while the check keeps failing.
849    match backend.health_check().await {
850        Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
851        Err(e) => tracing::warn!(
852            repo = %repo,
853            error = %e,
854            "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
855        ),
856    }
857
858    let task_manager = Arc::new(match event_tx {
859        Some(tx) => {
860            TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
861        }
862        None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
863    });
864    let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
865
866    Ok(ProjectEngine {
867        repo,
868        project_dir,
869        backend,
870        task_manager,
871        runner,
872        store,
873    })
874}
875
876/**
877 * Split the configured project list into the ones not running yet and the names of the ones that
878 * went away. Kept separate from `sync_project_engines` so the comparison can be tested without a
879 * GitHub backend: getting it wrong either spawns a second set of subscribers for a project already
880 * running, or silently never picks a new one up
881 **/
882fn diff_configured_projects(
883    configured: &[(String, PathBuf)],
884    active_repos: &HashSet<String>,
885) -> (Vec<(String, PathBuf)>, Vec<String>) {
886    let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
887
888    let added = configured
889        .iter()
890        .filter(|(repo, _)| !active_repos.contains(repo))
891        .cloned()
892        .collect();
893    let removed = active_repos
894        .iter()
895        .filter(|repo| !configured_repos.contains(repo.as_str()))
896        .cloned()
897        .collect();
898
899    (added, removed)
900}
901
902/// Pick up projects added to `projects:` since startup, and report the ones
903/// that were removed.
904///
905/// A removed project keeps running until the next restart: its subscriber
906/// tasks and any agent it already dispatched outlive the config read
907#[expect(
908    clippy::too_many_arguments,
909    reason = "a new project engine needs every shared engine handle"
910)]
911async fn sync_project_engines(
912    project_engines: &mut Vec<ProjectEngine>,
913    active_repos: &mut HashSet<String>,
914    event_bus: &events::EventBus,
915    tmux: &Arc<TmuxManager>,
916    capture: &Arc<CaptureService>,
917    semaphore: &Arc<Semaphore>,
918    review_semaphore: &Arc<Semaphore>,
919    weight_tx: &mpsc::Sender<WeightSignal>,
920    router: &Arc<RwLock<Router>>,
921    dispatching: &Arc<dashmap::DashMap<String, String>>,
922) {
923    let configured = match config::get_projects_with_paths() {
924        Ok(p) if !p.is_empty() => p,
925        Ok(_) => return,
926        Err(e) => {
927            tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
928            return;
929        }
930    };
931    let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
932        return;
933    };
934
935    let (added, removed) = diff_configured_projects(&configured, active_repos);
936    for repo in removed {
937        tracing::info!(
938            repo = %repo,
939            "project removed from config, takes effect after restart"
940        );
941    }
942
943    for (repo, project_dir) in added {
944        config::watch(&project_dir.join(".orch.yml"));
945        let engine = match connect_project_engine(
946            repo.clone(),
947            project_dir,
948            Arc::clone(&store),
949            Some(event_bus.sender()),
950        )
951        .await
952        {
953            Ok(engine) => engine,
954            Err(e) => {
955                // A failed health check no longer lands here, so this is a backend that could
956                // not be built at all, such as a repo slug that is not `owner/name`
957                tracing::warn!(
958                    repo = %repo,
959                    err = %e,
960                    "project added to config but its backend could not be built, retrying on the next config change"
961                );
962                continue;
963            }
964        };
965
966        subscribers::dispatch::spawn(
967            event_bus.subscribe(),
968            Arc::clone(&engine.backend),
969            Arc::clone(tmux),
970            Arc::clone(&engine.runner),
971            Arc::clone(capture),
972            Arc::clone(semaphore),
973            Arc::clone(&engine.task_manager),
974            weight_tx.clone(),
975            Arc::clone(router),
976            Arc::clone(dispatching),
977            Arc::clone(&engine.store),
978            engine.repo.clone(),
979        );
980        subscribers::review::spawn(
981            event_bus.subscribe(),
982            Arc::clone(&engine.backend),
983            Arc::clone(tmux),
984            Arc::clone(review_semaphore),
985            Arc::clone(&engine.task_manager),
986            Arc::clone(router),
987            Arc::clone(dispatching),
988            Arc::clone(&engine.store),
989            engine.repo.clone(),
990        );
991        subscribers::unblock::spawn(
992            event_bus.subscribe(),
993            Arc::clone(&engine.backend),
994            Arc::clone(&engine.task_manager),
995            Arc::clone(&engine.store),
996            engine.repo.clone(),
997        );
998
999        active_repos.insert(repo.clone());
1000        project_engines.push(engine);
1001        tracing::info!(repo = %repo, "project added from config, now syncing");
1002    }
1003}
1004
1005/// Initialize all project engines from config.
1006///
1007/// Returns a vector of ProjectEngine, one for each configured project.
1008async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1009    let projects = config::get_projects_with_paths()?;
1010    let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1011    tracing::info!(repos = ?repos, "loading projects from config");
1012
1013    if projects.is_empty() {
1014        let config_path = home::config_path().map_or_else(
1015            |_| "~/.orch/config.yml".to_string(),
1016            |p| p.display().to_string(),
1017        );
1018        anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1019    }
1020
1021    let mut engines = Vec::new();
1022
1023    // Initialize unified task store once — all project engines share the same SQLite file.
1024    // Creating one pool here avoids N separate connection pools for the same database.
1025    let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1026    if cluster::enabled() {
1027        cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1028        local::serve((*store).clone()).await?;
1029    }
1030
1031    // Load persisted model cooldowns and register store for future writes.
1032    // Runs once here so the global cooldown store is not overwritten per project.
1033    engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1034
1035    for (repo, project_dir) in projects {
1036        tracing::info!(repo = %repo, "initializing project engine");
1037        config::watch(&project_dir.join(".orch.yml"));
1038
1039        engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1040    }
1041
1042    Ok(engines)
1043}
1044
1045async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1046    for engine in project_engines {
1047        let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1048        let repo_root_path = PathBuf::from(&repo_root);
1049        let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1050        let worktrees = list_project_worktrees(&repo_root_path).await?;
1051
1052        // Fetch refs once for all worktrees in this project, so rebase below
1053        // operates on up-to-date origin/* refs without N sequential fetches.
1054        if let Err(e) = git_ops::git()
1055            .await
1056            .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1057            .output_with_context()
1058            .await
1059        {
1060            tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1061        }
1062
1063        let mut orphans_removed: u32 = 0;
1064        let mut invalid_removed: u32 = 0;
1065        let mut cleaned: u32 = 0;
1066        let mut valid_kept: u32 = 0;
1067
1068        for worktree_dir in worktrees {
1069            let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1070                continue;
1071            };
1072
1073            // Validate gitdir link before doing anything else — a broken .git file
1074            // means git commands on this worktree will fail with "not a git repository".
1075            if !validate_worktree_gitdir(&worktree_dir).await {
1076                tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1077                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1078                    .await;
1079                invalid_removed += 1;
1080                continue;
1081            }
1082
1083            let Some(task_id) = task_id_from_worktree_name(name) else {
1084                tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1085                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1086                    .await;
1087                orphans_removed += 1;
1088                continue;
1089            };
1090
1091            let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1092                tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1093                remove_worktree_and_branch(
1094                    &task_id,
1095                    &worktree_dir,
1096                    Some(name),
1097                    &repo_root_path,
1098                    false,
1099                )
1100                .await;
1101                orphans_removed += 1;
1102                continue;
1103            };
1104
1105            let task = engine.store.get(store_id).await?;
1106            let branch_name = if task.branch.is_empty() {
1107                name
1108            } else {
1109                task.branch.as_str()
1110            };
1111
1112            match task.status {
1113                TaskStatus::New
1114                | TaskStatus::Routed
1115                | TaskStatus::InProgress
1116                | TaskStatus::NeedsReview
1117                | TaskStatus::InReview => {
1118                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1119                    match rebase_worktree_on_origin_main(&worktree_dir, &default_branch).await {
1120                        Err(e) => {
1121                            // If rebase failed we already made efforts to stash and
1122                            // restore changes safely. Log the error and reset the
1123                            // task as before. The worktree removal path is used
1124                            // to avoid leaving corrupted worktrees around.
1125                            tracing::warn!(repo = %engine.repo, task_id = %task_id, err = %e, "startup rebase failed, resetting task");
1126                            abort_worktree_rebase(&worktree_dir).await;
1127                            let keep_remote = task.pr_number.is_some();
1128                            remove_worktree_and_branch(
1129                                &task_id,
1130                                &worktree_dir,
1131                                Some(branch_name),
1132                                &repo_root_path,
1133                                keep_remote,
1134                            )
1135                            .await;
1136                            match engine.store.resolve_task_id(&engine.repo, &task_id).await {
1137                                Ok(Some(reset_id)) => {
1138                                    if let Err(e) = engine.store.reset_to_new(reset_id).await {
1139                                        tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1140                                    }
1141                                }
1142                                Ok(None) => {
1143                                    tracing::warn!(
1144                                        task_id,
1145                                        "task not found in store during reconciliation"
1146                                    );
1147                                }
1148                                Err(e) => {
1149                                    tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1150                                }
1151                            }
1152                            if let Err(e) = engine
1153                                .task_manager
1154                                .update_task_status(&ExternalId(task_id.clone()), Status::New)
1155                                .await
1156                            {
1157                                tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1158                            }
1159                            cleaned += 1;
1160                        }
1161                        _ => {
1162                            valid_kept += 1;
1163                        }
1164                    }
1165                }
1166                // A blocked or waiting task resumes from this work, and without a PR the branch
1167                // may hold the only copy of it. An approved task's PR is updated on GitHub
1168                TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1169                    valid_kept += 1;
1170                }
1171                TaskStatus::Done => {
1172                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1173                    remove_worktree_and_branch_for(
1174                        &task_id,
1175                        &worktree_dir,
1176                        Some(branch_name),
1177                        &repo_root_path,
1178                        false,
1179                        task.pr_number.is_some(),
1180                    )
1181                    .await;
1182                    cleaned += 1;
1183                }
1184            }
1185        }
1186
1187        tracing::info!(
1188            repo = %engine.repo,
1189            orphans_removed,
1190            invalid_removed,
1191            cleaned,
1192            valid_kept,
1193            "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1194        );
1195
1196        // Prune stale .git/worktrees/ entries whose directories no longer exist.
1197        // This handles entries left behind by crashes or manual directory removal.
1198        let prune = Command::new("git")
1199            .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1200            .output_with_context()
1201            .await;
1202        match prune {
1203            Ok(output) if output.status.success() => {
1204                tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1205            }
1206            Ok(output) => {
1207                let stderr = String::from_utf8_lossy(&output.stderr);
1208                tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1209            }
1210            Err(e) => {
1211                tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1212            }
1213        }
1214    }
1215
1216    Ok(())
1217}
1218
1219/// Sync Fibonacci estimates stored in SQLite to the GitHub Projects board for
1220/// all tasks that have a positive estimate.
1221///
1222/// Runs once on startup to catch tasks that were routed before the estimate
1223/// sync feature was available or before `project_estimate_field_id` was
1224/// configured. The underlying mutations are idempotent, so re-syncing an
1225/// already-correct estimate is harmless.
1226async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1227    for engine in project_engines {
1228        let tasks = match engine.store.list_external_tasks_with_estimates().await {
1229            Ok(t) => t,
1230            Err(e) => {
1231                tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1232                continue;
1233            }
1234        };
1235
1236        if tasks.is_empty() {
1237            continue;
1238        }
1239
1240        let mut synced: u32 = 0;
1241        let mut failed: u32 = 0;
1242        for (external_id, estimate) in &tasks {
1243            match engine
1244                .backend
1245                .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1246                .await
1247            {
1248                Ok(()) => synced += 1,
1249                Err(e) => {
1250                    tracing::debug!(
1251                        repo = %engine.repo,
1252                        task_id = %external_id,
1253                        estimate,
1254                        err = %e,
1255                        "startup estimate reconciliation: sync failed"
1256                    );
1257                    failed += 1;
1258                }
1259            }
1260        }
1261
1262        if synced > 0 || failed > 0 {
1263            tracing::info!(
1264                repo = %engine.repo,
1265                synced,
1266                failed,
1267                "startup estimate reconciliation complete"
1268            );
1269        }
1270    }
1271}
1272
1273/// Read per-project channel configuration from `.orch.yml`.
1274fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1275    let config_path = project_dir.join(".orch.yml");
1276    if !config_path.exists() {
1277        return ProjectChannelConfig::default();
1278    }
1279    let content = match fs::read_to_string(&config_path) {
1280        Ok(c) => c,
1281        Err(e) => {
1282            tracing::warn!(
1283                path = %config_path.display(),
1284                error = %e,
1285                "failed to read .orch.yml; channel config will use defaults"
1286            );
1287            return ProjectChannelConfig::default();
1288        }
1289    };
1290    let val: serde_norway::Value = match serde_norway::from_str(&content) {
1291        Ok(v) => v,
1292        Err(e) => {
1293            tracing::warn!(
1294                path = %config_path.display(),
1295                error = %e,
1296                "failed to parse .orch.yml; channel config will use defaults"
1297            );
1298            return ProjectChannelConfig::default();
1299        }
1300    };
1301    let Some(channels) = val.get("channels") else {
1302        return ProjectChannelConfig::default();
1303    };
1304    let get_str = |section: &str, key: &str| -> Option<String> {
1305        channels
1306            .get(section)
1307            .and_then(|s| s.get(key))
1308            .and_then(|v| v.as_str())
1309            .map(String::from)
1310    };
1311    ProjectChannelConfig {
1312        telegram_topic_id: get_str("telegram", "topic_id"),
1313        discord_channel_id: get_str("discord", "channel_id"),
1314    }
1315}
1316
1317/// Fetch the latest orch release tag from GitHub via the `gh` CLI (async).
1318///
1319/// Returns the version string (without the leading "v"), or `None` when the
1320/// check fails (no network, unauthenticated, `gh` not installed, etc.).
1321async fn fetch_latest_release_version() -> Option<String> {
1322    let output = Command::new("gh")
1323        .args([
1324            "api",
1325            "repos/gabrielkoerich/orch/releases/latest",
1326            "--jq",
1327            ".tag_name",
1328        ])
1329        .output()
1330        .await
1331        .ok()?;
1332    if !output.status.success() {
1333        return None;
1334    }
1335    let tag = String::from_utf8(output.stdout).ok()?;
1336    let version = tag.trim().trim_start_matches('v').to_string();
1337    if version.is_empty() {
1338        None
1339    } else {
1340        Some(version)
1341    }
1342}
1343
1344/// Check if a newer orch release is available and send channel notifications
1345/// asking the operator to upgrade manually. Upgrading is operator-only — the
1346/// service never runs `brew` itself.
1347///
1348/// Uses the store's KV to throttle checks (`upgrade:last_check_at`) and
1349/// deduplicate notifications (`upgrade:last_notified_version`). Each distinct
1350/// latest version is notified about once; the notification repeats only when
1351/// an even newer version is released.
1352async fn check_and_notify_upgrade(
1353    store: &Arc<TaskStore>,
1354    channels: &Arc<ChannelRegistry>,
1355    current_version: &str,
1356    check_interval: Duration,
1357) {
1358    let last_check_key = "upgrade:last_check_at";
1359    let last_notified_key = "upgrade:last_notified_version";
1360
1361    // Throttle: only check once per interval.
1362    let now = chrono::Utc::now().timestamp();
1363    if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1364        && let Ok(last_check) = last_check_str.parse::<i64>()
1365        && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1366    {
1367        return;
1368    }
1369
1370    // Record check timestamp (best-effort).
1371    let _ = store.kv_set(last_check_key, &now.to_string()).await;
1372
1373    let Some(latest) = fetch_latest_release_version().await else {
1374        tracing::debug!("upgrade check: failed to fetch latest release version");
1375        return;
1376    };
1377
1378    if latest == current_version {
1379        tracing::debug!(current = %current_version, "orch is up to date");
1380        return;
1381    }
1382
1383    tracing::warn!(
1384        current_version = %current_version,
1385        latest_version = %latest,
1386        "orch upgrade available"
1387    );
1388
1389    // Operator-only upgrade: send channel notifications so the operator can
1390    // upgrade on their own schedule. The service never runs brew itself.
1391
1392    // Deduplicate: don't re-notify for the same latest version.
1393    if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1394        && last_notified == latest
1395    {
1396        tracing::debug!(latest = %latest, "upgrade already notified");
1397        return;
1398    }
1399
1400    let msg_telegram = format!(
1401        "⚠️ <b>Orch Upgrade Available</b>\n\n\
1402         Service: <code>{current_version}</code>\n\
1403         Latest:  <code>{latest}</code>\n\n\
1404         Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1405    );
1406
1407    let msg_discord = format!(
1408        "⚠️ **Orch Upgrade Available**\n\n\
1409         Service: `{current_version}`\n\
1410         Latest:  `{latest}`\n\n\
1411         Run: `brew update && brew upgrade orch && brew services restart orch`"
1412    );
1413
1414    let msg_slack = format!(
1415        "⚠️ *Orch Upgrade Available*\n\n\
1416         Service: `{current_version}`\n\
1417         Latest:  `{latest}`\n\n\
1418         Run: `brew update && brew upgrade orch && brew services restart orch`"
1419    );
1420
1421    for channel in channels.iter() {
1422        let (body, metadata, topic_id) = match channel.name() {
1423            "telegram" => (
1424                msg_telegram.clone(),
1425                serde_json::json!({"preformatted_html": true}),
1426                None,
1427            ),
1428            "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1429            "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1430            _ => continue,
1431        };
1432
1433        let msg = OutgoingMessage {
1434            thread_id: "upgrade".to_string(),
1435            body,
1436            reply_to: None,
1437            metadata,
1438            topic_id,
1439        };
1440
1441        if let Err(e) = channel.send(&msg).await {
1442            tracing::warn!(
1443                channel = channel.name(),
1444                ?e,
1445                "failed to send upgrade notification"
1446            );
1447        }
1448    }
1449
1450    // Record that we notified for this latest version.
1451    let _ = store.kv_set(last_notified_key, &latest).await;
1452}
1453
1454/// Send a one-time escalation notification when the startup retry loop has been
1455/// spinning for longer than `startup_failure_escalation_secs`.
1456///
1457/// This is called *before* the channel registry is initialised (the retry loop
1458/// sits above it in `serve()`), so we construct channel objects directly from
1459/// config — mirroring the approach used during normal channel registration, but
1460/// without relying on `ChannelRegistry`.
1461async fn send_startup_escalation_notification(elapsed_secs: u64) {
1462    #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1463    let duration = notification::format_duration(elapsed_secs as f64);
1464
1465    let msg_telegram = format!(
1466        "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1467         The engine has been retrying backend connections for {duration}\n\
1468         without success. All task processing, routing, and scheduled\n\
1469         jobs are paused. Check service logs with `orch log -f`."
1470    );
1471
1472    let msg_discord = format!(
1473        "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1474         The engine has been retrying backend connections for {duration}\n\
1475         without success. All task processing, routing, and scheduled\n\
1476         jobs are paused. Check service logs with `orch log -f`."
1477    );
1478
1479    let msg_slack = format!(
1480        "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1481         The engine has been retrying backend connections for {duration}\n\
1482         without success. All task processing, routing, and scheduled\n\
1483         jobs are paused. Check service logs with `orch log -f`."
1484    );
1485
1486    // Telegram
1487    let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1488    let tg_chat_id = config::get("channels.telegram.chat_id");
1489    if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1490        && !token.is_empty()
1491        && !chat_id.is_empty()
1492        && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1493    {
1494        let msg = OutgoingMessage {
1495            thread_id: "startup_escalation".to_string(),
1496            body: msg_telegram.clone(),
1497            reply_to: None,
1498            metadata: serde_json::json!({ "preformatted_html": true }),
1499            topic_id: config::get("channels.telegram.general_topic_id").ok(),
1500        };
1501        match channel.send(&msg).await {
1502            Err(e) => {
1503                tracing::warn!(?e, "failed to send startup escalation via telegram");
1504            }
1505            _ => {
1506                tracing::info!("startup escalation notification sent to telegram");
1507            }
1508        }
1509    }
1510
1511    // Discord
1512    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1513        let channel_id = config::get("channels.discord.channel_id")
1514            .or_else(|_| config::get("channels.discord.general_channel_id"));
1515        if !token.is_empty()
1516            && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1517        {
1518            let shard_id = config::get("channels.discord.shard_id")
1519                .ok()
1520                .and_then(|v| v.parse::<u64>().ok())
1521                .unwrap_or(0);
1522            let shard_count = config::get("channels.discord.shard_count")
1523                .ok()
1524                .and_then(|v| v.parse::<u64>().ok())
1525                .unwrap_or(1);
1526            if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1527                let msg = OutgoingMessage {
1528                    thread_id: "startup_escalation".to_string(),
1529                    body: msg_discord.clone(),
1530                    reply_to: None,
1531                    metadata: serde_json::Value::Null,
1532                    topic_id: None,
1533                };
1534                match gateway.send(&msg).await {
1535                    Err(e) => {
1536                        tracing::warn!(?e, "failed to send startup escalation via discord");
1537                    }
1538                    _ => {
1539                        tracing::info!("startup escalation notification sent to discord");
1540                    }
1541                }
1542            }
1543        }
1544    }
1545
1546    // Slack
1547    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1548        let channel_id = config::get("channels.slack.channel_id");
1549        if !token.is_empty()
1550            && let Ok(cid) = channel_id
1551            && let Ok(channel) = SlackChannel::new(token, Some(cid))
1552        {
1553            let msg = OutgoingMessage {
1554                thread_id: "startup_escalation".to_string(),
1555                body: msg_slack.clone(),
1556                reply_to: None,
1557                metadata: serde_json::Value::Null,
1558                topic_id: None,
1559            };
1560            match channel.send(&msg).await {
1561                Err(e) => {
1562                    tracing::warn!(?e, "failed to send startup escalation via slack");
1563                }
1564                _ => {
1565                    tracing::info!("startup escalation notification sent to slack");
1566                }
1567            }
1568        }
1569    }
1570}
1571
1572/// Start the orch service.
1573///
1574/// This is the main entry point — called by `orch serve`.
1575pub async fn serve() -> anyhow::Result<()> {
1576    let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1577    if killed > 0 {
1578        tracing::info!(count = killed, "terminated stale orch serve processes");
1579    }
1580
1581    tracing::info!("orch engine starting");
1582
1583    // Pre-create standard directories with async I/O so subsequent synchronous
1584    // callers (state_dir, orch_home, etc.) find them already present and their
1585    // create_dir_all calls become fast no-ops.
1586    if let Err(e) = home::init_dirs().await {
1587        tracing::warn!("failed to initialize orch directories: {e}");
1588    }
1589
1590    // Pre-warm the config cache with async I/O so get() / get_list() calls in
1591    // hot async paths never block a Tokio thread on std::fs::read_to_string.
1592    if let Err(e) = config::warm_cache().await {
1593        tracing::warn!("failed to warm config cache: {e}");
1594    }
1595
1596    let mut config = EngineConfig::from_config();
1597
1598    tracing::info!("internal database ready");
1599
1600    // Initialize project engines — retry with backoff so a network outage at
1601    // startup doesn't cause a crash-loop (launchd KeepAlive would restart us
1602    // immediately, filling the error log with 600+ identical messages).
1603    //
1604    // IMPORTANT: errors from init_project_engines() must NEVER propagate past
1605    // this loop to main().  If they did, anyhow would write "Error: …" to stderr,
1606    // which brew routes to orch.error.log — polluting it even when projects ARE
1607    // configured correctly but GitHub is temporarily unreachable.
1608    let startup_escalation_secs = config.startup_failure_escalation_secs;
1609    let mut project_engines = {
1610        let mut delay_secs = 5u64;
1611        let mut attempt = 0u32;
1612        let mut escalated = false;
1613        let start = Instant::now();
1614        loop {
1615            attempt += 1;
1616            match init_project_engines().await {
1617                Ok(engines) => break engines,
1618                Err(e) => {
1619                    let elapsed = start.elapsed().as_secs();
1620
1621                    // Short blips (under the escalation threshold) stay at WARN
1622                    // so they don't flood the operator's attention. Once the
1623                    // outage crosses the threshold, escalate to ERROR-level
1624                    // logging and send a one-time push notification to configured
1625                    // channels — "transient" has clearly become "prolonged."
1626                    if startup_escalation_secs > 0
1627                        && !escalated
1628                        && elapsed >= startup_escalation_secs
1629                    {
1630                        escalated = true;
1631                        #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1632                        let outage = notification::format_duration(elapsed as f64);
1633                        tracing::error!(
1634                            delay_secs,
1635                            attempt,
1636                            elapsed_secs = elapsed,
1637                            threshold_secs = startup_escalation_secs,
1638                            "project backends unavailable for {outage} — escalating startup failure: {e}",
1639                        );
1640                        // Send one-time notification to configured channels.
1641                        // Channels aren't registered yet at this stage, so we
1642                        // construct them directly from config (mirrors orch notify).
1643                        send_startup_escalation_notification(elapsed).await;
1644                    } else {
1645                        tracing::warn!(
1646                            delay_secs,
1647                            attempt,
1648                            "project backends unavailable, retrying: {e}"
1649                        );
1650                    }
1651
1652                    time::sleep(Duration::from_secs(delay_secs)).await;
1653                    delay_secs = (delay_secs * 2).min(120);
1654                }
1655            }
1656        }
1657    };
1658
1659    tracing::info!(
1660        project_count = project_engines.len(),
1661        "initialized project engines"
1662    );
1663
1664    // Full set of currently-configured project repos, used by the per-tick global
1665    // sweeps (e.g. `dispatch_routed_tasks_global`) to distinguish "orphaned" tasks
1666    // (repo no longer in config) from tasks belonging to some other active project.
1667    let mut active_repos: HashSet<String> =
1668        project_engines.iter().map(|e| e.repo.clone()).collect();
1669
1670    // Create the event bus for task status transitions
1671    let event_bus = events::EventBus::new(256);
1672    if let Err(e) = event_bus.start_ws_server().await {
1673        tracing::warn!(
1674            ?e,
1675            "failed to start event websocket server, continuing without it"
1676        );
1677    }
1678
1679    // Check for a newer orch release in the background and log if the service is behind.
1680    // The periodic checker sends channel notifications after channels are initialized;
1681    // upgrading is operator-only and never runs automatically.
1682    {
1683        let current = env!("ORCH_VERSION").to_string();
1684        tokio::spawn(async move {
1685            if let Some(latest) = fetch_latest_release_version().await
1686                && latest != current
1687            {
1688                tracing::warn!(
1689                    current_version = %current,
1690                    latest_version = %latest,
1691                    "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1692                );
1693            }
1694        });
1695    }
1696
1697    // Re-create task managers with shared store and event bus
1698    for engine in &mut project_engines {
1699        engine.task_manager = Arc::new(TaskManager::with_events(
1700            Arc::clone(&engine.backend),
1701            Arc::clone(&engine.store),
1702            engine.repo.clone(),
1703            event_bus.sender(),
1704        ));
1705    }
1706
1707    // Startup reconciliation decides task status, so only the leader runs it
1708    let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1709    if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1710        tracing::warn!(err = %e, "startup worktree reconciliation failed");
1711    }
1712
1713    if leads {
1714        // Run estimate reconciliation in the background so it never blocks startup.
1715        // Each GraphQL mutation takes ~0.8 s; with hundreds of tasks this would
1716        // otherwise delay the engine reaching its main loop by several minutes.
1717        let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1718        tokio::spawn(async move {
1719            reconcile_startup_estimates(&engines_snapshot).await;
1720        });
1721    }
1722
1723    // Build ChannelRouter from global config + per-project configs
1724    let global_channel_config = GlobalChannelConfig {
1725        telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1726        discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1727        control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1728        control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1729    };
1730    let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1731        .iter()
1732        .map(|e| {
1733            let cfg = read_project_channel_config(&e.project_dir);
1734            (e.repo.clone(), cfg)
1735        })
1736        .collect();
1737    let channel_router = Arc::new(ChannelRouter::new(
1738        &global_channel_config,
1739        &project_channel_configs,
1740    ));
1741    tracing::info!(
1742        projects = ?channel_router.projects(),
1743        "channel router initialized"
1744    );
1745
1746    // Initialize tmux manager (shared across all projects)
1747    let tmux = Arc::new(TmuxManager::new());
1748    // Kill leftover sessions from a previous process. This is daemon start only,
1749    // never a tick, because nextest makes a once-flag in tick_all kill live agents
1750    tick::startup_cleanup(&tmux).await;
1751
1752    // Initialize transport
1753    let transport = Arc::new(Transport::new());
1754
1755    // Initialize capture service and start background loop
1756    let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1757    let capture_for_tick = Arc::clone(&capture);
1758    tokio::spawn(async move {
1759        capture.start().await;
1760    });
1761
1762    // Initialize channel registry
1763    let mut channel_registry = ChannelRegistry::new();
1764
1765    // Try to initialize Telegram channel
1766    if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1767        && !token.is_empty()
1768    {
1769        let chat_id = config::get("channels.telegram.chat_id").ok();
1770        match TelegramChannel::new(token, chat_id) {
1771            Ok(telegram) => {
1772                if let Err(e) = telegram.health_check().await {
1773                    tracing::warn!(?e, "telegram channel health check failed, skipping");
1774                } else {
1775                    channel_registry.register(Box::new(telegram));
1776                    tracing::info!("telegram channel registered");
1777                }
1778            }
1779            Err(e) => {
1780                tracing::warn!(?e, "failed to create telegram channel, skipping");
1781            }
1782        }
1783    }
1784
1785    // Try to initialize Discord Gateway channel (websocket)
1786    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1787        && !token.is_empty()
1788    {
1789        let channel_id = config::get("channels.discord.channel_id").ok();
1790        let shard_id = config::get("channels.discord.shard_id")
1791            .ok()
1792            .and_then(|v| v.parse::<u64>().ok())
1793            .unwrap_or(0);
1794        let shard_count = config::get("channels.discord.shard_count")
1795            .ok()
1796            .and_then(|v| v.parse::<u64>().ok())
1797            .unwrap_or(1);
1798        match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1799            Ok(discord) => {
1800                if let Err(e) = discord.health_check().await {
1801                    tracing::warn!(?e, "discord gateway health check failed, skipping");
1802                } else {
1803                    channel_registry.register(Box::new(discord));
1804                    tracing::info!(shard_id, shard_count, "discord gateway registered");
1805                }
1806            }
1807            Err(e) => {
1808                tracing::warn!(?e, "failed to create discord gateway, skipping");
1809            }
1810        }
1811    }
1812
1813    // Try to initialize Slack channel
1814    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1815        && !token.is_empty()
1816    {
1817        let channel_id = config::get("channels.slack.channel_id").ok();
1818        match SlackChannel::new(token, channel_id) {
1819            Ok(slack) => {
1820                if let Err(e) = slack.health_check().await {
1821                    tracing::warn!(?e, "slack channel health check failed, skipping");
1822                } else {
1823                    channel_registry.register(Box::new(slack));
1824                    tracing::info!("slack channel registered");
1825                }
1826            }
1827            Err(e) => {
1828                tracing::warn!(?e, "failed to create slack channel, skipping");
1829            }
1830        }
1831    }
1832
1833    // Initialize tmux channel with transport for output streaming
1834    let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1835    channel_registry.register(Box::new(tmux_channel));
1836    tracing::info!("tmux channel registered");
1837
1838    // Start all channels and collect their message receivers
1839    let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1840    for channel in channel_registry.iter() {
1841        match channel.start().await {
1842            Ok(rx) => {
1843                tracing::info!(channel = channel.name(), "channel started");
1844                channel_receivers.push(rx);
1845            }
1846            Err(e) => {
1847                tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1848            }
1849        }
1850    }
1851
1852    // Wrap channel registry in Arc for shared access (notification dispatcher needs it)
1853    let channel_registry = Arc::new(channel_registry);
1854
1855    // Spawn tasks to handle incoming channel messages (if any channels are active)
1856    let transport_for_messages = Arc::clone(&transport);
1857    let tmux_for_messages = Arc::clone(&tmux);
1858    let capture_for_messages = Arc::clone(&capture_for_tick);
1859    let channels_for_messages = Arc::clone(&channel_registry);
1860    // Lightweight engine references for command execution and task creation
1861    let engine_refs: Vec<EngineRef> = project_engines
1862        .iter()
1863        .map(|e| {
1864            (
1865                e.repo.clone(),
1866                Arc::clone(&e.backend),
1867                Arc::clone(&e.task_manager),
1868                Some(Arc::clone(&e.store)),
1869            )
1870        })
1871        .collect();
1872    // Shared in-memory map for pending project picks (General channel multi-project picker).
1873    let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1874    let router_for_messages = Arc::clone(&channel_router);
1875    for mut rx in channel_receivers {
1876        let transport = Arc::clone(&transport_for_messages);
1877        let tmux = Arc::clone(&tmux_for_messages);
1878        let capture = Arc::clone(&capture_for_messages);
1879        let channels = Arc::clone(&channels_for_messages);
1880        let engine_refs = engine_refs.clone();
1881        let ch_router = Arc::clone(&router_for_messages);
1882        let picks = Arc::clone(&pending_picks);
1883        tokio::spawn(async move {
1884            while let Some(msg) = rx.recv().await {
1885                tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1886                channel_handler::handle_channel_message(
1887                    msg,
1888                    &transport,
1889                    &tmux,
1890                    &capture,
1891                    &channels,
1892                    &engine_refs,
1893                    &ch_router,
1894                    &picks,
1895                )
1896                .await;
1897            }
1898        });
1899    }
1900
1901    // Post-restart confirmation: if the service was restarted via the /restart control command,
1902    // a KV marker was stored before shutdown. Send a confirmation to the originating channel now
1903    // that we're back online. Use a short delay to let the channel connections re-establish.
1904    {
1905        let restart_channels = Arc::clone(&channel_registry);
1906        let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1907        if let Some(store) = restart_store {
1908            tokio::spawn(async move {
1909                time::sleep(Duration::from_secs(5)).await;
1910                match store.kv_get("control:restart_pending").await {
1911                    Ok(Some(conv_key)) => {
1912                        if let Some((ch_name, thread_id, topic_id)) =
1913                            transport::parse_conversation_key(&conv_key)
1914                        {
1915                            for ch in restart_channels.iter() {
1916                                if ch.name() == ch_name {
1917                                    let msg = OutgoingMessage {
1918                                        thread_id: thread_id.to_string(),
1919                                        body: "orch restarted successfully.".to_string(),
1920                                        reply_to: None,
1921                                        metadata: serde_json::json!({}),
1922                                        topic_id: topic_id.map(ToString::to_string),
1923                                    };
1924                                    if let Err(e) = ch.send(&msg).await {
1925                                        tracing::warn!(
1926                                            channel = ch_name,
1927                                            error = %e,
1928                                            "failed to send restart confirmation"
1929                                        );
1930                                    }
1931                                    break;
1932                                }
1933                            }
1934                        }
1935                        if let Err(e) = store.kv_delete("control:restart_pending").await {
1936                            tracing::warn!(error = %e, "failed to clear control:restart_pending");
1937                        }
1938                    }
1939                    Ok(None) => {} // no pending restart, nothing to do
1940                    Err(e) => {
1941                        tracing::warn!(error = %e, "failed to read control:restart_pending");
1942                    }
1943                }
1944            });
1945        }
1946    }
1947
1948    // Spawn notification dispatcher — reads task completion notifications
1949    // from transport and broadcasts to configured channels.
1950    //
1951    // Routing priority:
1952    // 1. Dedicated channel: if the notification has a `repo` and the
1953    //    ChannelRouter maps it to a topic/channel, send there with `topic_id`.
1954    // 2. Subscribed channels: send to each (channel, thread_id) subscribed to
1955    //    the repo, using `format_with_project()` so recipients know the source.
1956    // 3. Fallback: if no repo or no dedicated channel, broadcast to all
1957    //    channels without a topic_id (legacy behaviour).
1958    {
1959        let mut notification_rx = transport.subscribe_notifications();
1960        let channels = Arc::clone(&channel_registry);
1961        let notif_router = Arc::clone(&channel_router);
1962        // Grab a store reference for subscription lookups.
1963        let notif_store: Option<Arc<TaskStore>> =
1964            project_engines.first().map(|e| Arc::clone(&e.store));
1965        tokio::spawn(async move {
1966            loop {
1967                match notification_rx.recv().await {
1968                    Ok(notification) => {
1969                        let level = NotificationLevel::from_config();
1970                        if !level.should_notify(&notification.status) {
1971                            tracing::debug!(
1972                                task_id = %notification.task_id,
1973                                status = %notification.status,
1974                                "notification suppressed by level={:?}",
1975                                level
1976                            );
1977                            continue;
1978                        }
1979
1980                        tracing::info!(
1981                            task_id = %notification.task_id,
1982                            status = %notification.status,
1983                            repo = ?notification.repo,
1984                            "routing notification to channels"
1985                        );
1986
1987                        // Track whether we sent to at least one dedicated/subscribed target.
1988                        let mut routed = false;
1989
1990                        // 0. Direct target: if notify_target is set, send straight to
1991                        //    that Telegram chat_id and skip all other routing.
1992                        if let Some(ref target) = notification.notify_target {
1993                            let telegram = channels.iter().find(|c| c.name() == "telegram");
1994                            if let Some(channel) = telegram {
1995                                let body = notification.format_telegram();
1996                                let metadata = serde_json::json!({
1997                                    "preformatted_html": true,
1998                                    "chat_id_override": target
1999                                });
2000                                let msg = OutgoingMessage {
2001                                    thread_id: notification.task_id.clone(),
2002                                    body,
2003                                    reply_to: None,
2004                                    metadata,
2005                                    topic_id: None,
2006                                };
2007                                match channel.send(&msg).await {
2008                                    Err(e) => {
2009                                        tracing::warn!(
2010                                            task_id = %notification.task_id,
2011                                            target,
2012                                            ?e,
2013                                            "failed to send to notify_target"
2014                                        );
2015                                    }
2016                                    _ => {
2017                                        routed = true;
2018                                    }
2019                                }
2020                            }
2021                        }
2022
2023                        if routed {
2024                            // notify_target handled — skip further routing
2025                        } else if let Some(repo) = notification.repo.as_deref() {
2026                            // Track destinations already sent to in this iteration so
2027                            // the dedicated (path 1) and subscribed (path 2) loops
2028                            // don't double-fire the same Telegram/Discord destination
2029                            // when a repo's configured topic and a `channel_subscriptions`
2030                            // row resolve to the same `(channel, topic_id)`.
2031                            let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2032
2033                            // 1. Dedicated channel targets for this repo.
2034                            for channel in channels.iter() {
2035                                let ch_name = channel.name();
2036                                if let Some(topic_id) =
2037                                    notif_router.target_for_project(repo, ch_name)
2038                                {
2039                                    let body = match ch_name {
2040                                        "telegram" => notification.format_telegram(),
2041                                        "discord" => notification.format_discord(),
2042                                        "slack" => notification.format_slack(),
2043                                        _ => continue,
2044                                    };
2045                                    let metadata = if ch_name == "telegram" {
2046                                        serde_json::json!({"preformatted_html": true})
2047                                    } else {
2048                                        serde_json::json!({})
2049                                    };
2050                                    let resolved_topic = Some(topic_id.to_string());
2051                                    let msg = OutgoingMessage {
2052                                        thread_id: notification.task_id.clone(),
2053                                        body,
2054                                        reply_to: None,
2055                                        metadata,
2056                                        topic_id: resolved_topic.clone(),
2057                                    };
2058                                    if let Err(e) = channel.send(&msg).await {
2059                                        tracing::warn!(
2060                                            channel = ch_name,
2061                                            task_id = %notification.task_id,
2062                                            ?e,
2063                                            "failed to send to dedicated channel"
2064                                        );
2065                                    } else {
2066                                        sent.insert((ch_name.to_string(), resolved_topic));
2067                                        routed = true;
2068                                    }
2069                                }
2070                            }
2071
2072                            // 2. Subscribed channels for this repo.
2073                            if let Some(store) = &notif_store {
2074                                match store.list_subscribers_for_repo(repo).await {
2075                                    Ok(subscribers) => {
2076                                        for (ch_name, thread_id, topic_id) in subscribers {
2077                                            let channel =
2078                                                channels.iter().find(|c| c.name() == ch_name);
2079                                            let Some(channel) = channel else {
2080                                                continue;
2081                                            };
2082                                            let resolved_topic = if topic_id.is_empty() {
2083                                                None
2084                                            } else {
2085                                                Some(topic_id.clone())
2086                                            };
2087                                            // Skip if a dedicated send to the same
2088                                            // (channel, topic_id) already happened.
2089                                            if sent.contains(&(
2090                                                ch_name.clone(),
2091                                                resolved_topic.clone(),
2092                                            )) {
2093                                                tracing::debug!(
2094                                                    channel = %ch_name,
2095                                                    task_id = %notification.task_id,
2096                                                    topic_id = ?resolved_topic,
2097                                                    "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2098                                                );
2099                                                continue;
2100                                            }
2101                                            let body = notification.format_with_project(&ch_name);
2102                                            let metadata = if ch_name == "telegram" {
2103                                                serde_json::json!({"preformatted_html": true})
2104                                            } else {
2105                                                serde_json::json!({})
2106                                            };
2107                                            let msg = OutgoingMessage {
2108                                                thread_id: thread_id.clone(),
2109                                                body,
2110                                                reply_to: None,
2111                                                metadata,
2112                                                topic_id: resolved_topic.clone(),
2113                                            };
2114                                            if let Err(e) = channel.send(&msg).await {
2115                                                tracing::warn!(
2116                                                    channel = %ch_name,
2117                                                    task_id = %notification.task_id,
2118                                                    ?e,
2119                                                    "failed to send to subscribed channel"
2120                                                );
2121                                            } else {
2122                                                sent.insert((ch_name.clone(), resolved_topic));
2123                                                routed = true;
2124                                            }
2125                                        }
2126                                    }
2127                                    Err(e) => {
2128                                        tracing::warn!(
2129                                            repo,
2130                                            ?e,
2131                                            "failed to list subscribers for repo"
2132                                        );
2133                                    }
2134                                }
2135                            }
2136                        }
2137
2138                        // 3. Fallback: broadcast to all channels when no routing happened.
2139                        if !routed {
2140                            for channel in channels.iter() {
2141                                let (body, should_send, is_telegram) = match channel.name() {
2142                                    "telegram" => (notification.format_telegram(), true, true),
2143                                    "discord" => (notification.format_discord(), true, false),
2144                                    "slack" => (notification.format_slack(), true, false),
2145                                    // GitHub is already handled by backend.post_comment()
2146                                    // tmux doesn't need task completion notifications
2147                                    _ => (String::new(), false, false),
2148                                };
2149
2150                                if !should_send {
2151                                    continue;
2152                                }
2153
2154                                let metadata = if is_telegram {
2155                                    serde_json::json!({"preformatted_html": true})
2156                                } else {
2157                                    serde_json::json!({})
2158                                };
2159                                let msg = OutgoingMessage {
2160                                    thread_id: notification.task_id.clone(),
2161                                    body,
2162                                    reply_to: None,
2163                                    metadata,
2164                                    topic_id: None,
2165                                };
2166
2167                                if let Err(e) = channel.send(&msg).await {
2168                                    tracing::warn!(
2169                                        channel = channel.name(),
2170                                        task_id = %notification.task_id,
2171                                        ?e,
2172                                        "failed to send notification"
2173                                    );
2174                                }
2175                            }
2176                        }
2177                    }
2178                    Err(RecvError::Lagged(n)) => {
2179                        tracing::warn!(missed = n, "notification receiver lagged");
2180                    }
2181                    Err(RecvError::Closed) => {
2182                        tracing::debug!("notification channel closed");
2183                        break;
2184                    }
2185                }
2186            }
2187        });
2188        tracing::info!("notification dispatcher started");
2189    }
2190
2191    // Notify used by webhook events to wake up the engine tick immediately
2192    let webhook_notify = Arc::new(Notify::new());
2193
2194    // `WebhookStatus::fallback_mode` is only shown by `orch webhook status`, the sync
2195    // tick always runs every `engine.sync_interval` whether or not webhooks work
2196    let webhook_port: Option<u16>;
2197    let mut webhook_healthy: bool;
2198    let mut last_webhook_health_check = Instant::now();
2199    // Start webhook server if configured
2200    let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2201    // No readable secret keeps the webhook server off, it never accepts unsigned requests
2202    let webhook_secret = if webhook_enabled {
2203        secret_ref::config_secret("webhook.secret")
2204            .await
2205            .ok()
2206            .filter(|s| !s.trim().is_empty())
2207    } else {
2208        None
2209    };
2210    if webhook_enabled && webhook_secret.is_none() {
2211        tracing::error!(
2212            "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2213        );
2214    }
2215    let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2216    let webhook_secret = webhook_secret.unwrap_or_default();
2217
2218    // Shared status updated by the server-spawn task and the health-check loop.
2219    let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2220
2221    if webhook_enabled {
2222        // Spawn the HTTP server with exponential-backoff retry on transient
2223        // bind errors (e.g. EADDRINUSE).  After MAX_ATTEMPTS the engine falls
2224        // back to polling and updates the shared status accordingly.
2225        const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2226        let port: u16 = config::get("webhook.port")
2227            .ok()
2228            .and_then(|v| v.parse().ok())
2229            .unwrap_or(8080);
2230        webhook_port = Some(port);
2231        let secret = webhook_secret.clone();
2232        let webhook_repo = project_engines
2233            .first()
2234            .map(|e| e.repo.clone())
2235            .unwrap_or_default();
2236
2237        let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2238
2239        // Initialise status: configured, not yet healthy.
2240        let to_save = {
2241            let mut s = webhook_status.lock().await;
2242            s.configured = true;
2243            s.port = Some(port);
2244            s.healthy = false;
2245            s.fallback_mode = false;
2246            s.clone()
2247        };
2248        to_save.save().await;
2249
2250        let status_for_spawn = Arc::clone(&webhook_status);
2251        tokio::spawn(async move {
2252            let mut attempt = 0u32;
2253            loop {
2254                attempt += 1;
2255                tracing::info!(attempt, port, "attempting to start webhook server");
2256                match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2257                    .await
2258                {
2259                    Ok(()) => {
2260                        tracing::info!(attempt, "webhook server exited cleanly");
2261                        break;
2262                    }
2263                    Err(e) => {
2264                        let reason = e.to_string();
2265                        tracing::error!(attempt, %reason, "webhook server failed");
2266                        if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2267                            let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2268                                "max attempts reached"
2269                            } else {
2270                                "non-transient error"
2271                            };
2272                            tracing::error!(
2273                                attempt,
2274                                %reason,
2275                                kind,
2276                                orch_webhook_in_fallback = true,
2277                                "webhook server giving up, switching to polling fallback"
2278                            );
2279                            let to_save = {
2280                                let mut s = status_for_spawn.lock().await;
2281                                s.fallback_mode = true;
2282                                s.healthy = false;
2283                                s.last_failure_reason = Some(reason);
2284                                s.startup_attempts = attempt;
2285                                s.clone()
2286                            };
2287                            to_save.save().await;
2288                            break;
2289                        }
2290                        let delay = webhook_backoff_delay(attempt);
2291                        tracing::warn!(
2292                            attempt,
2293                            delay_ms = delay.as_millis(),
2294                            %reason,
2295                            "webhook bind failed (transient), retrying with backoff"
2296                        );
2297                        {
2298                            let to_save = {
2299                                let mut s = status_for_spawn.lock().await;
2300                                s.startup_attempts = attempt;
2301                                s.last_failure_reason = Some(reason);
2302                                s.clone()
2303                            };
2304                            to_save.save().await;
2305                        }
2306                        time::sleep(delay).await;
2307                    }
2308                }
2309            }
2310        });
2311
2312        // Spawn the message forwarding task (reads from webhook channel)
2313        let notify = Arc::clone(&webhook_notify);
2314        let transport_for_webhook = Arc::clone(&transport);
2315        tokio::spawn(async move {
2316            while let Some(msg) = rx.recv().await {
2317                tracing::info!(
2318                    channel = %msg.channel,
2319                    thread = %msg.thread_id,
2320                    event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2321                    action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2322                    "webhook event received, triggering immediate tick"
2323                );
2324                let route = transport_for_webhook.route(&msg).await;
2325                tracing::info!(
2326                    channel = %msg.channel,
2327                    route = ?route,
2328                    "webhook event routed"
2329                );
2330                // Wake up the engine tick immediately instead of waiting up to 10s
2331                notify.notify_one();
2332            }
2333        });
2334
2335        webhook_healthy = true;
2336        tracing::info!(port, orch_webhook_up = true, "webhook server started");
2337    } else {
2338        webhook_port = None;
2339        webhook_healthy = false;
2340        {
2341            let to_save = {
2342                let mut s = webhook_status.lock().await;
2343                s.configured = false;
2344                s.fallback_mode = true;
2345                s.clone()
2346            };
2347            to_save.save().await;
2348        }
2349        tracing::info!(
2350            orch_webhook_in_fallback = true,
2351            "webhook server disabled, using polling fallback mode"
2352        );
2353    }
2354
2355    start_relay_client(&project_engines, &webhook_notify);
2356    app::start_loader();
2357
2358    // Agent router (selects agent + model per task) - shared across projects.
2359    // Router::from_config() may run `opencode models` when the config uses opencode.
2360    // spawn_blocking keeps the Tokio runtime thread free during that I/O
2361    let router = Arc::new(RwLock::new(
2362        match task::spawn_blocking(Router::from_config).await {
2363            Ok(r) => r,
2364            Err(e) => {
2365                tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2366                task::spawn_blocking(|| Router::new(RouterConfig::default()))
2367                    .await
2368                    .unwrap_or_else(|e2| {
2369                        panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2370                    })
2371            }
2372        },
2373    ));
2374    {
2375        let r = router.read().await;
2376        tracing::info!(
2377            mode = %r.config.mode,
2378            agents = ?r.available_agents,
2379            fallback = %r.config.fallback_executor,
2380            "router initialized"
2381        );
2382    }
2383
2384    // Concurrency limiter (shared across all projects). `parallel_in_force` is the
2385    // limit actually handed out, which lags `config.max_parallel` after a shrink
2386    // until the running tasks give their permits back.
2387    let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2388    let mut parallel_in_force = config.max_parallel;
2389    // Reviews get their own pool, one reviewer per worker, so a busy worker pool never
2390    // starves a review (#3971)
2391    let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2392    let mut review_in_force = config.max_parallel;
2393
2394    // In-memory dispatch guard: tracks task IDs currently being dispatched.
2395    // Guards against GitHub API eventual consistency — after update_status(InProgress),
2396    // the label-removal webhook can trigger an immediate tick where list_by_status(Routed)
2397    // still returns the task (search index propagation delay). The tmux session does not
2398    // exist until the runner completes worktree setup (~10s later), so session_exists
2399    // alone is insufficient. Keyed by "{repo}/{task_id}".
2400    let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2401
2402    // In-memory guard to prevent double-spawn of auto-merge background tasks.
2403    // Since review_open_prs runs every sync_tick (~10s) but CI polling can take
2404    // up to 10 minutes, we track which tasks already have an auto-merge in flight.
2405    let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2406
2407    // Subscribe to config file changes for hot reload
2408    let mut config_rx = config::subscribe();
2409
2410    // Track sync interval
2411    let mut last_sync = Instant::now();
2412
2413    // Track upgrade check interval
2414    let mut last_upgrade_check: Option<Instant> = None;
2415
2416    // Reset epoch of the GitHub rate-limit window already reported, so the log line,
2417    // the KV flag and the channel notification fire once per window instead of per tick
2418    let mut rate_limit_reported: Option<i64> = None;
2419
2420    // Channel for weight signals from task runners back to the router
2421    let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2422
2423    // A restart ends every review session, so reset each in_review task left without one
2424    for engine in project_engines.iter().filter(|_| leads) {
2425        let reset =
2426            reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2427
2428        if reset > 0 {
2429            tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2430        }
2431    }
2432
2433    // Spawn dispatch subscribers — react to Routed events immediately
2434    for engine in &project_engines {
2435        subscribers::dispatch::spawn(
2436            event_bus.subscribe(),
2437            Arc::clone(&engine.backend),
2438            Arc::clone(&tmux),
2439            Arc::clone(&engine.runner),
2440            Arc::clone(&capture_for_tick),
2441            Arc::clone(&semaphore),
2442            Arc::clone(&engine.task_manager),
2443            weight_tx.clone(),
2444            Arc::clone(&router),
2445            Arc::clone(&dispatching),
2446            Arc::clone(&engine.store),
2447            engine.repo.clone(),
2448        );
2449    }
2450    tracing::info!("dispatch subscriber started");
2451
2452    // Spawn review subscribers — react to NeedsReview events immediately
2453    for engine in &project_engines {
2454        subscribers::review::spawn(
2455            event_bus.subscribe(),
2456            Arc::clone(&engine.backend),
2457            Arc::clone(&tmux),
2458            Arc::clone(&review_semaphore),
2459            Arc::clone(&engine.task_manager),
2460            Arc::clone(&router),
2461            Arc::clone(&dispatching),
2462            Arc::clone(&engine.store),
2463            engine.repo.clone(),
2464        );
2465    }
2466    tracing::info!("review subscriber started");
2467
2468    // Spawn unblock subscribers — react to Done events, unblock parent tasks immediately
2469    for engine in &project_engines {
2470        subscribers::unblock::spawn(
2471            event_bus.subscribe(),
2472            Arc::clone(&engine.backend),
2473            Arc::clone(&engine.task_manager),
2474            Arc::clone(&engine.store),
2475            engine.repo.clone(),
2476        );
2477    }
2478    tracing::info!("unblock subscriber started");
2479
2480    // Spawn notify subscriber — reacts to ALL events, pushes to transport.
2481    // Spawned once (not per-project) since the transport handles all repos.
2482    subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2483    tracing::info!("notify subscriber started");
2484
2485    // Main loop
2486    tracing::info!(
2487        tick = ?config.tick_interval,
2488        sync = ?config.sync_interval,
2489        parallel = config.max_parallel,
2490        "entering main loop"
2491    );
2492    // Values of the settings that only take effect at startup, so a reload can
2493    // tell the operator their edit needs a restart.
2494    let mut restart_only = restart_only_snapshot();
2495
2496    let mut interval = time::interval(config.tick_interval);
2497
2498    // Signal handlers (launchd/systemd send SIGTERM to stop services, SIGHUP for reload/restart)
2499    let mut sigterm = unix::signal(SignalKind::terminate())?;
2500    let mut sighup = unix::signal(SignalKind::hangup())?;
2501
2502    // Guard to prevent concurrent sync_tick runs (sync is spawned as a background task).
2503    let sync_in_progress = Arc::new(AtomicBool::new(false));
2504
2505    // Tick watchdog: detects when the main loop hasn't completed a tick in > 60s.
2506    // Updates an atomic timestamp at the end of each tick iteration.
2507    let last_tick_epoch = Arc::new(AtomicU64::new(
2508        SystemTime::now()
2509            .duration_since(std_time::UNIX_EPOCH)
2510            .unwrap_or_default()
2511            .as_secs(),
2512    ));
2513    {
2514        let last_tick_epoch = Arc::clone(&last_tick_epoch);
2515        let tick_interval_secs = config.tick_interval.as_secs();
2516        tokio::spawn(async move {
2517            // Check every 30s whether the main loop is still ticking.
2518            let mut watchdog_interval = time::interval(Duration::from_secs(30));
2519            // Own suspend baseline, independent of the main loop's checkpoint(). The
2520            // watchdog's interval timer also freezes during a host suspend, so this
2521            // detects the gap from its own wake-up instead of racing the main loop's
2522            // checkpoint() for the same GAP_LOG entry (see #3544).
2523            let mut suspend_detector = SuspendDetector::new();
2524            loop {
2525                watchdog_interval.tick().await;
2526                let own_gap = suspend_detector.check();
2527                let last = last_tick_epoch.load(Ordering::Relaxed);
2528                let now = SystemTime::now()
2529                    .duration_since(std_time::UNIX_EPOCH)
2530                    .unwrap_or_default()
2531                    .as_secs();
2532                let stale_secs = now.saturating_sub(last);
2533                // Warn if the tick loop hasn't completed in > 6× the tick interval (at least 60s).
2534                let threshold = (tick_interval_secs * 6).max(60);
2535                if stale_secs > threshold {
2536                    // A host suspend/resume gap fully explains a stale window that
2537                    // isn't an actual stall — the process (and its timers) were
2538                    // simply not scheduled while the host was asleep. Prefer this
2539                    // watchdog's own detector (immune to the checkpoint race above);
2540                    // fall back to the shared log in case the main loop's checkpoint()
2541                    // already ran and recorded a larger gap.
2542                    let shared_gap =
2543                        engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2544                    let gap = match (own_gap, shared_gap) {
2545                        (Some(a), Some(b)) => Some(a.max(b)),
2546                        (Some(a), None) => Some(a),
2547                        (None, Some(b)) => Some(b),
2548                        (None, None) => None,
2549                    };
2550                    if let Some(gap) = gap {
2551                        let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2552                        if gap_secs >= stale_secs {
2553                            tracing::info!(
2554                                stale_secs,
2555                                threshold,
2556                                suspend_gap_secs = gap_secs,
2557                                "tick loop stale window explained by host suspend/resume, not a stall"
2558                            );
2559                        } else {
2560                            tracing::error!(
2561                                stale_secs,
2562                                threshold,
2563                                suspend_gap_secs = gap_secs,
2564                                "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2565                                stale_secs,
2566                                threshold,
2567                                gap_secs,
2568                            );
2569                        }
2570                    } else {
2571                        tracing::error!(
2572                            stale_secs,
2573                            threshold,
2574                            "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2575                            stale_secs,
2576                            threshold,
2577                        );
2578                    }
2579                }
2580            }
2581        });
2582    }
2583
2584    loop {
2585        tokio::select! {
2586            _ = interval.tick() => {
2587                let tick_start = Instant::now();
2588
2589                // Detect a host suspend/resume gap once per iteration, in one place.
2590                // The watchdog and stuck-task recovery both read from this instead of
2591                // re-deriving suspend information from their own wall-clock deltas.
2592                if let Some(gap) = engine_suspend::checkpoint() {
2593                    tracing::info!(
2594                        gap_secs = gap.num_seconds(),
2595                        "host suspend/resume gap detected, discounting from stuck-task age checks"
2596                    );
2597                }
2598
2599                // Finish a pending shrink of the dispatch pool as running tasks
2600                // release their permits. Only logs when a permit was actually
2601                // taken back, so a long-running task does not log every tick.
2602                if review_in_force != config.max_parallel {
2603                    review_in_force =
2604                        resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2605                }
2606                if parallel_in_force != config.max_parallel {
2607                    let resized =
2608                        resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2609                    if resized != parallel_in_force {
2610                        parallel_in_force = resized;
2611                        tracing::info!(
2612                            in_force = parallel_in_force,
2613                            target = config.max_parallel,
2614                            "dispatch pool resized"
2615                        );
2616                    }
2617                }
2618
2619                // Tick weight recovery for rate-limited agents
2620                {
2621                    let mut rw = router.write().await;
2622                    rw.tick_weight_recovery();
2623                }
2624
2625                // Report the GitHub rate limit once per window, then tick anyway. Calls
2626                // into GitHub now fail fast instead of sleeping until the reset, so the
2627                // local phases keep moving and the watchdog does not see a stalled loop.
2628                if let Some(remaining) = GhHttp::is_rate_limited() {
2629                    let reset_at = chrono::Utc::now()
2630                        .timestamp()
2631                        .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2632                    // Tolerate jitter in the header so one window reports once
2633                    if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2634                        rate_limit_reported = Some(reset_at);
2635                        report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2636                    }
2637                } else if let Some(prev) = rate_limit_reported.take() {
2638                    tracing::info!(
2639                        limited_until = prev,
2640                        "GitHub API rate limit cleared"
2641                    );
2642                    if let Some(engine) = project_engines.first() {
2643                        let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2644                    }
2645                }
2646
2647                // Core tick: poll tasks for all projects
2648                tick::tick_all(
2649                    &project_engines,
2650                    &tmux,
2651                    &capture_for_tick,
2652                    &semaphore,
2653                    &config,
2654                    &router,
2655                    &weight_tx,
2656                    &dispatching,
2657                    Some(&transport),
2658                    &active_repos,
2659                    &auto_merge_in_flight,
2660                ).await;
2661
2662                // Drain pending weight signals after tick processing so the router
2663                // learns from outcomes produced by this iteration's task dispatches.
2664                while let Ok(signal) = weight_rx.try_recv() {
2665                    let mut rw = router.write().await;
2666                    match signal {
2667                        WeightSignal::RateLimited { ref agent } => {
2668                            rw.record_rate_limit(agent);
2669                        }
2670                        WeightSignal::Success { ref agent } => {
2671                            rw.record_success(agent);
2672                        }
2673                        WeightSignal::Failed { .. }
2674                        | WeightSignal::Rerouted
2675                        | WeightSignal::Blocked
2676                        | WeightSignal::None => {}
2677                    }
2678                }
2679
2680                // Periodic sync (less frequent) — spawned as a background task so it
2681                // cannot block the main tick loop. A stalled sync (hung HTTP call, slow
2682                // DB query, lock contention) previously parked all tokio workers for 12+
2683                // minutes because it ran inline. See issue #2574.
2684                if last_sync.elapsed() >= config.sync_interval {
2685                    if sync_in_progress.compare_exchange(
2686                        false,
2687                        true,
2688                        Ordering::AcqRel,
2689                        Ordering::Relaxed,
2690                    ).is_ok() {
2691                        spawn_sync_pass(
2692                            &project_engines,
2693                            &tmux,
2694                            &config,
2695                            &router,
2696                            &dispatching,
2697                            &auto_merge_in_flight,
2698                            &sync_in_progress,
2699                            "periodic",
2700                        );
2701                    } else {
2702                        tracing::debug!("sync tick still in progress, skipping");
2703                    }
2704                    // Records sync *schedule* time (not completion). This prevents
2705                    // re-entering the sync branch every tick while one is in flight.
2706                    // The sync_in_progress guard handles actual concurrency control.
2707                    last_sync = Instant::now();
2708                }
2709
2710                // Periodic webhook health check
2711                if webhook_enabled {
2712                    let health_check_interval = config.webhook_health_check_interval
2713                        .unwrap_or(Duration::from_secs(60));
2714
2715                    if last_webhook_health_check.elapsed() >= health_check_interval {
2716                        if let Some(port) = webhook_port {
2717                            let (health, failure_reason) =
2718                                github::check_webhook_health(port).await;
2719                            if health != webhook_healthy {
2720                                webhook_healthy = health;
2721                                if webhook_healthy {
2722                                    tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2723                                } else {
2724                                    tracing::warn!(
2725                                        port,
2726                                        orch_webhook_up = false,
2727                                        reason = failure_reason.as_deref().unwrap_or("unknown"),
2728                                        "webhook health check failed"
2729                                    );
2730                                }
2731                            }
2732                            // Persist updated status.
2733                            {
2734                                let to_save = {
2735                                    let mut s = webhook_status.lock().await;
2736                                    s.healthy = health;
2737                                    s.last_check_utc = Some(chrono::Utc::now());
2738                                    if health {
2739                                        s.last_failure_reason = None;
2740                                    } else if failure_reason.is_some() {
2741                                        s.last_failure_reason = failure_reason;
2742                                    }
2743                                    s.clone()
2744                                };
2745                                to_save.save().await;
2746                            }
2747                        }
2748                        last_webhook_health_check = Instant::now();
2749                    }
2750                }
2751
2752                // Periodic upgrade check — notify channels when a newer release is available.
2753                if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2754                    if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2755                        let channels = Arc::clone(&channel_registry);
2756                        let current = env!("ORCH_VERSION").to_string();
2757                        let check_interval =
2758                            Duration::from_secs(config.upgrade_check_interval);
2759                        tokio::spawn(async move {
2760                            check_and_notify_upgrade(&store, &channels, &current, check_interval)
2761                                .await;
2762                        });
2763                    }
2764                    last_upgrade_check = Some(Instant::now());
2765                }
2766
2767                // Update watchdog timestamp + log tick duration.
2768                let tick_elapsed = tick_start.elapsed();
2769                if tick_elapsed.as_secs() > 30 {
2770                    tracing::warn!(
2771                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2772                        "slow tick"
2773                    );
2774                } else {
2775                    tracing::debug!(
2776                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2777                        "tick complete"
2778                    );
2779                }
2780                last_tick_epoch.store(
2781                    SystemTime::now()
2782                        .duration_since(std_time::UNIX_EPOCH)
2783                        .unwrap_or_default()
2784                        .as_secs(),
2785                    Ordering::Relaxed,
2786                );
2787            }
2788            // Webhook events trigger an immediate tick (bypass polling interval)
2789            () = webhook_notify.notified() => {
2790                if let Some(remaining) = GhHttp::is_rate_limited() {
2791                    tracing::warn!(
2792                        remaining_secs = remaining.as_secs(),
2793                        "GitHub API rate-limited, skipping webhook-triggered tick"
2794                    );
2795                } else {
2796                    tracing::info!("webhook event triggered immediate tick");
2797
2798                    tick::tick_all(
2799                        &project_engines,
2800                        &tmux,
2801                        &capture_for_tick,
2802                        &semaphore,
2803                        &config,
2804                        &router,
2805                        &weight_tx,
2806                        &dispatching,
2807                        Some(&transport),
2808                        &active_repos,
2809                        &auto_merge_in_flight,
2810                    ).await;
2811
2812                // Drain pending weight signals after webhook-triggered tick processing.
2813                while let Ok(signal) = weight_rx.try_recv() {
2814                    let mut rw = router.write().await;
2815                    match signal {
2816                        WeightSignal::RateLimited { ref agent } => {
2817                            rw.record_rate_limit(agent);
2818                        }
2819                        WeightSignal::Success { ref agent } => {
2820                            rw.record_success(agent);
2821                        }
2822                        WeightSignal::Failed { .. }
2823                        | WeightSignal::Rerouted
2824                        | WeightSignal::Blocked
2825                        | WeightSignal::None => {}
2826                    }
2827                }
2828
2829                // Also attempt to run an immediate sync (ingest/cleanup) in the
2830                // background so webhook-driven events (e.g. new issues) are
2831                // picked up without waiting for the periodic sync interval.
2832                // Reuse the same sync_in_progress guard used by the periodic
2833                // branch to avoid concurrent syncs.
2834                if sync_in_progress.compare_exchange(
2835                    false,
2836                    true,
2837                    Ordering::AcqRel,
2838                    Ordering::Relaxed,
2839                ).is_ok() {
2840                    spawn_sync_pass(
2841                        &project_engines,
2842                        &tmux,
2843                        &config,
2844                        &router,
2845                        &dispatching,
2846                        &auto_merge_in_flight,
2847                        &sync_in_progress,
2848                        "webhook",
2849                    );
2850                    // Record schedule time so periodic sync doesn't immediately re-run.
2851                    last_sync = Instant::now();
2852                } else {
2853                    tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2854                }
2855                }
2856
2857                // Also reset the interval so we don't get a redundant tick right after
2858                interval.reset();
2859            }
2860            result = config_rx.recv() => {
2861                // A notification only arrives for a file that parsed, so there is no
2862                // broken-config case to guard against here (see `config::reload_file`).
2863                let reload = match result {
2864                    Ok(path) => {
2865                        tracing::info!(path = %path.display(), "config file changed, reloading");
2866                        true
2867                    }
2868                    Err(RecvError::Lagged(n)) => {
2869                        tracing::warn!(missed = n, "config change receiver lagged, reloading");
2870                        true
2871                    }
2872                    Err(RecvError::Closed) => {
2873                        tracing::warn!("config change channel closed");
2874                        false
2875                    }
2876                };
2877
2878                if reload {
2879                    // Reload engine config
2880                    let new_config = EngineConfig::from_config();
2881                    log_engine_config_changes(&config, &new_config);
2882
2883                    let new_restart_only = restart_only_snapshot();
2884                    log_restart_only_changes(&restart_only, &new_restart_only);
2885                    restart_only = new_restart_only;
2886
2887                    let tick_changed = new_config.tick_interval != config.tick_interval;
2888                    config = new_config;
2889
2890                    // Reset tick interval if it changed
2891                    if tick_changed {
2892                        interval = time::interval(config.tick_interval);
2893                        tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2894                    }
2895
2896                    // Resize the dispatch pool. A shrink only takes the permits that are
2897                    // free right now; the rest are taken as running tasks finish.
2898                    if review_in_force != config.max_parallel {
2899                        review_in_force = resize_dispatch_pool(
2900                            &review_semaphore,
2901                            review_in_force,
2902                            config.max_parallel,
2903                        );
2904                    }
2905                    if parallel_in_force != config.max_parallel {
2906                        parallel_in_force = resize_dispatch_pool(
2907                            &semaphore,
2908                            parallel_in_force,
2909                            config.max_parallel,
2910                        );
2911                        tracing::info!(
2912                            in_force = parallel_in_force,
2913                            target = config.max_parallel,
2914                            "dispatch pool resized"
2915                        );
2916                    }
2917
2918                    // Reload router config (async-safe: use reload_async to avoid blocking the runtime)
2919                    {
2920                        let mut router_guard = router.write().await;
2921                        router_guard.reload_async().await;
2922                    }
2923
2924                    // Pick up projects added to `projects:`
2925                    sync_project_engines(
2926                        &mut project_engines,
2927                        &mut active_repos,
2928                        &event_bus,
2929                        &tmux,
2930                        &capture_for_tick,
2931                        &semaphore,
2932                        &review_semaphore,
2933                        &weight_tx,
2934                        &router,
2935                        &dispatching,
2936                    ).await;
2937
2938                    tracing::info!(
2939                        tick = ?config.tick_interval,
2940                        sync = ?config.sync_interval,
2941                        parallel = config.max_parallel,
2942                        projects = project_engines.len(),
2943                        "config reloaded"
2944                    );
2945                }
2946            }
2947            signal_name = async {
2948                tokio::select! {
2949                    _ = signal::ctrl_c() => "SIGINT",
2950                    _ = sigterm.recv() => "SIGTERM",
2951                    _ = sighup.recv() => "SIGHUP",
2952                }
2953            } => {
2954                tracing::info!(signal = signal_name, "beginning graceful shutdown");
2955
2956                // Wrap the entire shutdown sequence in a timeout to enforce the configured limit.
2957                // This prevents indefinite blocking if status updates or tmux operations hang.
2958                let shutdown_result = time::timeout(
2959                    config.graceful_shutdown_timeout,
2960                    async {
2961                        // Reset in_progress tasks to routed so they re-dispatch after restart.
2962                        // Also reset in_review tasks to needs_review — their review agent
2963                        // tmux sessions will be killed when the process exits.
2964                        let mut reset_count = 0u32;
2965                        let mut review_reset_count = 0u32;
2966                        for engine in &project_engines {
2967                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
2968                                for task in &tasks {
2969                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
2970                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
2971                                    } else {
2972                                        if let Ok(Some(store_id)) =
2973                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
2974                                            && let Err(e) = engine
2975                                                .store
2976                                                .finalize_incomplete_runs(
2977                                                    store_id,
2978                                                    "aborted",
2979                                                    "graceful shutdown: reset in_progress task to routed",
2980                                                )
2981                                                .await
2982                                            {
2983                                                tracing::warn!(
2984                                                    task_id = task.id.0,
2985                                                    ?e,
2986                                                    "failed to finalize incomplete runs on shutdown"
2987                                                );
2988                                            }
2989                                        reset_count += 1;
2990                                    }
2991                                }
2992                            }
2993                            // Also reset internal in_progress tasks
2994                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
2995                                for task in &tasks {
2996                                    let task_id = format!("internal:{}", task.id);
2997                                    if let Err(e) = engine.task_manager.update_task_status(
2998                                        &ExternalId(task_id.clone()),
2999                                        Status::Routed,
3000                                    ).await {
3001                                        tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3002                                    } else {
3003                                        if let Err(e) = engine
3004                                            .store
3005                                            .finalize_incomplete_runs(
3006                                                task.id,
3007                                                "aborted",
3008                                                "graceful shutdown: reset in_progress task to routed",
3009                                            )
3010                                            .await
3011                                        {
3012                                            tracing::warn!(
3013                                                task_id,
3014                                                ?e,
3015                                                "failed to finalize incomplete internal runs on shutdown"
3016                                            );
3017                                        }
3018                                        reset_count += 1;
3019                                    }
3020                                }
3021                            }
3022                            // Reset in_review tasks — review agent sessions die on shutdown
3023                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3024                                for task in &tasks {
3025                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3026                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3027                                    } else {
3028                                        if let Ok(Some(store_id)) =
3029                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3030                                            && let Err(e) = engine
3031                                                .store
3032                                                .finalize_incomplete_runs(
3033                                                    store_id,
3034                                                    "aborted",
3035                                                    "graceful shutdown: reset in_review task to needs_review",
3036                                                )
3037                                                .await
3038                                            {
3039                                                tracing::warn!(
3040                                                    task_id = task.id.0,
3041                                                    ?e,
3042                                                    "failed to finalize incomplete review runs on shutdown"
3043                                                );
3044                                            }
3045                                        review_reset_count += 1;
3046                                    }
3047                                }
3048                            }
3049                            // Also reset internal in_review tasks
3050                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3051                                for task in &tasks {
3052                                    let task_id = format!("internal:{}", task.id);
3053                                    if let Err(e) = engine.task_manager.update_task_status(
3054                                        &ExternalId(task_id.clone()),
3055                                        Status::NeedsReview,
3056                                    ).await {
3057                                        tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3058                                    } else {
3059                                        if let Err(e) = engine
3060                                            .store
3061                                            .finalize_incomplete_runs(
3062                                                task.id,
3063                                                "aborted",
3064                                                "graceful shutdown: reset in_review task to needs_review",
3065                                            )
3066                                            .await
3067                                        {
3068                                            tracing::warn!(
3069                                                task_id,
3070                                                ?e,
3071                                                "failed to finalize incomplete internal review runs on shutdown"
3072                                            );
3073                                        }
3074                                        review_reset_count += 1;
3075                                    }
3076                                }
3077                            }
3078                        }
3079                        if reset_count > 0 {
3080                            tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3081                        }
3082                        if review_reset_count > 0 {
3083                            tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3084                        }
3085
3086                        // Kill all orch-managed tmux sessions so stale sessions don't
3087                        // block dispatch after restart (session_exists check).
3088                        let mut killed_sessions = Vec::new();
3089                        if let Ok(sessions) = tmux.list_sessions().await {
3090                            for session in &sessions {
3091                                if session.name.starts_with("orch-") {
3092                                    match tmux.kill_session(&session.name).await { Err(e) => {
3093                                        tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3094                                    } _ => {
3095                                        killed_sessions.push(session.name.clone());
3096                                    }}
3097                                }
3098                            }
3099                            if !killed_sessions.is_empty() {
3100                                tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3101                            }
3102                        }
3103
3104                        // Wait for sessions to actually die before exiting.
3105                        // Prevents race where process exits before tmux finishes cleanup.
3106                        if !killed_sessions.is_empty() {
3107                            let still_alive = tmux
3108                                .wait_for_sessions_dead(
3109                                    &killed_sessions,
3110                                    Duration::from_millis(100),
3111                                    Duration::from_secs(5),
3112                                )
3113                                .await;
3114                            if still_alive > 0 {
3115                                tracing::warn!(
3116                                    still_alive,
3117                                    "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3118                                );
3119                            }
3120                            tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3121                        }
3122                    },
3123                ).await;
3124
3125                if shutdown_result.is_err() {
3126                    tracing::error!(
3127                        timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3128                        "graceful shutdown timed out, forcing exit"
3129                    );
3130                }
3131
3132                break;
3133            }
3134        }
3135    }
3136
3137    // Clean up event bus port file and service version file
3138    events::cleanup_port_file().await;
3139    local::cleanup().await;
3140    events::cleanup_version_file();
3141
3142    // transport and channels drop here at end of scope
3143    let _ = transport;
3144    tracing::info!("orch engine stopped");
3145    Ok(())
3146}
3147
3148#[cfg(test)]
3149mod tests {
3150    use std::sync::Arc;
3151
3152    use super::*;
3153    use crate::backends::test_helpers::NoopBackend;
3154    use crate::channels::capture::CaptureService;
3155    use crate::channels::transport::Transport;
3156    use crate::tmux::SessionGuard;
3157    use crate::tmux::TmuxManager;
3158
3159    async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3160        let id = store
3161            .upsert_external(&crate::store::UpsertExternal {
3162                repo: "owner/repo",
3163                ext_id: number,
3164                title: "Under review",
3165                body: "",
3166                author: "",
3167                url: "",
3168                labels: &[],
3169                origin: "github",
3170            })
3171            .await
3172            .unwrap();
3173        store.update_status(id, TaskStatus::InReview).await.unwrap();
3174        id
3175    }
3176
3177    #[tokio::test]
3178    async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3179        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3180        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3181        let task_manager =
3182            TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3183        let tmux = TmuxManager::isolated();
3184        let orphaned = in_review_task(&store, "11").await;
3185        let live = in_review_task(&store, "12").await;
3186        let session = tmux
3187            .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3188            .await
3189            .unwrap();
3190        let _guard = SessionGuard(session);
3191
3192        let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3193
3194        assert_eq!(reset, 1);
3195        assert_eq!(
3196            store.get(orphaned).await.unwrap().status,
3197            TaskStatus::NeedsReview
3198        );
3199        assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3200    }
3201
3202    #[test]
3203    fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3204        let semaphore = Arc::new(Semaphore::new(4));
3205        let held: Vec<_> = (0..4)
3206            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3207            .collect();
3208        assert!(
3209            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3210            "pool is full"
3211        );
3212
3213        let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3214        assert_eq!(in_force, 8);
3215
3216        let grown: Vec<_> = (0..4)
3217            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3218            .collect();
3219        assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3220        assert!(
3221            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3222            "pool is full"
3223        );
3224    }
3225
3226    #[test]
3227    fn shrinking_max_parallel_never_cancels_a_running_task() {
3228        let semaphore = Arc::new(Semaphore::new(8));
3229        let mut held: Vec<_> = (0..8)
3230            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3231            .collect();
3232
3233        // Nothing is free, so all 8 tasks keep their permits and keep running.
3234        let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3235        assert_eq!(in_force, 8);
3236        assert_eq!(held.len(), 8);
3237
3238        // Three finish. Their permits are taken back instead of handed out again.
3239        held.truncate(5);
3240        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3241        assert_eq!(in_force, 5);
3242        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3243
3244        // One more finishes and the pool settles at the new limit.
3245        held.truncate(4);
3246        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3247        assert_eq!(in_force, 4);
3248        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3249
3250        // A slot freed after that is handed out, since 4 is now the real limit.
3251        held.truncate(3);
3252        assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3253    }
3254
3255    #[test]
3256    fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3257        let configured = vec![
3258            ("owner/one".to_string(), PathBuf::from("/srv/one")),
3259            ("owner/two".to_string(), PathBuf::from("/srv/two")),
3260        ];
3261        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3262
3263        let (added, removed) = diff_configured_projects(&configured, &active);
3264        assert_eq!(
3265            added,
3266            vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3267            "only the new project is started, so owner/one keeps its single set of subscribers"
3268        );
3269        assert!(removed.is_empty(), "{removed:?}");
3270    }
3271
3272    #[test]
3273    fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3274        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3275        let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3276            .into_iter()
3277            .collect();
3278
3279        let (added, removed) = diff_configured_projects(&configured, &active);
3280        assert!(added.is_empty(), "nothing new to start");
3281        assert_eq!(removed, vec!["owner/gone".to_string()]);
3282    }
3283
3284    #[test]
3285    fn an_unchanged_project_list_starts_nothing() {
3286        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3287        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3288
3289        assert_eq!(
3290            diff_configured_projects(&configured, &active),
3291            (vec![], vec![])
3292        );
3293    }
3294
3295    #[test]
3296    fn restart_only_keys_are_the_settings_read_once_at_startup() {
3297        // Guards the docs table: every key listed here is documented as needing
3298        // `orch service restart`, and nothing reloadable may be added to it.
3299        assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3300    }
3301
3302    #[test]
3303    fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3304        let before = vec![Some("bot_token: a\n".to_string()), None];
3305
3306        assert!(
3307            log_restart_only_changes(&before, &before.clone()).is_empty(),
3308            "a reload that did not touch these keys must stay quiet"
3309        );
3310
3311        // An edited token and a newly added webhook block are both changes
3312        let after = vec![
3313            Some("bot_token: b\n".to_string()),
3314            Some("port: 8080\n".to_string()),
3315        ];
3316        assert_eq!(
3317            log_restart_only_changes(&before, &after),
3318            ["channels", "webhook"]
3319        );
3320
3321        // Only the edited key is reported when the other is untouched
3322        let token_only = vec![Some("bot_token: b\n".to_string()), None];
3323        assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3324    }
3325
3326    #[test]
3327    fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3328        let semaphore = Arc::new(Semaphore::new(4));
3329        assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3330        assert_eq!(semaphore.available_permits(), 4);
3331    }
3332
3333    #[test]
3334    fn live_session_stuck_limit_follows_the_run_timeout() {
3335        let config = EngineConfig {
3336            stuck_timeout: 1800,
3337            run_timeouts: RunTimeouts {
3338                base: 1800,
3339                medium: 2700,
3340                complex: 3600,
3341            },
3342            ..EngineConfig::default()
3343        };
3344        assert_eq!(
3345            config.live_session_stuck_limit(Some("simple")),
3346            1800 + RUN_TIMEOUT_GRACE_SECS
3347        );
3348        assert_eq!(
3349            config.live_session_stuck_limit(Some("medium")),
3350            2700 + RUN_TIMEOUT_GRACE_SECS
3351        );
3352        assert_eq!(
3353            config.live_session_stuck_limit(Some("complex")),
3354            3600 + RUN_TIMEOUT_GRACE_SECS
3355        );
3356
3357        let long_stuck = EngineConfig {
3358            stuck_timeout: 7200,
3359            ..config
3360        };
3361        assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3362    }
3363
3364    #[test]
3365    fn engine_config_defaults() {
3366        let config = EngineConfig::default();
3367        assert_eq!(config.tick_interval, Duration::from_secs(10));
3368        assert_eq!(config.sync_interval, Duration::from_secs(45));
3369        assert_eq!(config.max_parallel, 4);
3370        assert_eq!(config.stuck_timeout, 1800);
3371        assert_eq!(config.no_session_stuck_timeout, 600);
3372        assert!(
3373            config.no_session_stuck_timeout < config.stuck_timeout,
3374            "no_session_stuck_timeout must be shorter than stuck_timeout"
3375        );
3376        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3377        assert_eq!(config.upgrade_check_interval, 3600);
3378    }
3379
3380    #[test]
3381    fn engine_config_from_config_uses_defaults_when_no_config() {
3382        let config = EngineConfig::from_config();
3383        assert_eq!(config.tick_interval, Duration::from_secs(10));
3384        assert_eq!(config.sync_interval, Duration::from_secs(45));
3385        // max_parallel may be overridden by user config; just check it's reasonable
3386        assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3387        assert_eq!(config.stuck_timeout, 1800);
3388        assert_eq!(config.no_session_stuck_timeout, 600);
3389        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3390        assert_eq!(config.upgrade_check_interval, 3600);
3391    }
3392
3393    #[test]
3394    fn engine_config_no_session_timeout_is_under_15_minutes() {
3395        let config = EngineConfig::default();
3396        assert!(
3397            config.no_session_stuck_timeout <= 900,
3398            "no_session_stuck_timeout ({}) exceeds 15 minutes",
3399            config.no_session_stuck_timeout
3400        );
3401    }
3402
3403    #[test]
3404    fn upgrade_check_runs_immediately_when_never_checked() {
3405        assert!(should_run_upgrade_check(None, 3600));
3406    }
3407
3408    #[test]
3409    fn upgrade_check_waits_for_interval_after_first_run() {
3410        assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3411    }
3412
3413    #[test]
3414    fn upgrade_check_can_be_disabled() {
3415        assert!(!should_run_upgrade_check(None, 0));
3416    }
3417
3418    #[test]
3419    fn configured_agents_parses_yaml_block_array() {
3420        // config::get returns serde_norway::to_string for arrays, which is YAML block format
3421        let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3422        let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3423        assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3424    }
3425
3426    #[test]
3427    fn configured_agents_parses_json_inline_array() {
3428        let json_str = r#"["claude","codex","olm"]"#;
3429        let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3430        assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3431    }
3432
3433    #[test]
3434    fn configured_agents_fallback_returns_default_agents() {
3435        // When no config is present, configured_agents() returns DEFAULT_AGENTS
3436        let defaults = configured_agents();
3437        assert!(defaults.contains(&"claude".to_string()));
3438        assert!(defaults.contains(&"codex".to_string()));
3439        assert!(defaults.contains(&"opencode".to_string()));
3440    }
3441
3442    /// Integration test that requires a fully working tmux environment with
3443    /// reliable capture-pane timing. Runs locally but not in CI (GitHub Actions
3444    /// has tmux installed but capture-pane timing is unreliable).
3445    ///
3446    /// DO NOT REMOVE #[ignore] — this test was un-ignored in PR #608 which
3447    /// broke CI. The `tmux -V` skip check is insufficient because tmux CAN
3448    /// create sessions on CI runners, but capture-pane output is empty or
3449    /// delayed, causing the assertion to fail after a 10s timeout.
3450    #[tokio::test]
3451    #[ignore = "capture-pane timing is unreliable on CI runners"]
3452    async fn integration_channel_to_tmux_to_capture() {
3453        // Create transport and capture service
3454        let transport = Arc::new(Transport::new());
3455        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3456        let tmux = TmuxManager::new();
3457
3458        let task_id = "testtask1".to_string();
3459        let session_name = format!("orch-test-{task_id}");
3460
3461        // Start a detached tmux session
3462        let _ = tmux
3463            .command()
3464            .args(["new-session", "-d", "-s", &session_name])
3465            .output()
3466            .await
3467            .expect("failed to start tmux session");
3468
3469        // Guard ensures session cleanup even on panic
3470        let _guard = SessionGuard(session_name.clone());
3471
3472        // Register session with capture service and transport binding
3473        capture
3474            .register_session("owner/repo", &task_id, &session_name)
3475            .await;
3476        transport
3477            .bind(
3478                "owner/repo",
3479                &task_id,
3480                &session_name,
3481                "telegram",
3482                "12345",
3483                None,
3484            )
3485            .await;
3486
3487        // Spawn capture.run() which exits when session is unregistered
3488        let capture_clone = Arc::clone(&capture);
3489        let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3490
3491        // Subscribe to transport output for this task
3492        let mut rx = transport
3493            .subscribe("owner/repo", &task_id)
3494            .await
3495            .expect("no subscription");
3496
3497        // Send a message into the tmux session via tmux send-keys (simulate channel input)
3498        let send = tmux
3499            .command()
3500            .args([
3501                "send-keys",
3502                "-t",
3503                &session_name,
3504                "echo from-channel",
3505                "Enter",
3506            ])
3507            .output()
3508            .await
3509            .expect("failed to send keys");
3510        assert!(send.status.success());
3511
3512        // Wait for an output chunk from capture -> transport
3513        let mut got = false;
3514        for _ in 0..20 {
3515            if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3516                && c.content.contains("from-channel")
3517            {
3518                got = true;
3519                break;
3520            }
3521        }
3522
3523        // Clean up: unregister session (tmux kill handled by SessionGuard on drop)
3524        capture.unregister_session("owner/repo", &task_id).await;
3525
3526        // Wait for capture.run to finish
3527        let _ = capture_handle.await;
3528
3529        assert!(got, "did not observe tmux output via capture/transport");
3530    }
3531
3532    // ── read_project_channel_config tests ─────────────────────────────────
3533
3534    #[test]
3535    fn read_project_channel_config_returns_defaults_when_no_file() {
3536        let dir = tempfile::tempdir().expect("tempdir");
3537        let cfg = read_project_channel_config(dir.path());
3538        assert!(cfg.telegram_topic_id.is_none());
3539        assert!(cfg.discord_channel_id.is_none());
3540    }
3541
3542    #[test]
3543    fn read_project_channel_config_parses_telegram_and_discord() {
3544        let dir = tempfile::tempdir().expect("tempdir");
3545        let orch_yml = dir.path().join(".orch.yml");
3546        fs::write(
3547            &orch_yml,
3548            "channels:\n  telegram:\n    topic_id: \"42\"\n  discord:\n    channel_id: \"9999\"\n",
3549        )
3550        .expect("write .orch.yml");
3551        let cfg = read_project_channel_config(dir.path());
3552        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3553        assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3554    }
3555
3556    #[test]
3557    fn read_project_channel_config_handles_partial_channels() {
3558        let dir = tempfile::tempdir().expect("tempdir");
3559        let orch_yml = dir.path().join(".orch.yml");
3560        fs::write(&orch_yml, "channels:\n  telegram:\n    topic_id: \"7\"\n")
3561            .expect("write .orch.yml");
3562        let cfg = read_project_channel_config(dir.path());
3563        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3564        assert!(cfg.discord_channel_id.is_none());
3565    }
3566
3567    #[test]
3568    fn read_project_channel_config_handles_malformed_yaml() {
3569        let dir = tempfile::tempdir().expect("tempdir");
3570        let orch_yml = dir.path().join(".orch.yml");
3571        fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3572        let cfg = read_project_channel_config(dir.path());
3573        // Should return defaults (not panic) and emit a warn! log with the parse error.
3574        assert!(cfg.telegram_topic_id.is_none());
3575        assert!(cfg.discord_channel_id.is_none());
3576    }
3577
3578    #[test]
3579    #[cfg(unix)]
3580    fn read_project_channel_config_handles_unreadable_file() {
3581        let dir = tempfile::tempdir().expect("tempdir");
3582        // A directory fails to read for every user, root included, where chmod 000 does not
3583        fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3584        let cfg = read_project_channel_config(dir.path());
3585        // Should return defaults (not panic) and emit a warn! log with the read error.
3586        assert!(cfg.telegram_topic_id.is_none());
3587        assert!(cfg.discord_channel_id.is_none());
3588    }
3589}