Skip to main content

orch/engine/
mod.rs

1//! Engine — the core orchestration loop.
2//!
3//! The engine is the central coordinator. It owns the tick loop, backend
4//! connections, channel registry, transport layer, and tmux session manager.
5//!
6//! Submodules handle the heavy lifting:
7//! - [`tick`] — core tick phases (session polling, stuck recovery, routing, dispatch)
8//! - [`sync`] — periodic sync operations (cleanup, PR review, mentions)
9//! - [`review`] — PR review pipeline (review agent, auto-merge, change handling)
10//! - [`cleanup`] — worktree cleanup and merged-PR detection
11//!
12//! This file contains struct definitions, project initialization, and the
13//! main event loop (`serve()`).
14
15pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40/// Standard Orch attribution footer for issue bodies, PR bodies, and comments.
41///
42/// Append to every user-visible string posted to GitHub so activity is
43/// clearly attributed to orch rather than appearing as a human post.
44use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{
58    local_branch_exists, remove_worktree_and_branch, remove_worktree_and_branch_for,
59};
60use crate::engine::router::{Router, RouterConfig};
61use crate::engine::runner::worktree::{
62    abort_worktree_rebase, list_project_worktrees, task_id_from_worktree_name,
63    validate_worktree_gitdir,
64};
65use crate::engine::tasks::TaskManager;
66use crate::github::http::{GhHttp, rate_limit_metrics};
67use crate::repo_context::REPO_CONTEXT;
68use crate::store::{TaskStatus, TaskStore};
69use crate::tmux::TmuxManager;
70use runner::WeightSignal;
71// AtomicBool/Ordering removed — shutdown is now immediate (reset tasks + break)
72use crate::backends::Status;
73use crate::backends::github::GitHubBackend;
74use crate::channels::github;
75use crate::channels::github::is_transient_bind_error;
76use crate::channels::github::webhook_backoff_delay;
77use crate::channels::notification;
78use crate::channels::notification::TaskNotification;
79use crate::channels::transport;
80use crate::cluster;
81use crate::cluster::local;
82use crate::engine::cooldown as engine_cooldown;
83use crate::engine::review as engine_review;
84use crate::engine::runner::git_ops;
85use crate::engine::runner::worktree;
86use crate::engine::suspend as engine_suspend;
87use crate::engine::suspend::SuspendDetector;
88use crate::engine::tasks as engine_tasks;
89use crate::github::app;
90use crate::home;
91use crate::secret_ref;
92use crate::store;
93use crate::ui;
94use crate::webhook_status::WebhookStatus;
95use futures::future;
96use std::collections::HashMap;
97use std::collections::HashSet;
98use std::fmt;
99use std::fmt::Debug;
100use std::fmt::Formatter;
101use std::fs;
102use std::path::Path;
103use std::path::PathBuf;
104use std::sync::Arc;
105use std::sync::atomic::AtomicBool;
106use std::sync::atomic::AtomicU64;
107use std::sync::atomic::Ordering;
108use std::time as std_time;
109use std::time::Duration;
110use std::time::Instant;
111use std::time::SystemTime;
112use tokio::process::Command;
113use tokio::signal;
114use tokio::signal::unix;
115use tokio::signal::unix::SignalKind;
116use tokio::sync::Mutex;
117use tokio::sync::broadcast::Sender;
118use tokio::sync::broadcast::error::RecvError;
119use tokio::sync::mpsc::Receiver;
120use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
121use tokio::task;
122use tokio::time;
123
124pub fn orch_footer() -> &'static str {
125    "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
126}
127
128/// Build a standard attribution footer for GitHub comments posted by orch bots.
129///
130/// `verb` is "Created", "Reviewed", or "Commented" depending on context.
131pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
132    match model {
133        Some(m) => format!(
134            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
135        ),
136        None => format!(
137            "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
138        ),
139    }
140}
141
142/// Lightweight reference tuple for channel message handling.
143type EngineRef = (
144    String,
145    Arc<dyn ExternalBackend>,
146    Arc<TaskManager>,
147    Option<Arc<TaskStore>>,
148);
149
150/// Per-project engine state.
151///
152/// Each project has its own backend, task runner, and task manager,
153/// but they share the global tmux manager, transport, and semaphore.
154#[derive(Clone)]
155pub struct ProjectEngine {
156    pub repo: String,
157    pub project_dir: PathBuf,
158    pub backend: Arc<dyn ExternalBackend>,
159    pub task_manager: Arc<TaskManager>,
160    pub runner: Arc<runner::TaskRunner>,
161    pub store: Arc<TaskStore>,
162}
163
164impl Debug for ProjectEngine {
165    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
166        f.debug_struct("ProjectEngine")
167            .field("repo", &self.repo)
168            .field("project_dir", &self.project_dir)
169            .finish_non_exhaustive()
170    }
171}
172
173/// A restart ends every review session, so an `in_review` task with no session has no reviewer.
174/// Hand it back to `needs_review` at start, so it does not wait for the stuck-review recovery
175pub(crate) async fn reset_orphaned_reviews(
176    task_manager: &TaskManager,
177    store: &Arc<TaskStore>,
178    repo: &str,
179    tmux: &TmuxManager,
180) -> usize {
181    // The store holds external and internal tasks, the second list is the legacy internal table
182    let mut ids: Vec<ExternalId> = store
183        .list_by_status(repo, TaskStatus::InReview)
184        .await
185        .unwrap_or_default()
186        .iter()
187        .map(|t| engine_tasks::store_task_to_external(t).id)
188        .collect();
189    if let Ok(legacy) = task_manager
190        .list_internal_by_status(TaskStatus::InReview)
191        .await
192    {
193        ids.extend(legacy.into_iter().map(|t| t.id));
194    }
195    ids.sort_by(|a, b| a.0.cmp(&b.0));
196    ids.dedup_by(|a, b| a.0 == b.0);
197
198    let mut reset = 0;
199    for id in &ids {
200        let session = tmux.session_name(repo, &format!("{}-review", id.0));
201
202        if tmux.session_exists(&session).await {
203            continue;
204        }
205
206        if let Err(e) = task_manager
207            .update_task_status(id, Status::NeedsReview)
208            .await
209        {
210            tracing::warn!(
211                task_id = id.0,
212                ?e,
213                "failed to reset an orphaned review at start"
214            );
215            continue;
216        }
217
218        if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
219            let reason = "startup: review session gone, reset to needs_review";
220
221            if let Err(e) = store
222                .finalize_incomplete_runs(store_id, "aborted", reason)
223                .await
224            {
225                tracing::warn!(
226                    task_id = id.0,
227                    ?e,
228                    "failed to finalize the orphaned review's runs"
229                );
230            }
231        }
232
233        reset += 1;
234    }
235
236    reset
237}
238
239/// Starts the relay client when `relay.url` is set. The token is read in the background, so a
240/// passbox prompt nobody answers never holds up the engine
241fn start_relay_client(store: &Arc<TaskStore>, notify: &Arc<Notify>) {
242    // The global config only: a project file must never point a secret at its own endpoint
243    let url = config::get_global("relay.url").unwrap_or_default();
244    let store = Arc::clone(store);
245
246    if url.is_empty() {
247        return;
248    }
249
250    let token_ref = config::get_global("relay.token").unwrap_or_default();
251    let notify = Arc::clone(notify);
252    tokio::spawn(async move {
253        let mut warned = false;
254        let token = loop {
255            let reference = token_ref.clone();
256            let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
257                .await
258                .map_err(anyhow::Error::from)
259                .and_then(|r| r);
260
261            match read {
262                Ok(token) if !token.is_empty() => break token,
263                Ok(_) => {
264                    tracing::warn!("relay.token is empty, the relay client stays off");
265                    return;
266                }
267                Err(e) if !warned => {
268                    tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
269                    warned = true;
270                }
271                Err(_) => {}
272            }
273
274            time::sleep(Duration::from_secs(300)).await;
275        };
276
277        relay::run(url, token, store, notify).await;
278    });
279}
280
281/**
282 * Runs `sync_tick` for every project at once, then the tail that is the same for all of them,
283 * in a spawned task. Sync time is the slowest project instead of the sum of all of them
284 * (#3864). The caller must already hold `sync_in_progress`; this releases it when done
285 **/
286#[expect(
287    clippy::too_many_arguments,
288    reason = "the spawned task takes its own handle to each piece of engine state"
289)]
290fn spawn_sync_pass(
291    project_engines: &[ProjectEngine],
292    tmux: &Arc<TmuxManager>,
293    engine_config: &EngineConfig,
294    router: &Arc<RwLock<Router>>,
295    dispatching: &Arc<dashmap::DashMap<String, String>>,
296    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
297    sync_in_progress: &Arc<AtomicBool>,
298    trigger: &'static str,
299) {
300    let engines = project_engines.to_vec();
301    let tmux = Arc::clone(tmux);
302    let engine_config = engine_config.clone();
303    let router = Arc::clone(router);
304    let dispatching = Arc::clone(dispatching);
305    let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
306    let guard = Arc::clone(sync_in_progress);
307
308    tokio::spawn(async move {
309        let started = Instant::now();
310        let mut passes = Vec::with_capacity(engines.len());
311        for engine in &engines {
312            passes.push(REPO_CONTEXT.scope(
313                engine.repo.clone(),
314                sync_one_project(
315                    engine,
316                    &tmux,
317                    &engine_config,
318                    &router,
319                    &dispatching,
320                    &auto_merge_in_flight,
321                ),
322            ));
323        }
324        future::join_all(passes).await;
325
326        sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
327        tracing::info!(
328            trigger,
329            elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
330            "sync tick complete"
331        );
332        guard.store(false, Ordering::Release);
333    });
334}
335
336async fn sync_one_project(
337    engine: &ProjectEngine,
338    tmux: &Arc<TmuxManager>,
339    engine_config: &EngineConfig,
340    router: &Arc<RwLock<Router>>,
341    dispatching: &Arc<dashmap::DashMap<String, String>>,
342    auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
343) {
344    if let Err(e) = Box::pin(sync::sync_tick(
345        &engine.backend,
346        tmux,
347        &engine.repo,
348        engine_config,
349        router,
350        &engine.task_manager,
351        &engine.store,
352        dispatching,
353        auto_merge_in_flight,
354    ))
355    .await
356    {
357        tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
358    }
359}
360
361/// Milliseconds a group of tick or sync phases may take before it is worth an operator's eye
362const SLOW_PHASE_GROUP_MS: u128 = 1_000;
363
364/**
365 * Logs how long each phase of a tick or sync group took, for one project. Tick and sync time
366 * grows with the project count, and the totals alone did not say which project or phase spent
367 * it (#3864). Info once a group crosses `SLOW_PHASE_GROUP_MS`, so a quiet install stays quiet
368 **/
369pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
370    let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
371    if total_ms >= SLOW_PHASE_GROUP_MS {
372        tracing::info!(group, repo, total_ms, ?phases, "phase timings");
373    } else {
374        tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
375    }
376}
377
378/// Return the configured agent list from top-level `agents` in config.yml,
379/// falling back to [`router::config::DEFAULT_AGENTS`] when not set.
380///
381/// This is the canonical source of truth for "which agents does orch know
382/// about" — use it instead of `DEFAULT_AGENTS` directly so that
383/// Claude-compatible agents added via config (e.g. `olm`) are recognized
384/// without code changes.
385///
386/// The key is top-level (`agents:`) rather than nested (`engine.agents:`)
387/// because `config::get` serializes nested YAML arrays via `serde_norway::to_string`
388/// which produces YAML block format, not JSON — breaking downstream parsing.
389pub fn configured_agents() -> Vec<String> {
390    if let Ok(agents_str) = config::get("agents")
391        && !agents_str.is_empty()
392        && agents_str != "[]"
393    {
394        // config::get returns serde_norway::to_string for arrays, which is
395        // YAML block format ("- claude\n- codex\n..."). Parse as YAML first.
396        if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
397            && !agents_arr.is_empty()
398        {
399            return agents_arr;
400        }
401        // Fallback: try JSON (inline arrays) or comma-separated
402        if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
403            && !agents_arr.is_empty()
404        {
405            return agents_arr;
406        }
407    }
408    router::config::DEFAULT_AGENTS
409        .iter()
410        .map(ToString::to_string)
411        .collect()
412}
413
414/// Seconds a live session may run past its own timeout before stuck recovery takes it. The
415/// `timeout` wrapper ends the agent at the limit, and the runner still collects the result
416pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
417
418/// The limit the `timeout` wrapper puts on one agent run, per complexity
419#[derive(Clone, Copy, Debug, PartialEq, Eq)]
420pub struct RunTimeouts {
421    pub base: u64,
422    pub medium: u64,
423    pub complex: u64,
424}
425
426impl Default for RunTimeouts {
427    fn default() -> Self {
428        Self {
429            base: 1800,
430            medium: 1800,
431            complex: 1800,
432        }
433    }
434}
435
436impl RunTimeouts {
437    /// `workflow.timeout_seconds` (at least 1800), raised by
438    /// `workflow.timeout_by_complexity.{medium,complex}` but never below the base
439    pub fn from_config() -> Self {
440        let read =
441            |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
442        let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
443        Self {
444            base,
445            medium: read("workflow.timeout_by_complexity.medium")
446                .unwrap_or(base)
447                .max(base),
448            complex: read("workflow.timeout_by_complexity.complex")
449                .unwrap_or(base)
450                .max(base),
451        }
452    }
453
454    pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
455        match complexity {
456            Some("complex") => self.complex,
457            Some("medium") => self.medium,
458            _ => self.base,
459        }
460    }
461}
462
463/// Engine configuration.
464#[derive(Clone, Debug)]
465pub struct EngineConfig {
466    /// Main tick interval
467    pub tick_interval: Duration,
468    /// GitHub sync interval (cleanup, PR review, mentions)
469    pub sync_interval: Duration,
470    /// Webhook health check interval (seconds)
471    pub webhook_health_check_interval: Option<Duration>,
472    /// Maximum parallel task executions
473    pub max_parallel: usize,
474    /// Stuck task timeout for tasks with an active tmux session (seconds)
475    pub stuck_timeout: u64,
476    /// Stuck task timeout for tasks with no active tmux session (seconds).
477    /// Shorter than `stuck_timeout` because no session means the agent has already exited.
478    pub no_session_stuck_timeout: u64,
479    /// Stuck task timeout for in_review tasks with no active tmux session (seconds).
480    /// Longer than `no_session_stuck_timeout` because a review agent ends its tmux session on
481    /// normal completion before its result reaches the engine. A shorter limit resets a task
482    /// whose finished review is still being delivered, and the result is dropped as stale.
483    /// `from_config` keeps it above the review agent's own timeout plus
484    /// `review::REVIEW_RESULT_DELIVERY_SECS`.
485    pub in_review_no_session_stuck_timeout: u64,
486    /// Per-run agent timeouts. A live session is not stuck before its own timeout runs out
487    pub run_timeouts: RunTimeouts,
488    /// Auto-create follow-up tasks when PR reviews request changes
489    pub auto_create_followup_on_changes: bool,
490    /// Graceful shutdown timeout — how long to wait for running agents before exiting.
491    pub graceful_shutdown_timeout: Duration,
492    /// Grace period before silence detection kicks in (seconds).
493    /// Allows time for agent startup, model download, API handshake.
494    pub silence_grace_period: u64,
495    /// Cooldown duration for a model detected as silent (seconds).
496    pub silence_cooldown: u64,
497    /// How often to check for a newer orch release and notify channels (seconds).
498    /// Set to 0 to disable. Default: 3600 (1 hour).
499    pub upgrade_check_interval: u64,
500    /// Seconds of continuous startup failure (GitHub unreachable / auth errors)
501    /// before the engine escalates from WARN to ERROR-level logging and sends a
502    /// one-time push notification to configured channels. Set to 0 to disable.
503    /// Default: 3600 (1 hour).
504    pub startup_failure_escalation_secs: u64,
505}
506
507impl Default for EngineConfig {
508    fn default() -> Self {
509        Self {
510            tick_interval: Duration::from_secs(10),
511            sync_interval: Duration::from_secs(45),
512            webhook_health_check_interval: Some(Duration::from_secs(60)),
513            max_parallel: 4,
514            stuck_timeout: 1800,
515            no_session_stuck_timeout: 600,
516            in_review_no_session_stuck_timeout: 1920,
517            run_timeouts: RunTimeouts::default(),
518            auto_create_followup_on_changes: true,
519            graceful_shutdown_timeout: Duration::from_secs(600),
520            silence_grace_period: 300,
521            silence_cooldown: 3600,
522            upgrade_check_interval: 3600,
523            startup_failure_escalation_secs: 3600,
524        }
525    }
526}
527
528impl EngineConfig {
529    /// Stuck limit for an in_progress task whose session is alive: `stuck_timeout`, raised to the
530    /// run's own timeout plus `RUN_TIMEOUT_GRACE_SECS`
531    pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
532        self.stuck_timeout.max(
533            self.run_timeouts
534                .for_complexity(complexity)
535                .saturating_add(RUN_TIMEOUT_GRACE_SECS),
536        )
537    }
538
539    /// Load engine configuration from config files, falling back to defaults.
540    pub fn from_config() -> Self {
541        let mut config = Self::default();
542
543        if let Ok(val) = config::get("engine.tick_interval") {
544            if let Ok(secs) = val.parse::<u64>() {
545                config.tick_interval = Duration::from_secs(secs);
546            } else {
547                tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
548            }
549        }
550
551        if let Ok(val) = config::get("engine.sync_interval") {
552            if let Ok(secs) = val.parse::<u64>() {
553                config.sync_interval = Duration::from_secs(secs);
554            } else {
555                tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
556            }
557        }
558
559        if let Ok(val) = config::get("engine.max_parallel") {
560            if let Ok(n) = val.parse::<usize>() {
561                config.max_parallel = n;
562            } else {
563                tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
564            }
565        }
566
567        if let Ok(val) = config::get("engine.stuck_timeout") {
568            if let Ok(secs) = val.parse::<u64>() {
569                config.stuck_timeout = secs;
570            } else {
571                tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
572            }
573        }
574
575        if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
576            if let Ok(secs) = val.parse::<u64>() {
577                config.no_session_stuck_timeout = secs;
578            } else {
579                tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
580            }
581        }
582
583        let mut in_review_from_config = false;
584        if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
585            if let Ok(secs) = val.parse::<u64>() {
586                config.in_review_no_session_stuck_timeout = secs;
587                in_review_from_config = true;
588            } else {
589                tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
590            }
591        }
592
593        // Keep the no-session review limit over the review agent's own timeout plus delivery
594        // time, whatever either is set to. Under that, recovery resets a task whose finished
595        // review is still on its way to the engine and the result is dropped as stale
596        let review_timeout = engine_review::review_timeout_seconds();
597        let floored = engine_review::no_session_review_floor(
598            config.in_review_no_session_stuck_timeout,
599            review_timeout,
600        );
601        if floored != config.in_review_no_session_stuck_timeout {
602            if in_review_from_config {
603                tracing::warn!(
604                    configured_secs = config.in_review_no_session_stuck_timeout,
605                    review_timeout_secs = review_timeout,
606                    raised_to_secs = floored,
607                    "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
608                );
609            } else {
610                tracing::debug!(
611                    review_timeout_secs = review_timeout,
612                    raised_to_secs = floored,
613                    "raised the default in_review no-session limit over a long review timeout"
614                );
615            }
616            config.in_review_no_session_stuck_timeout = floored;
617        }
618
619        config.run_timeouts = RunTimeouts::from_config();
620
621        if let Ok(val) = config::get("engine.webhook_health_check_interval") {
622            if let Ok(secs) = val.parse::<u64>() {
623                if secs > 0 {
624                    config.webhook_health_check_interval = Some(Duration::from_secs(secs));
625                } else {
626                    config.webhook_health_check_interval = None;
627                }
628            } else {
629                tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
630            }
631        }
632
633        if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
634            config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
635        }
636
637        if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
638            if let Ok(secs) = val.parse::<u64>() {
639                config.graceful_shutdown_timeout = Duration::from_secs(secs);
640            } else {
641                tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
642            }
643        }
644
645        if let Ok(val) = config::get("engine.silence_grace_period") {
646            if let Ok(secs) = val.parse::<u64>() {
647                config.silence_grace_period = secs;
648            } else {
649                tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
650            }
651        }
652
653        if let Ok(val) = config::get("engine.silence_cooldown") {
654            if let Ok(secs) = val.parse::<u64>() {
655                config.silence_cooldown = secs;
656            } else {
657                tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
658            }
659        }
660
661        if let Ok(val) = config::get("engine.upgrade_check_interval") {
662            if let Ok(secs) = val.parse::<u64>() {
663                config.upgrade_check_interval = secs;
664            } else {
665                tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
666            }
667        }
668
669        if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
670            if let Ok(secs) = val.parse::<u64>() {
671                config.startup_failure_escalation_secs = secs;
672            } else {
673                tracing::warn!(
674                    key = "engine.startup_failure_escalation_secs",
675                    value = %val,
676                    default_secs = config.startup_failure_escalation_secs,
677                    "invalid value for engine.startup_failure_escalation_secs, using default"
678                );
679            }
680        }
681
682        config
683    }
684}
685
686/// KV key holding the epoch second the GitHub core window resets, so `orch task status`
687/// can show the limit from a separate process.
688pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
689
690/// Format an epoch second as a local `HH:MM:SS` for operator-facing output.
691pub fn format_rate_limited_until(reset_at: i64) -> String {
692    chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
693        || reset_at.to_string(),
694        |t| {
695            t.with_timezone(&chrono::Local)
696                .format("%H:%M:%S %Z")
697                .to_string()
698        },
699    )
700}
701
702/// Announce a GitHub rate-limit window on every operator surface: the log (`orch log`),
703/// a KV flag (`orch task status`) and the channels (Telegram, Discord).
704async fn report_rate_limited(
705    reset_at: i64,
706    store: Option<&Arc<TaskStore>>,
707    transport: Option<&Arc<Transport>>,
708) {
709    let m = rate_limit_metrics();
710    let until = format_rate_limited_until(reset_at);
711    tracing::warn!(
712        reset_at,
713        rate_limit_hits = m.hits,
714        skipped_calls = m.skipped_calls,
715        five_xx_hits = m.five_xx_hits,
716        rest_remaining = ?m.rest_remaining,
717        graphql_remaining = ?m.graphql_remaining,
718        "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
719    );
720
721    if let Some(store) = store
722        && let Err(e) = store
723            .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
724            .await
725    {
726        tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
727    }
728
729    if let Some(transport) = transport {
730        transport.push_notification(TaskNotification {
731            task_id: "github".to_string(),
732            title: "GitHub rate limit".to_string(),
733            status: "blocked".to_string(),
734            agent: "orch".to_string(),
735            duration_seconds: 0.0,
736            summary: format!(
737                "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
738            ),
739            repo: None,
740            notify_target: None,
741            pr_number: None,
742        });
743    }
744}
745
746/// Grow or shrink the dispatch pool toward `target` without cancelling running work.
747///
748/// Growing adds permits at once. Shrinking can only take back the permits that
749/// are free right now, so the return value is the limit actually in force; call
750/// again as running tasks release theirs
751fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
752    if target > in_force {
753        semaphore.add_permits(target - in_force);
754        return target;
755    }
756
757    if target < in_force {
758        return in_force - semaphore.forget_permits(in_force - target);
759    }
760
761    in_force
762}
763
764/// Config subtrees the engine only reads at startup: the channel gateways hold
765/// long-lived websockets and the webhook listener binds its port once.
766const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
767
768/// Read the restart-only subtrees so a later reload can tell they changed.
769fn restart_only_snapshot() -> Vec<Option<String>> {
770    RESTART_ONLY_KEYS
771        .iter()
772        .map(|key| config::get(key).ok())
773        .collect()
774}
775
776/// Report the restart-only keys an edit changed. Only the key is logged, never
777/// the value, because `channels.*` holds bot tokens
778fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
779    let changed: Vec<&'static str> = RESTART_ONLY_KEYS
780        .iter()
781        .zip(old)
782        .zip(new)
783        .filter(|((_, old), new)| old != new)
784        .map(|((key, _), _)| *key)
785        .collect();
786
787    for key in &changed {
788        tracing::info!(key = key, "config changed, takes effect after restart");
789    }
790
791    changed
792}
793
794/// Log every engine setting whose value changed across a config reload, so
795/// `orch log` shows what the operator's edit actually did.
796fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
797    macro_rules! log_changed {
798        ($($key:literal => $field:ident),+ $(,)?) => {
799            $(if old.$field != new.$field {
800                tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
801            })+
802        };
803    }
804
805    log_changed!(
806        "engine.tick_interval" => tick_interval,
807        "engine.sync_interval" => sync_interval,
808        "engine.webhook_health_check_interval" => webhook_health_check_interval,
809        "engine.max_parallel" => max_parallel,
810        "engine.stuck_timeout" => stuck_timeout,
811        "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
812        "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
813        "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
814        "engine.silence_grace_period" => silence_grace_period,
815        "engine.silence_cooldown" => silence_cooldown,
816        "engine.upgrade_check_interval" => upgrade_check_interval,
817        "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
818        "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
819    );
820}
821
822fn should_run_upgrade_check(
823    last_upgrade_check: Option<Instant>,
824    upgrade_check_interval: u64,
825) -> bool {
826    if upgrade_check_interval == 0 {
827        return false;
828    }
829
830    last_upgrade_check
831        .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
832}
833
834/// Connect one project's backend and build its task manager and runner over
835/// the shared store.
836///
837/// `event_tx` wires status transitions into the event bus; pass `None` during
838/// startup, where the bus does not exist yet and task managers are rebuilt
839/// once it does
840async fn connect_project_engine(
841    repo: String,
842    project_dir: PathBuf,
843    store: Arc<TaskStore>,
844    event_tx: Option<Sender<events::TaskEvent>>,
845) -> anyhow::Result<ProjectEngine> {
846    let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
847
848    // A failed health check (rate limit, network, auth) must not drop the project: it would have
849    // no review subscriber until the next restart. Later ticks hit the API again and retry on
850    // their own, so build the engine and warn while the check keeps failing.
851    match backend.health_check().await {
852        Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
853        Err(e) => tracing::warn!(
854            repo = %repo,
855            error = %e,
856            "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
857        ),
858    }
859
860    let task_manager = Arc::new(match event_tx {
861        Some(tx) => {
862            TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
863        }
864        None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
865    });
866    let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
867
868    Ok(ProjectEngine {
869        repo,
870        project_dir,
871        backend,
872        task_manager,
873        runner,
874        store,
875    })
876}
877
878/**
879 * Split the configured project list into the ones not running yet and the names of the ones that
880 * went away. Kept separate from `sync_project_engines` so the comparison can be tested without a
881 * GitHub backend: getting it wrong either spawns a second set of subscribers for a project already
882 * running, or silently never picks a new one up
883 **/
884fn diff_configured_projects(
885    configured: &[(String, PathBuf)],
886    active_repos: &HashSet<String>,
887) -> (Vec<(String, PathBuf)>, Vec<String>) {
888    let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
889
890    let added = configured
891        .iter()
892        .filter(|(repo, _)| !active_repos.contains(repo))
893        .cloned()
894        .collect();
895    let removed = active_repos
896        .iter()
897        .filter(|repo| !configured_repos.contains(repo.as_str()))
898        .cloned()
899        .collect();
900
901    (added, removed)
902}
903
904/// Pick up projects added to `projects:` since startup, and report the ones
905/// that were removed.
906///
907/// A removed project keeps running until the next restart: its subscriber
908/// tasks and any agent it already dispatched outlive the config read
909#[expect(
910    clippy::too_many_arguments,
911    reason = "a new project engine needs every shared engine handle"
912)]
913async fn sync_project_engines(
914    store: &Arc<TaskStore>,
915    project_engines: &mut Vec<ProjectEngine>,
916    active_repos: &mut HashSet<String>,
917    event_bus: &events::EventBus,
918    tmux: &Arc<TmuxManager>,
919    capture: &Arc<CaptureService>,
920    semaphore: &Arc<Semaphore>,
921    review_semaphore: &Arc<Semaphore>,
922    weight_tx: &mpsc::Sender<WeightSignal>,
923    router: &Arc<RwLock<Router>>,
924    dispatching: &Arc<dashmap::DashMap<String, String>>,
925) {
926    let configured = match config::get_projects_with_paths() {
927        Ok(p) if !p.is_empty() => p,
928        Ok(_) => return,
929        Err(e) => {
930            tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
931            return;
932        }
933    };
934    let (added, removed) = diff_configured_projects(&configured, active_repos);
935    for repo in removed {
936        tracing::info!(
937            repo = %repo,
938            "project removed from config, takes effect after restart"
939        );
940    }
941
942    for (repo, project_dir) in added {
943        config::watch(&project_dir.join(".orch.yml"));
944        let engine = match connect_project_engine(
945            repo.clone(),
946            project_dir,
947            Arc::clone(store),
948            Some(event_bus.sender()),
949        )
950        .await
951        {
952            Ok(engine) => engine,
953            Err(e) => {
954                // A failed health check no longer lands here, so this is a backend that could
955                // not be built at all, such as a repo slug that is not `owner/name`
956                tracing::warn!(
957                    repo = %repo,
958                    err = %e,
959                    "project added to config but its backend could not be built, retrying on the next config change"
960                );
961                continue;
962            }
963        };
964
965        subscribers::dispatch::spawn(
966            event_bus.subscribe(),
967            Arc::clone(&engine.backend),
968            Arc::clone(tmux),
969            Arc::clone(&engine.runner),
970            Arc::clone(capture),
971            Arc::clone(semaphore),
972            Arc::clone(&engine.task_manager),
973            weight_tx.clone(),
974            Arc::clone(router),
975            Arc::clone(dispatching),
976            Arc::clone(&engine.store),
977            engine.repo.clone(),
978        );
979        subscribers::review::spawn(
980            event_bus.subscribe(),
981            Arc::clone(&engine.backend),
982            Arc::clone(tmux),
983            Arc::clone(review_semaphore),
984            Arc::clone(&engine.task_manager),
985            Arc::clone(router),
986            Arc::clone(dispatching),
987            Arc::clone(&engine.store),
988            engine.repo.clone(),
989        );
990        subscribers::unblock::spawn(
991            event_bus.subscribe(),
992            Arc::clone(&engine.backend),
993            Arc::clone(&engine.task_manager),
994            Arc::clone(&engine.store),
995            engine.repo.clone(),
996        );
997
998        active_repos.insert(repo.clone());
999        project_engines.push(engine);
1000        tracing::info!(repo = %repo, "project added from config, now syncing");
1001    }
1002}
1003
1004/// Initialize all project engines from config.
1005///
1006/// Returns a vector of ProjectEngine, one for each configured project.
1007/// The task store every project shares, with the cluster and its local write endpoint when on
1008async fn open_shared_store() -> anyhow::Result<Arc<TaskStore>> {
1009    let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1010    if cluster::enabled() {
1011        cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1012        local::serve((*store).clone()).await?;
1013    }
1014    Ok(store)
1015}
1016
1017/// One engine per configured project. None is fine, the web UI adds the first one and
1018/// `sync_project_engines` starts it
1019async fn init_project_engines(store: &Arc<TaskStore>) -> anyhow::Result<Vec<ProjectEngine>> {
1020    let projects = config::get_projects_with_paths()?;
1021    let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1022    tracing::info!(repos = ?repos, "loading projects from config");
1023
1024    if projects.is_empty() {
1025        tracing::info!("no projects yet, add one with `orch ui` or `orch project add`");
1026    }
1027
1028    let mut engines = Vec::new();
1029    for (repo, project_dir) in projects {
1030        tracing::info!(repo = %repo, "initializing project engine");
1031        config::watch(&project_dir.join(".orch.yml"));
1032
1033        engines.push(connect_project_engine(repo, project_dir, Arc::clone(store), None).await?);
1034    }
1035
1036    Ok(engines)
1037}
1038
1039async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1040    for engine in project_engines {
1041        let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1042        let repo_root_path = PathBuf::from(&repo_root);
1043        let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1044        let worktrees = list_project_worktrees(&repo_root_path).await?;
1045
1046        // Fetch refs once for all worktrees in this project, so rebase below
1047        // operates on up-to-date origin/* refs without N sequential fetches.
1048        if let Err(e) = git_ops::git()
1049            .await
1050            .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1051            .output_with_context()
1052            .await
1053        {
1054            tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1055        }
1056
1057        let mut orphans_removed: u32 = 0;
1058        let mut invalid_removed: u32 = 0;
1059        let mut cleaned: u32 = 0;
1060        let mut valid_kept: u32 = 0;
1061
1062        for worktree_dir in worktrees {
1063            let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1064                continue;
1065            };
1066
1067            // Validate gitdir link before doing anything else — a broken .git file
1068            // means git commands on this worktree will fail with "not a git repository".
1069            if !validate_worktree_gitdir(&worktree_dir).await {
1070                tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1071                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1072                    .await;
1073                invalid_removed += 1;
1074                continue;
1075            }
1076
1077            let Some(task_id) = task_id_from_worktree_name(name) else {
1078                tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1079                remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1080                    .await;
1081                orphans_removed += 1;
1082                continue;
1083            };
1084
1085            let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1086                tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1087                remove_worktree_and_branch(
1088                    &task_id,
1089                    &worktree_dir,
1090                    Some(name),
1091                    &repo_root_path,
1092                    false,
1093                )
1094                .await;
1095                orphans_removed += 1;
1096                continue;
1097            };
1098
1099            let task = engine.store.get(store_id).await?;
1100            let branch_name = if task.branch.is_empty() {
1101                name
1102            } else {
1103                task.branch.as_str()
1104            };
1105
1106            match task.status {
1107                TaskStatus::New
1108                | TaskStatus::Routed
1109                | TaskStatus::InProgress
1110                | TaskStatus::NeedsReview
1111                | TaskStatus::InReview => {
1112                    let keep_conflict = worktree::keep_conflicted_rebase(
1113                        task.merge_conflict_retries,
1114                        task.pr_number,
1115                    );
1116                    if keep_conflict && git_ops::rebase_in_progress(&worktree_dir).await {
1117                        tracing::info!(
1118                            repo = %engine.repo,
1119                            task_id = %task_id,
1120                            worktree = %worktree_dir.display(),
1121                            "keeping the conflicted rebase auto_merge left for the agent"
1122                        );
1123                        valid_kept += 1;
1124                        continue;
1125                    }
1126                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1127                    let origin_branch = format!("origin/{default_branch}");
1128                    let outcome = git_ops::stash_rebase_restore(
1129                        &worktree_dir,
1130                        &origin_branch,
1131                        git_ops::StashRebaseOpts {
1132                            abort_on_failure: false,
1133                        },
1134                    )
1135                    .await;
1136                    let conflict_left = git_ops::rebase_in_progress(&worktree_dir).await;
1137                    let failed_err = match outcome {
1138                        Ok(
1139                            git_ops::RebaseOutcome::Succeeded | git_ops::RebaseOutcome::Skipped(_),
1140                        ) => None,
1141                        Ok(git_ops::RebaseOutcome::Failed(e)) if keep_conflict && conflict_left => {
1142                            tracing::info!(
1143                                repo = %engine.repo,
1144                                task_id = %task_id,
1145                                err = %e,
1146                                "startup rebase hit a content conflict, leaving it for the agent"
1147                            );
1148                            None
1149                        }
1150                        Ok(git_ops::RebaseOutcome::Failed(e)) => Some(e),
1151                        Err(e) => Some(e.to_string()),
1152                    };
1153                    if let Some(e) = failed_err {
1154                        reset_startup_task_after_failed_rebase(
1155                            engine,
1156                            &task_id,
1157                            &worktree_dir,
1158                            branch_name,
1159                            &repo_root_path,
1160                            task.pr_number.is_some(),
1161                            &e,
1162                        )
1163                        .await;
1164                        cleaned += 1;
1165                    } else {
1166                        valid_kept += 1;
1167                    }
1168                }
1169                // A blocked or waiting task resumes from this work, and without a PR the branch
1170                // may hold the only copy of it. An approved task's PR is updated on GitHub
1171                TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1172                    valid_kept += 1;
1173                }
1174                TaskStatus::Done => {
1175                    tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1176                    remove_worktree_and_branch_for(
1177                        &task_id,
1178                        &worktree_dir,
1179                        Some(branch_name),
1180                        &repo_root_path,
1181                        false,
1182                        task.pr_number.is_some(),
1183                    )
1184                    .await;
1185                    cleaned += 1;
1186                }
1187            }
1188        }
1189
1190        tracing::info!(
1191            repo = %engine.repo,
1192            orphans_removed,
1193            invalid_removed,
1194            cleaned,
1195            valid_kept,
1196            "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1197        );
1198
1199        // Prune stale .git/worktrees/ entries whose directories no longer exist.
1200        // This handles entries left behind by crashes or manual directory removal.
1201        let prune = Command::new("git")
1202            .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1203            .output_with_context()
1204            .await;
1205        match prune {
1206            Ok(output) if output.status.success() => {
1207                tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1208            }
1209            Ok(output) => {
1210                let stderr = String::from_utf8_lossy(&output.stderr);
1211                tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1212            }
1213            Err(e) => {
1214                tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1215            }
1216        }
1217    }
1218
1219    Ok(())
1220}
1221
1222async fn reset_startup_task_after_failed_rebase(
1223    engine: &ProjectEngine,
1224    task_id: &str,
1225    worktree_dir: &Path,
1226    branch_name: &str,
1227    repo_root: &Path,
1228    keep_remote: bool,
1229    err: &str,
1230) {
1231    tracing::warn!(repo = %engine.repo, task_id, err, "startup rebase failed, resetting task");
1232    abort_worktree_rebase(worktree_dir).await;
1233    remove_worktree_and_branch(
1234        task_id,
1235        worktree_dir,
1236        Some(branch_name),
1237        repo_root,
1238        keep_remote,
1239    )
1240    .await;
1241    let keep_branch = local_branch_exists(repo_root, branch_name).await;
1242    match engine.store.resolve_task_id(&engine.repo, task_id).await {
1243        Ok(Some(reset_id)) => {
1244            let result = if keep_branch {
1245                engine.store.reset_to_new_keep_branch(reset_id).await
1246            } else {
1247                engine.store.reset_to_new(reset_id).await
1248            };
1249            if let Err(e) = result {
1250                tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1251            }
1252        }
1253        Ok(None) => {
1254            tracing::warn!(task_id, "task not found in store during reconciliation");
1255        }
1256        Err(e) => {
1257            tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1258        }
1259    }
1260    if let Err(e) = engine
1261        .task_manager
1262        .update_task_status(&ExternalId(task_id.to_string()), Status::New)
1263        .await
1264    {
1265        tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1266    }
1267}
1268
1269/// Sync Fibonacci estimates stored in SQLite to the GitHub Projects board for
1270/// all tasks that have a positive estimate.
1271///
1272/// Runs once on startup to catch tasks that were routed before the estimate
1273/// sync feature was available or before `project_estimate_field_id` was
1274/// configured. The underlying mutations are idempotent, so re-syncing an
1275/// already-correct estimate is harmless.
1276async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1277    for engine in project_engines {
1278        let tasks = match engine.store.list_external_tasks_with_estimates().await {
1279            Ok(t) => t,
1280            Err(e) => {
1281                tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1282                continue;
1283            }
1284        };
1285
1286        if tasks.is_empty() {
1287            continue;
1288        }
1289
1290        let mut synced: u32 = 0;
1291        let mut failed: u32 = 0;
1292        for (external_id, estimate) in &tasks {
1293            match engine
1294                .backend
1295                .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1296                .await
1297            {
1298                Ok(()) => synced += 1,
1299                Err(e) => {
1300                    tracing::debug!(
1301                        repo = %engine.repo,
1302                        task_id = %external_id,
1303                        estimate,
1304                        err = %e,
1305                        "startup estimate reconciliation: sync failed"
1306                    );
1307                    failed += 1;
1308                }
1309            }
1310        }
1311
1312        if synced > 0 || failed > 0 {
1313            tracing::info!(
1314                repo = %engine.repo,
1315                synced,
1316                failed,
1317                "startup estimate reconciliation complete"
1318            );
1319        }
1320    }
1321}
1322
1323/// Read per-project channel configuration from `.orch.yml`.
1324fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1325    let config_path = project_dir.join(".orch.yml");
1326    if !config_path.exists() {
1327        return ProjectChannelConfig::default();
1328    }
1329    let content = match fs::read_to_string(&config_path) {
1330        Ok(c) => c,
1331        Err(e) => {
1332            tracing::warn!(
1333                path = %config_path.display(),
1334                error = %e,
1335                "failed to read .orch.yml; channel config will use defaults"
1336            );
1337            return ProjectChannelConfig::default();
1338        }
1339    };
1340    let val: serde_norway::Value = match serde_norway::from_str(&content) {
1341        Ok(v) => v,
1342        Err(e) => {
1343            tracing::warn!(
1344                path = %config_path.display(),
1345                error = %e,
1346                "failed to parse .orch.yml; channel config will use defaults"
1347            );
1348            return ProjectChannelConfig::default();
1349        }
1350    };
1351    let Some(channels) = val.get("channels") else {
1352        return ProjectChannelConfig::default();
1353    };
1354    let get_str = |section: &str, key: &str| -> Option<String> {
1355        channels
1356            .get(section)
1357            .and_then(|s| s.get(key))
1358            .and_then(|v| v.as_str())
1359            .map(String::from)
1360    };
1361    ProjectChannelConfig {
1362        telegram_topic_id: get_str("telegram", "topic_id"),
1363        discord_channel_id: get_str("discord", "channel_id"),
1364    }
1365}
1366
1367/// Fetch the latest orch release tag from GitHub via the `gh` CLI (async).
1368///
1369/// Returns the version string (without the leading "v"), or `None` when the
1370/// check fails (no network, unauthenticated, `gh` not installed, etc.).
1371async fn fetch_latest_release_version() -> Option<String> {
1372    let output = Command::new("gh")
1373        .args([
1374            "api",
1375            "repos/gabrielkoerich/orch/releases/latest",
1376            "--jq",
1377            ".tag_name",
1378        ])
1379        .output()
1380        .await
1381        .ok()?;
1382    if !output.status.success() {
1383        return None;
1384    }
1385    let tag = String::from_utf8(output.stdout).ok()?;
1386    let version = tag.trim().trim_start_matches('v').to_string();
1387    if version.is_empty() {
1388        None
1389    } else {
1390        Some(version)
1391    }
1392}
1393
1394/// Check if a newer orch release is available and send channel notifications
1395/// asking the operator to upgrade manually. Upgrading is operator-only — the
1396/// service never runs `brew` itself.
1397///
1398/// Uses the store's KV to throttle checks (`upgrade:last_check_at`) and
1399/// deduplicate notifications (`upgrade:last_notified_version`). Each distinct
1400/// latest version is notified about once; the notification repeats only when
1401/// an even newer version is released.
1402async fn check_and_notify_upgrade(
1403    store: &Arc<TaskStore>,
1404    channels: &Arc<ChannelRegistry>,
1405    current_version: &str,
1406    check_interval: Duration,
1407) {
1408    let last_check_key = "upgrade:last_check_at";
1409    let last_notified_key = "upgrade:last_notified_version";
1410
1411    // Throttle: only check once per interval.
1412    let now = chrono::Utc::now().timestamp();
1413    if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1414        && let Ok(last_check) = last_check_str.parse::<i64>()
1415        && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1416    {
1417        return;
1418    }
1419
1420    // Record check timestamp (best-effort).
1421    let _ = store.kv_set(last_check_key, &now.to_string()).await;
1422
1423    let Some(latest) = fetch_latest_release_version().await else {
1424        tracing::debug!("upgrade check: failed to fetch latest release version");
1425        return;
1426    };
1427
1428    if latest == current_version {
1429        tracing::debug!(current = %current_version, "orch is up to date");
1430        return;
1431    }
1432
1433    tracing::warn!(
1434        current_version = %current_version,
1435        latest_version = %latest,
1436        "orch upgrade available"
1437    );
1438
1439    // Operator-only upgrade: send channel notifications so the operator can
1440    // upgrade on their own schedule. The service never runs brew itself.
1441
1442    // Deduplicate: don't re-notify for the same latest version.
1443    if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1444        && last_notified == latest
1445    {
1446        tracing::debug!(latest = %latest, "upgrade already notified");
1447        return;
1448    }
1449
1450    let msg_telegram = format!(
1451        "⚠️ <b>Orch Upgrade Available</b>\n\n\
1452         Service: <code>{current_version}</code>\n\
1453         Latest:  <code>{latest}</code>\n\n\
1454         Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1455    );
1456
1457    let msg_discord = format!(
1458        "⚠️ **Orch Upgrade Available**\n\n\
1459         Service: `{current_version}`\n\
1460         Latest:  `{latest}`\n\n\
1461         Run: `brew update && brew upgrade orch && brew services restart orch`"
1462    );
1463
1464    let msg_slack = format!(
1465        "⚠️ *Orch Upgrade Available*\n\n\
1466         Service: `{current_version}`\n\
1467         Latest:  `{latest}`\n\n\
1468         Run: `brew update && brew upgrade orch && brew services restart orch`"
1469    );
1470
1471    for channel in channels.iter() {
1472        let (body, metadata, topic_id) = match channel.name() {
1473            "telegram" => (
1474                msg_telegram.clone(),
1475                serde_json::json!({"preformatted_html": true}),
1476                None,
1477            ),
1478            "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1479            "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1480            _ => continue,
1481        };
1482
1483        let msg = OutgoingMessage {
1484            thread_id: "upgrade".to_string(),
1485            body,
1486            reply_to: None,
1487            metadata,
1488            topic_id,
1489        };
1490
1491        if let Err(e) = channel.send(&msg).await {
1492            tracing::warn!(
1493                channel = channel.name(),
1494                ?e,
1495                "failed to send upgrade notification"
1496            );
1497        }
1498    }
1499
1500    // Record that we notified for this latest version.
1501    let _ = store.kv_set(last_notified_key, &latest).await;
1502}
1503
1504/// Send a one-time escalation notification when the startup retry loop has been
1505/// spinning for longer than `startup_failure_escalation_secs`.
1506///
1507/// This is called *before* the channel registry is initialised (the retry loop
1508/// sits above it in `serve()`), so we construct channel objects directly from
1509/// config — mirroring the approach used during normal channel registration, but
1510/// without relying on `ChannelRegistry`.
1511async fn send_startup_escalation_notification(elapsed_secs: u64) {
1512    #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1513    let duration = notification::format_duration(elapsed_secs as f64);
1514
1515    let msg_telegram = format!(
1516        "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1517         The engine has been retrying backend connections for {duration}\n\
1518         without success. All task processing, routing, and scheduled\n\
1519         jobs are paused. Check service logs with `orch log -f`."
1520    );
1521
1522    let msg_discord = format!(
1523        "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1524         The engine has been retrying backend connections for {duration}\n\
1525         without success. All task processing, routing, and scheduled\n\
1526         jobs are paused. Check service logs with `orch log -f`."
1527    );
1528
1529    let msg_slack = format!(
1530        "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1531         The engine has been retrying backend connections for {duration}\n\
1532         without success. All task processing, routing, and scheduled\n\
1533         jobs are paused. Check service logs with `orch log -f`."
1534    );
1535
1536    // Telegram
1537    let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1538    let tg_chat_id = config::get("channels.telegram.chat_id");
1539    if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1540        && !token.is_empty()
1541        && !chat_id.is_empty()
1542        && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1543    {
1544        let msg = OutgoingMessage {
1545            thread_id: "startup_escalation".to_string(),
1546            body: msg_telegram.clone(),
1547            reply_to: None,
1548            metadata: serde_json::json!({ "preformatted_html": true }),
1549            topic_id: config::get("channels.telegram.general_topic_id").ok(),
1550        };
1551        match channel.send(&msg).await {
1552            Err(e) => {
1553                tracing::warn!(?e, "failed to send startup escalation via telegram");
1554            }
1555            _ => {
1556                tracing::info!("startup escalation notification sent to telegram");
1557            }
1558        }
1559    }
1560
1561    // Discord
1562    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1563        let channel_id = config::get("channels.discord.channel_id")
1564            .or_else(|_| config::get("channels.discord.general_channel_id"));
1565        if !token.is_empty()
1566            && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1567        {
1568            let shard_id = config::get("channels.discord.shard_id")
1569                .ok()
1570                .and_then(|v| v.parse::<u64>().ok())
1571                .unwrap_or(0);
1572            let shard_count = config::get("channels.discord.shard_count")
1573                .ok()
1574                .and_then(|v| v.parse::<u64>().ok())
1575                .unwrap_or(1);
1576            if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1577                let msg = OutgoingMessage {
1578                    thread_id: "startup_escalation".to_string(),
1579                    body: msg_discord.clone(),
1580                    reply_to: None,
1581                    metadata: serde_json::Value::Null,
1582                    topic_id: None,
1583                };
1584                match gateway.send(&msg).await {
1585                    Err(e) => {
1586                        tracing::warn!(?e, "failed to send startup escalation via discord");
1587                    }
1588                    _ => {
1589                        tracing::info!("startup escalation notification sent to discord");
1590                    }
1591                }
1592            }
1593        }
1594    }
1595
1596    // Slack
1597    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1598        let channel_id = config::get("channels.slack.channel_id");
1599        if !token.is_empty()
1600            && let Ok(cid) = channel_id
1601            && let Ok(channel) = SlackChannel::new(token, Some(cid))
1602        {
1603            let msg = OutgoingMessage {
1604                thread_id: "startup_escalation".to_string(),
1605                body: msg_slack.clone(),
1606                reply_to: None,
1607                metadata: serde_json::Value::Null,
1608                topic_id: None,
1609            };
1610            match channel.send(&msg).await {
1611                Err(e) => {
1612                    tracing::warn!(?e, "failed to send startup escalation via slack");
1613                }
1614                _ => {
1615                    tracing::info!("startup escalation notification sent to slack");
1616                }
1617            }
1618        }
1619    }
1620}
1621
1622/// Start the orch service.
1623///
1624/// This is the main entry point — called by `orch serve`.
1625pub async fn serve() -> anyhow::Result<()> {
1626    let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1627    if killed > 0 {
1628        tracing::info!(count = killed, "terminated stale orch serve processes");
1629    }
1630
1631    tracing::info!("orch engine starting");
1632
1633    // Pre-create standard directories with async I/O so subsequent synchronous
1634    // callers (state_dir, orch_home, etc.) find them already present and their
1635    // create_dir_all calls become fast no-ops.
1636    if let Err(e) = home::init_dirs().await {
1637        tracing::warn!("failed to initialize orch directories: {e}");
1638    }
1639
1640    // Pre-warm the config cache with async I/O so get() / get_list() calls in
1641    // hot async paths never block a Tokio thread on std::fs::read_to_string.
1642    if let Err(e) = config::warm_cache().await {
1643        tracing::warn!("failed to warm config cache: {e}");
1644    }
1645
1646    let mut config = EngineConfig::from_config();
1647
1648    // One store for every project, opened before them so the web UI can run with none. A failure
1649    // here retries like a project failure did, so launchd does not restart the service in a loop
1650    let store = {
1651        let mut delay_secs = 5u64;
1652        let mut escalated = false;
1653        let start = Instant::now();
1654        loop {
1655            match open_shared_store().await {
1656                Ok(store) => break store,
1657                Err(e) => {
1658                    let elapsed = start.elapsed().as_secs();
1659                    let escalation = config.startup_failure_escalation_secs;
1660                    // Escalates once like a project that will not start, then keeps retrying
1661                    if escalation > 0 && !escalated && elapsed >= escalation {
1662                        escalated = true;
1663                        tracing::error!(
1664                            elapsed_secs = elapsed,
1665                            "the store or the cluster has not started, escalating: {e:#}"
1666                        );
1667                        send_startup_escalation_notification(elapsed).await;
1668                    } else {
1669                        tracing::warn!(
1670                            delay_secs,
1671                            "could not open the store or start the cluster, retrying: {e:#}"
1672                        );
1673                    }
1674
1675                    time::sleep(Duration::from_secs(delay_secs)).await;
1676                    delay_secs = (delay_secs * 2).min(120);
1677                }
1678            }
1679        }
1680    };
1681
1682    // Load persisted model cooldowns and register store for future writes.
1683    // Runs once here so the global cooldown store is not overwritten per project.
1684    engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1685    tracing::info!("internal database ready");
1686
1687    // Create the event bus for task status transitions
1688    let event_bus = events::EventBus::new(256);
1689    if let Err(e) = event_bus.start_ws_server().await {
1690        tracing::warn!(
1691            ?e,
1692            "failed to start event websocket server, continuing without it"
1693        );
1694    }
1695
1696    // Before the projects, so a first run with none can add one from the UI
1697    if let Err(e) = ui::serve(Arc::clone(&store), event_bus.sender()).await {
1698        tracing::warn!(?e, "failed to start the web UI, continuing without it");
1699    }
1700
1701    // Initialize project engines — retry with backoff so a network outage at
1702    // startup doesn't cause a crash-loop (launchd KeepAlive would restart us
1703    // immediately, filling the error log with 600+ identical messages).
1704    //
1705    // IMPORTANT: errors from init_project_engines() must NEVER propagate past
1706    // this loop to main().  If they did, anyhow would write "Error: …" to stderr,
1707    // which brew routes to orch.error.log — polluting it even when projects ARE
1708    // configured correctly but GitHub is temporarily unreachable.
1709    let startup_escalation_secs = config.startup_failure_escalation_secs;
1710    let mut project_engines = {
1711        let mut delay_secs = 5u64;
1712        let mut attempt = 0u32;
1713        let mut escalated = false;
1714        let start = Instant::now();
1715        loop {
1716            attempt += 1;
1717            match init_project_engines(&store).await {
1718                Ok(engines) => break engines,
1719                Err(e) => {
1720                    let elapsed = start.elapsed().as_secs();
1721
1722                    // Short blips (under the escalation threshold) stay at WARN
1723                    // so they don't flood the operator's attention. Once the
1724                    // outage crosses the threshold, escalate to ERROR-level
1725                    // logging and send a one-time push notification to configured
1726                    // channels — "transient" has clearly become "prolonged."
1727                    if startup_escalation_secs > 0
1728                        && !escalated
1729                        && elapsed >= startup_escalation_secs
1730                    {
1731                        escalated = true;
1732                        #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1733                        let outage = notification::format_duration(elapsed as f64);
1734                        tracing::error!(
1735                            delay_secs,
1736                            attempt,
1737                            elapsed_secs = elapsed,
1738                            threshold_secs = startup_escalation_secs,
1739                            "project backends unavailable for {outage} — escalating startup failure: {e}",
1740                        );
1741                        // Send one-time notification to configured channels.
1742                        // Channels aren't registered yet at this stage, so we
1743                        // construct them directly from config (mirrors orch notify).
1744                        send_startup_escalation_notification(elapsed).await;
1745                    } else {
1746                        tracing::warn!(
1747                            delay_secs,
1748                            attempt,
1749                            "project backends unavailable, retrying: {e}"
1750                        );
1751                    }
1752
1753                    time::sleep(Duration::from_secs(delay_secs)).await;
1754                    delay_secs = (delay_secs * 2).min(120);
1755                }
1756            }
1757        }
1758    };
1759
1760    tracing::info!(
1761        project_count = project_engines.len(),
1762        "initialized project engines"
1763    );
1764
1765    // Full set of currently-configured project repos, used by the per-tick global
1766    // sweeps (e.g. `dispatch_routed_tasks_global`) to distinguish "orphaned" tasks
1767    // (repo no longer in config) from tasks belonging to some other active project.
1768    let mut active_repos: HashSet<String> =
1769        project_engines.iter().map(|e| e.repo.clone()).collect();
1770
1771    // Check for a newer orch release in the background and log if the service is behind.
1772    // The periodic checker sends channel notifications after channels are initialized;
1773    // upgrading is operator-only and never runs automatically.
1774    {
1775        let current = env!("ORCH_VERSION").to_string();
1776        tokio::spawn(async move {
1777            if let Some(latest) = fetch_latest_release_version().await
1778                && latest != current
1779            {
1780                tracing::warn!(
1781                    current_version = %current,
1782                    latest_version = %latest,
1783                    "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1784                );
1785            }
1786        });
1787    }
1788
1789    // Re-create task managers with shared store and event bus
1790    for engine in &mut project_engines {
1791        engine.task_manager = Arc::new(TaskManager::with_events(
1792            Arc::clone(&engine.backend),
1793            Arc::clone(&engine.store),
1794            engine.repo.clone(),
1795            event_bus.sender(),
1796        ));
1797    }
1798
1799    // Startup reconciliation decides task status, so only the leader runs it
1800    let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1801    if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1802        tracing::warn!(err = %e, "startup worktree reconciliation failed");
1803    }
1804
1805    if leads {
1806        // Run estimate reconciliation in the background so it never blocks startup.
1807        // Each GraphQL mutation takes ~0.8 s; with hundreds of tasks this would
1808        // otherwise delay the engine reaching its main loop by several minutes.
1809        let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1810        tokio::spawn(async move {
1811            reconcile_startup_estimates(&engines_snapshot).await;
1812        });
1813    }
1814
1815    // Build ChannelRouter from global config + per-project configs
1816    let global_channel_config = GlobalChannelConfig {
1817        telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1818        discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1819        control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1820        control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1821    };
1822    let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1823        .iter()
1824        .map(|e| {
1825            let cfg = read_project_channel_config(&e.project_dir);
1826            (e.repo.clone(), cfg)
1827        })
1828        .collect();
1829    let channel_router = Arc::new(ChannelRouter::new(
1830        &global_channel_config,
1831        &project_channel_configs,
1832    ));
1833    tracing::info!(
1834        projects = ?channel_router.projects(),
1835        "channel router initialized"
1836    );
1837
1838    // Initialize tmux manager (shared across all projects)
1839    let tmux = Arc::new(TmuxManager::new());
1840    // Kill leftover sessions from a previous process. This is daemon start only,
1841    // never a tick, because nextest makes a once-flag in tick_all kill live agents
1842    tick::startup_cleanup(&tmux).await;
1843
1844    // Initialize transport
1845    let transport = Arc::new(Transport::new());
1846
1847    // Initialize capture service and start background loop
1848    let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1849    let capture_for_tick = Arc::clone(&capture);
1850    tokio::spawn(async move {
1851        capture.start().await;
1852    });
1853
1854    // Initialize channel registry
1855    let mut channel_registry = ChannelRegistry::new();
1856
1857    // Try to initialize Telegram channel
1858    if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1859        && !token.is_empty()
1860    {
1861        let chat_id = config::get("channels.telegram.chat_id").ok();
1862        match TelegramChannel::new(token, chat_id) {
1863            Ok(telegram) => {
1864                if let Err(e) = telegram.health_check().await {
1865                    tracing::warn!(?e, "telegram channel health check failed, skipping");
1866                } else {
1867                    channel_registry.register(Box::new(telegram));
1868                    tracing::info!("telegram channel registered");
1869                }
1870            }
1871            Err(e) => {
1872                tracing::warn!(?e, "failed to create telegram channel, skipping");
1873            }
1874        }
1875    }
1876
1877    // Try to initialize Discord Gateway channel (websocket)
1878    if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1879        && !token.is_empty()
1880    {
1881        let channel_id = config::get("channels.discord.channel_id").ok();
1882        let shard_id = config::get("channels.discord.shard_id")
1883            .ok()
1884            .and_then(|v| v.parse::<u64>().ok())
1885            .unwrap_or(0);
1886        let shard_count = config::get("channels.discord.shard_count")
1887            .ok()
1888            .and_then(|v| v.parse::<u64>().ok())
1889            .unwrap_or(1);
1890        match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1891            Ok(discord) => {
1892                if let Err(e) = discord.health_check().await {
1893                    tracing::warn!(?e, "discord gateway health check failed, skipping");
1894                } else {
1895                    channel_registry.register(Box::new(discord));
1896                    tracing::info!(shard_id, shard_count, "discord gateway registered");
1897                }
1898            }
1899            Err(e) => {
1900                tracing::warn!(?e, "failed to create discord gateway, skipping");
1901            }
1902        }
1903    }
1904
1905    // Try to initialize Slack channel
1906    if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1907        && !token.is_empty()
1908    {
1909        let channel_id = config::get("channels.slack.channel_id").ok();
1910        match SlackChannel::new(token, channel_id) {
1911            Ok(slack) => {
1912                if let Err(e) = slack.health_check().await {
1913                    tracing::warn!(?e, "slack channel health check failed, skipping");
1914                } else {
1915                    channel_registry.register(Box::new(slack));
1916                    tracing::info!("slack channel registered");
1917                }
1918            }
1919            Err(e) => {
1920                tracing::warn!(?e, "failed to create slack channel, skipping");
1921            }
1922        }
1923    }
1924
1925    // Initialize tmux channel with transport for output streaming
1926    let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1927    channel_registry.register(Box::new(tmux_channel));
1928    tracing::info!("tmux channel registered");
1929
1930    // Start all channels and collect their message receivers
1931    let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1932    for channel in channel_registry.iter() {
1933        match channel.start().await {
1934            Ok(rx) => {
1935                tracing::info!(channel = channel.name(), "channel started");
1936                channel_receivers.push(rx);
1937            }
1938            Err(e) => {
1939                tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1940            }
1941        }
1942    }
1943
1944    // Wrap channel registry in Arc for shared access (notification dispatcher needs it)
1945    let channel_registry = Arc::new(channel_registry);
1946
1947    // Spawn tasks to handle incoming channel messages (if any channels are active)
1948    let transport_for_messages = Arc::clone(&transport);
1949    let tmux_for_messages = Arc::clone(&tmux);
1950    let capture_for_messages = Arc::clone(&capture_for_tick);
1951    let channels_for_messages = Arc::clone(&channel_registry);
1952    // Lightweight engine references for command execution and task creation
1953    let engine_refs: Vec<EngineRef> = project_engines
1954        .iter()
1955        .map(|e| {
1956            (
1957                e.repo.clone(),
1958                Arc::clone(&e.backend),
1959                Arc::clone(&e.task_manager),
1960                Some(Arc::clone(&e.store)),
1961            )
1962        })
1963        .collect();
1964    // Shared in-memory map for pending project picks (General channel multi-project picker).
1965    let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1966    let router_for_messages = Arc::clone(&channel_router);
1967    for mut rx in channel_receivers {
1968        let transport = Arc::clone(&transport_for_messages);
1969        let tmux = Arc::clone(&tmux_for_messages);
1970        let capture = Arc::clone(&capture_for_messages);
1971        let channels = Arc::clone(&channels_for_messages);
1972        let engine_refs = engine_refs.clone();
1973        let ch_router = Arc::clone(&router_for_messages);
1974        let picks = Arc::clone(&pending_picks);
1975        tokio::spawn(async move {
1976            while let Some(msg) = rx.recv().await {
1977                tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1978                channel_handler::handle_channel_message(
1979                    msg,
1980                    &transport,
1981                    &tmux,
1982                    &capture,
1983                    &channels,
1984                    &engine_refs,
1985                    &ch_router,
1986                    &picks,
1987                )
1988                .await;
1989            }
1990        });
1991    }
1992
1993    // Post-restart confirmation: if the service was restarted via the /restart control command,
1994    // a KV marker was stored before shutdown. Send a confirmation to the originating channel now
1995    // that we're back online. Use a short delay to let the channel connections re-establish.
1996    {
1997        let restart_channels = Arc::clone(&channel_registry);
1998        let restart_store = Some(Arc::clone(&store));
1999        if let Some(store) = restart_store {
2000            tokio::spawn(async move {
2001                time::sleep(Duration::from_secs(5)).await;
2002                match store.kv_get("control:restart_pending").await {
2003                    Ok(Some(conv_key)) => {
2004                        if let Some((ch_name, thread_id, topic_id)) =
2005                            transport::parse_conversation_key(&conv_key)
2006                        {
2007                            for ch in restart_channels.iter() {
2008                                if ch.name() == ch_name {
2009                                    let msg = OutgoingMessage {
2010                                        thread_id: thread_id.to_string(),
2011                                        body: "orch restarted successfully.".to_string(),
2012                                        reply_to: None,
2013                                        metadata: serde_json::json!({}),
2014                                        topic_id: topic_id.map(ToString::to_string),
2015                                    };
2016                                    if let Err(e) = ch.send(&msg).await {
2017                                        tracing::warn!(
2018                                            channel = ch_name,
2019                                            error = %e,
2020                                            "failed to send restart confirmation"
2021                                        );
2022                                    }
2023                                    break;
2024                                }
2025                            }
2026                        }
2027                        if let Err(e) = store.kv_delete("control:restart_pending").await {
2028                            tracing::warn!(error = %e, "failed to clear control:restart_pending");
2029                        }
2030                    }
2031                    Ok(None) => {} // no pending restart, nothing to do
2032                    Err(e) => {
2033                        tracing::warn!(error = %e, "failed to read control:restart_pending");
2034                    }
2035                }
2036            });
2037        }
2038    }
2039
2040    // Spawn notification dispatcher — reads task completion notifications
2041    // from transport and broadcasts to configured channels.
2042    //
2043    // Routing priority:
2044    // 1. Dedicated channel: if the notification has a `repo` and the
2045    //    ChannelRouter maps it to a topic/channel, send there with `topic_id`.
2046    // 2. Subscribed channels: send to each (channel, thread_id) subscribed to
2047    //    the repo, using `format_with_project()` so recipients know the source.
2048    // 3. Fallback: if no repo or no dedicated channel, broadcast to all
2049    //    channels without a topic_id (legacy behaviour).
2050    {
2051        let mut notification_rx = transport.subscribe_notifications();
2052        let channels = Arc::clone(&channel_registry);
2053        let notif_router = Arc::clone(&channel_router);
2054        // Grab a store reference for subscription lookups.
2055        let notif_store: Option<Arc<TaskStore>> = Some(Arc::clone(&store));
2056        tokio::spawn(async move {
2057            loop {
2058                match notification_rx.recv().await {
2059                    Ok(notification) => {
2060                        let level = NotificationLevel::from_config();
2061                        if !level.should_notify(&notification.status) {
2062                            tracing::debug!(
2063                                task_id = %notification.task_id,
2064                                status = %notification.status,
2065                                "notification suppressed by level={:?}",
2066                                level
2067                            );
2068                            continue;
2069                        }
2070
2071                        tracing::info!(
2072                            task_id = %notification.task_id,
2073                            status = %notification.status,
2074                            repo = ?notification.repo,
2075                            "routing notification to channels"
2076                        );
2077
2078                        // Track whether we sent to at least one dedicated/subscribed target.
2079                        let mut routed = false;
2080
2081                        // 0. Direct target: if notify_target is set, send straight to
2082                        //    that Telegram chat_id and skip all other routing.
2083                        if let Some(ref target) = notification.notify_target {
2084                            let telegram = channels.iter().find(|c| c.name() == "telegram");
2085                            if let Some(channel) = telegram {
2086                                let body = notification.format_telegram();
2087                                let metadata = serde_json::json!({
2088                                    "preformatted_html": true,
2089                                    "chat_id_override": target
2090                                });
2091                                let msg = OutgoingMessage {
2092                                    thread_id: notification.task_id.clone(),
2093                                    body,
2094                                    reply_to: None,
2095                                    metadata,
2096                                    topic_id: None,
2097                                };
2098                                match channel.send(&msg).await {
2099                                    Err(e) => {
2100                                        tracing::warn!(
2101                                            task_id = %notification.task_id,
2102                                            target,
2103                                            ?e,
2104                                            "failed to send to notify_target"
2105                                        );
2106                                    }
2107                                    _ => {
2108                                        routed = true;
2109                                    }
2110                                }
2111                            }
2112                        }
2113
2114                        if routed {
2115                            // notify_target handled — skip further routing
2116                        } else if let Some(repo) = notification.repo.as_deref() {
2117                            // Track destinations already sent to in this iteration so
2118                            // the dedicated (path 1) and subscribed (path 2) loops
2119                            // don't double-fire the same Telegram/Discord destination
2120                            // when a repo's configured topic and a `channel_subscriptions`
2121                            // row resolve to the same `(channel, topic_id)`.
2122                            let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2123
2124                            // 1. Dedicated channel targets for this repo.
2125                            for channel in channels.iter() {
2126                                let ch_name = channel.name();
2127                                if let Some(topic_id) =
2128                                    notif_router.target_for_project(repo, ch_name)
2129                                {
2130                                    let body = match ch_name {
2131                                        "telegram" => notification.format_telegram(),
2132                                        "discord" => notification.format_discord(),
2133                                        "slack" => notification.format_slack(),
2134                                        _ => continue,
2135                                    };
2136                                    let metadata = if ch_name == "telegram" {
2137                                        serde_json::json!({"preformatted_html": true})
2138                                    } else {
2139                                        serde_json::json!({})
2140                                    };
2141                                    let resolved_topic = Some(topic_id.to_string());
2142                                    let msg = OutgoingMessage {
2143                                        thread_id: notification.task_id.clone(),
2144                                        body,
2145                                        reply_to: None,
2146                                        metadata,
2147                                        topic_id: resolved_topic.clone(),
2148                                    };
2149                                    if let Err(e) = channel.send(&msg).await {
2150                                        tracing::warn!(
2151                                            channel = ch_name,
2152                                            task_id = %notification.task_id,
2153                                            ?e,
2154                                            "failed to send to dedicated channel"
2155                                        );
2156                                    } else {
2157                                        sent.insert((ch_name.to_string(), resolved_topic));
2158                                        routed = true;
2159                                    }
2160                                }
2161                            }
2162
2163                            // 2. Subscribed channels for this repo.
2164                            if let Some(store) = &notif_store {
2165                                match store.list_subscribers_for_repo(repo).await {
2166                                    Ok(subscribers) => {
2167                                        for (ch_name, thread_id, topic_id) in subscribers {
2168                                            let channel =
2169                                                channels.iter().find(|c| c.name() == ch_name);
2170                                            let Some(channel) = channel else {
2171                                                continue;
2172                                            };
2173                                            let resolved_topic = if topic_id.is_empty() {
2174                                                None
2175                                            } else {
2176                                                Some(topic_id.clone())
2177                                            };
2178                                            // Skip if a dedicated send to the same
2179                                            // (channel, topic_id) already happened.
2180                                            if sent.contains(&(
2181                                                ch_name.clone(),
2182                                                resolved_topic.clone(),
2183                                            )) {
2184                                                tracing::debug!(
2185                                                    channel = %ch_name,
2186                                                    task_id = %notification.task_id,
2187                                                    topic_id = ?resolved_topic,
2188                                                    "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2189                                                );
2190                                                continue;
2191                                            }
2192                                            let body = notification.format_with_project(&ch_name);
2193                                            let metadata = if ch_name == "telegram" {
2194                                                serde_json::json!({"preformatted_html": true})
2195                                            } else {
2196                                                serde_json::json!({})
2197                                            };
2198                                            let msg = OutgoingMessage {
2199                                                thread_id: thread_id.clone(),
2200                                                body,
2201                                                reply_to: None,
2202                                                metadata,
2203                                                topic_id: resolved_topic.clone(),
2204                                            };
2205                                            if let Err(e) = channel.send(&msg).await {
2206                                                tracing::warn!(
2207                                                    channel = %ch_name,
2208                                                    task_id = %notification.task_id,
2209                                                    ?e,
2210                                                    "failed to send to subscribed channel"
2211                                                );
2212                                            } else {
2213                                                sent.insert((ch_name.clone(), resolved_topic));
2214                                                routed = true;
2215                                            }
2216                                        }
2217                                    }
2218                                    Err(e) => {
2219                                        tracing::warn!(
2220                                            repo,
2221                                            ?e,
2222                                            "failed to list subscribers for repo"
2223                                        );
2224                                    }
2225                                }
2226                            }
2227                        }
2228
2229                        // 3. Fallback: broadcast to all channels when no routing happened.
2230                        if !routed {
2231                            for channel in channels.iter() {
2232                                let (body, should_send, is_telegram) = match channel.name() {
2233                                    "telegram" => (notification.format_telegram(), true, true),
2234                                    "discord" => (notification.format_discord(), true, false),
2235                                    "slack" => (notification.format_slack(), true, false),
2236                                    // GitHub is already handled by backend.post_comment()
2237                                    // tmux doesn't need task completion notifications
2238                                    _ => (String::new(), false, false),
2239                                };
2240
2241                                if !should_send {
2242                                    continue;
2243                                }
2244
2245                                let metadata = if is_telegram {
2246                                    serde_json::json!({"preformatted_html": true})
2247                                } else {
2248                                    serde_json::json!({})
2249                                };
2250                                let msg = OutgoingMessage {
2251                                    thread_id: notification.task_id.clone(),
2252                                    body,
2253                                    reply_to: None,
2254                                    metadata,
2255                                    topic_id: None,
2256                                };
2257
2258                                if let Err(e) = channel.send(&msg).await {
2259                                    tracing::warn!(
2260                                        channel = channel.name(),
2261                                        task_id = %notification.task_id,
2262                                        ?e,
2263                                        "failed to send notification"
2264                                    );
2265                                }
2266                            }
2267                        }
2268                    }
2269                    Err(RecvError::Lagged(n)) => {
2270                        tracing::warn!(missed = n, "notification receiver lagged");
2271                    }
2272                    Err(RecvError::Closed) => {
2273                        tracing::debug!("notification channel closed");
2274                        break;
2275                    }
2276                }
2277            }
2278        });
2279        tracing::info!("notification dispatcher started");
2280    }
2281
2282    // Notify used by webhook events to wake up the engine tick immediately
2283    let webhook_notify = Arc::new(Notify::new());
2284
2285    // `WebhookStatus::fallback_mode` is only shown by `orch webhook status`, the sync
2286    // tick always runs every `engine.sync_interval` whether or not webhooks work
2287    let webhook_port: Option<u16>;
2288    let mut webhook_healthy: bool;
2289    let mut last_webhook_health_check = Instant::now();
2290    // Start webhook server if configured
2291    let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2292    // No readable secret keeps the webhook server off, it never accepts unsigned requests
2293    let webhook_secret = if webhook_enabled {
2294        secret_ref::config_secret("webhook.secret")
2295            .await
2296            .ok()
2297            .filter(|s| !s.trim().is_empty())
2298    } else {
2299        None
2300    };
2301    if webhook_enabled && webhook_secret.is_none() {
2302        tracing::error!(
2303            "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2304        );
2305    }
2306    let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2307    let webhook_secret = webhook_secret.unwrap_or_default();
2308
2309    // Shared status updated by the server-spawn task and the health-check loop.
2310    let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2311
2312    if webhook_enabled {
2313        // Spawn the HTTP server with exponential-backoff retry on transient
2314        // bind errors (e.g. EADDRINUSE).  After MAX_ATTEMPTS the engine falls
2315        // back to polling and updates the shared status accordingly.
2316        const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2317        let port: u16 = config::get("webhook.port")
2318            .ok()
2319            .and_then(|v| v.parse().ok())
2320            .unwrap_or(8080);
2321        webhook_port = Some(port);
2322        let secret = webhook_secret.clone();
2323        let webhook_repo = project_engines
2324            .first()
2325            .map(|e| e.repo.clone())
2326            .unwrap_or_default();
2327
2328        let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2329
2330        // Initialise status: configured, not yet healthy.
2331        let to_save = {
2332            let mut s = webhook_status.lock().await;
2333            s.configured = true;
2334            s.port = Some(port);
2335            s.healthy = false;
2336            s.fallback_mode = false;
2337            s.clone()
2338        };
2339        to_save.save().await;
2340
2341        let status_for_spawn = Arc::clone(&webhook_status);
2342        tokio::spawn(async move {
2343            let mut attempt = 0u32;
2344            loop {
2345                attempt += 1;
2346                tracing::info!(attempt, port, "attempting to start webhook server");
2347                match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2348                    .await
2349                {
2350                    Ok(()) => {
2351                        tracing::info!(attempt, "webhook server exited cleanly");
2352                        break;
2353                    }
2354                    Err(e) => {
2355                        let reason = e.to_string();
2356                        tracing::error!(attempt, %reason, "webhook server failed");
2357                        if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2358                            let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2359                                "max attempts reached"
2360                            } else {
2361                                "non-transient error"
2362                            };
2363                            tracing::error!(
2364                                attempt,
2365                                %reason,
2366                                kind,
2367                                orch_webhook_in_fallback = true,
2368                                "webhook server giving up, switching to polling fallback"
2369                            );
2370                            let to_save = {
2371                                let mut s = status_for_spawn.lock().await;
2372                                s.fallback_mode = true;
2373                                s.healthy = false;
2374                                s.last_failure_reason = Some(reason);
2375                                s.startup_attempts = attempt;
2376                                s.clone()
2377                            };
2378                            to_save.save().await;
2379                            break;
2380                        }
2381                        let delay = webhook_backoff_delay(attempt);
2382                        tracing::warn!(
2383                            attempt,
2384                            delay_ms = delay.as_millis(),
2385                            %reason,
2386                            "webhook bind failed (transient), retrying with backoff"
2387                        );
2388                        {
2389                            let to_save = {
2390                                let mut s = status_for_spawn.lock().await;
2391                                s.startup_attempts = attempt;
2392                                s.last_failure_reason = Some(reason);
2393                                s.clone()
2394                            };
2395                            to_save.save().await;
2396                        }
2397                        time::sleep(delay).await;
2398                    }
2399                }
2400            }
2401        });
2402
2403        // Spawn the message forwarding task (reads from webhook channel)
2404        let notify = Arc::clone(&webhook_notify);
2405        let transport_for_webhook = Arc::clone(&transport);
2406        tokio::spawn(async move {
2407            while let Some(msg) = rx.recv().await {
2408                tracing::info!(
2409                    channel = %msg.channel,
2410                    thread = %msg.thread_id,
2411                    event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2412                    action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2413                    "webhook event received, triggering immediate tick"
2414                );
2415                let route = transport_for_webhook.route(&msg).await;
2416                tracing::info!(
2417                    channel = %msg.channel,
2418                    route = ?route,
2419                    "webhook event routed"
2420                );
2421                // Wake up the engine tick immediately instead of waiting up to 10s
2422                notify.notify_one();
2423            }
2424        });
2425
2426        webhook_healthy = true;
2427        tracing::info!(port, orch_webhook_up = true, "webhook server started");
2428    } else {
2429        webhook_port = None;
2430        webhook_healthy = false;
2431        {
2432            let to_save = {
2433                let mut s = webhook_status.lock().await;
2434                s.configured = false;
2435                s.fallback_mode = true;
2436                s.clone()
2437            };
2438            to_save.save().await;
2439        }
2440        tracing::info!(
2441            orch_webhook_in_fallback = true,
2442            "webhook server disabled, using polling fallback mode"
2443        );
2444    }
2445
2446    start_relay_client(&store, &webhook_notify);
2447    app::start_loader();
2448
2449    // Agent router (selects agent + model per task) - shared across projects.
2450    // Router::from_config() may run `opencode models` when the config uses opencode.
2451    // spawn_blocking keeps the Tokio runtime thread free during that I/O
2452    let router = Arc::new(RwLock::new(
2453        match task::spawn_blocking(Router::from_config).await {
2454            Ok(r) => r,
2455            Err(e) => {
2456                tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2457                task::spawn_blocking(|| Router::new(RouterConfig::default()))
2458                    .await
2459                    .unwrap_or_else(|e2| {
2460                        panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2461                    })
2462            }
2463        },
2464    ));
2465    {
2466        let r = router.read().await;
2467        tracing::info!(
2468            mode = %r.config.mode,
2469            agents = ?r.available_agents,
2470            fallback = %r.config.fallback_executor,
2471            "router initialized"
2472        );
2473    }
2474
2475    // Concurrency limiter (shared across all projects). `parallel_in_force` is the
2476    // limit actually handed out, which lags `config.max_parallel` after a shrink
2477    // until the running tasks give their permits back.
2478    let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2479    let mut parallel_in_force = config.max_parallel;
2480    // Reviews get their own pool, one reviewer per worker, so a busy worker pool never
2481    // starves a review (#3971)
2482    let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2483    let mut review_in_force = config.max_parallel;
2484
2485    // In-memory dispatch guard: tracks task IDs currently being dispatched.
2486    // Guards against GitHub API eventual consistency — after update_status(InProgress),
2487    // the label-removal webhook can trigger an immediate tick where list_by_status(Routed)
2488    // still returns the task (search index propagation delay). The tmux session does not
2489    // exist until the runner completes worktree setup (~10s later), so session_exists
2490    // alone is insufficient. Keyed by "{repo}/{task_id}".
2491    let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2492
2493    // In-memory guard to prevent double-spawn of auto-merge background tasks.
2494    // Since review_open_prs runs every sync_tick (~10s) but CI polling can take
2495    // up to 10 minutes, we track which tasks already have an auto-merge in flight.
2496    let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2497
2498    // Subscribe to config file changes for hot reload
2499    let mut config_rx = config::subscribe();
2500
2501    // Track sync interval
2502    let mut last_sync = Instant::now();
2503
2504    // Track upgrade check interval
2505    let mut last_upgrade_check: Option<Instant> = None;
2506
2507    // Reset epoch of the GitHub rate-limit window already reported, so the log line,
2508    // the KV flag and the channel notification fire once per window instead of per tick
2509    let mut rate_limit_reported: Option<i64> = None;
2510
2511    // Channel for weight signals from task runners back to the router
2512    let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2513
2514    // A restart ends every review session, so reset each in_review task left without one
2515    for engine in project_engines.iter().filter(|_| leads) {
2516        let reset =
2517            reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2518
2519        if reset > 0 {
2520            tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2521        }
2522    }
2523
2524    // Spawn dispatch subscribers — react to Routed events immediately
2525    for engine in &project_engines {
2526        subscribers::dispatch::spawn(
2527            event_bus.subscribe(),
2528            Arc::clone(&engine.backend),
2529            Arc::clone(&tmux),
2530            Arc::clone(&engine.runner),
2531            Arc::clone(&capture_for_tick),
2532            Arc::clone(&semaphore),
2533            Arc::clone(&engine.task_manager),
2534            weight_tx.clone(),
2535            Arc::clone(&router),
2536            Arc::clone(&dispatching),
2537            Arc::clone(&engine.store),
2538            engine.repo.clone(),
2539        );
2540    }
2541    tracing::info!("dispatch subscriber started");
2542
2543    // Spawn review subscribers — react to NeedsReview events immediately
2544    for engine in &project_engines {
2545        subscribers::review::spawn(
2546            event_bus.subscribe(),
2547            Arc::clone(&engine.backend),
2548            Arc::clone(&tmux),
2549            Arc::clone(&review_semaphore),
2550            Arc::clone(&engine.task_manager),
2551            Arc::clone(&router),
2552            Arc::clone(&dispatching),
2553            Arc::clone(&engine.store),
2554            engine.repo.clone(),
2555        );
2556    }
2557    tracing::info!("review subscriber started");
2558
2559    // Spawn unblock subscribers — react to Done events, unblock parent tasks immediately
2560    for engine in &project_engines {
2561        subscribers::unblock::spawn(
2562            event_bus.subscribe(),
2563            Arc::clone(&engine.backend),
2564            Arc::clone(&engine.task_manager),
2565            Arc::clone(&engine.store),
2566            engine.repo.clone(),
2567        );
2568    }
2569    tracing::info!("unblock subscriber started");
2570
2571    // Spawn notify subscriber — reacts to ALL events, pushes to transport.
2572    // Spawned once (not per-project) since the transport handles all repos.
2573    subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2574    tracing::info!("notify subscriber started");
2575
2576    // Main loop
2577    tracing::info!(
2578        tick = ?config.tick_interval,
2579        sync = ?config.sync_interval,
2580        parallel = config.max_parallel,
2581        "entering main loop"
2582    );
2583    // Values of the settings that only take effect at startup, so a reload can
2584    // tell the operator their edit needs a restart.
2585    let mut restart_only = restart_only_snapshot();
2586
2587    let mut interval = time::interval(config.tick_interval);
2588
2589    // Signal handlers (launchd/systemd send SIGTERM to stop services, SIGHUP for reload/restart)
2590    let mut sigterm = unix::signal(SignalKind::terminate())?;
2591    let mut sighup = unix::signal(SignalKind::hangup())?;
2592
2593    // Guard to prevent concurrent sync_tick runs (sync is spawned as a background task).
2594    let sync_in_progress = Arc::new(AtomicBool::new(false));
2595
2596    // Tick watchdog: detects when the main loop hasn't completed a tick in > 60s.
2597    // Updates an atomic timestamp at the end of each tick iteration.
2598    let last_tick_epoch = Arc::new(AtomicU64::new(
2599        SystemTime::now()
2600            .duration_since(std_time::UNIX_EPOCH)
2601            .unwrap_or_default()
2602            .as_secs(),
2603    ));
2604    {
2605        let last_tick_epoch = Arc::clone(&last_tick_epoch);
2606        let tick_interval_secs = config.tick_interval.as_secs();
2607        tokio::spawn(async move {
2608            // Check every 30s whether the main loop is still ticking.
2609            let mut watchdog_interval = time::interval(Duration::from_secs(30));
2610            // Own suspend baseline, independent of the main loop's checkpoint(). The
2611            // watchdog's interval timer also freezes during a host suspend, so this
2612            // detects the gap from its own wake-up instead of racing the main loop's
2613            // checkpoint() for the same GAP_LOG entry (see #3544).
2614            let mut suspend_detector = SuspendDetector::new();
2615            loop {
2616                watchdog_interval.tick().await;
2617                let own_gap = suspend_detector.check();
2618                let last = last_tick_epoch.load(Ordering::Relaxed);
2619                let now = SystemTime::now()
2620                    .duration_since(std_time::UNIX_EPOCH)
2621                    .unwrap_or_default()
2622                    .as_secs();
2623                let stale_secs = now.saturating_sub(last);
2624                // Warn if the tick loop hasn't completed in > 6× the tick interval (at least 60s).
2625                let threshold = (tick_interval_secs * 6).max(60);
2626                if stale_secs > threshold {
2627                    // A host suspend/resume gap fully explains a stale window that
2628                    // isn't an actual stall — the process (and its timers) were
2629                    // simply not scheduled while the host was asleep. Prefer this
2630                    // watchdog's own detector (immune to the checkpoint race above);
2631                    // fall back to the shared log in case the main loop's checkpoint()
2632                    // already ran and recorded a larger gap.
2633                    let shared_gap =
2634                        engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2635                    let gap = match (own_gap, shared_gap) {
2636                        (Some(a), Some(b)) => Some(a.max(b)),
2637                        (Some(a), None) => Some(a),
2638                        (None, Some(b)) => Some(b),
2639                        (None, None) => None,
2640                    };
2641                    if let Some(gap) = gap {
2642                        let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2643                        if gap_secs >= stale_secs {
2644                            tracing::info!(
2645                                stale_secs,
2646                                threshold,
2647                                suspend_gap_secs = gap_secs,
2648                                "tick loop stale window explained by host suspend/resume, not a stall"
2649                            );
2650                        } else {
2651                            tracing::error!(
2652                                stale_secs,
2653                                threshold,
2654                                suspend_gap_secs = gap_secs,
2655                                "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2656                                stale_secs,
2657                                threshold,
2658                                gap_secs,
2659                            );
2660                        }
2661                    } else {
2662                        tracing::error!(
2663                            stale_secs,
2664                            threshold,
2665                            "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2666                            stale_secs,
2667                            threshold,
2668                        );
2669                    }
2670                }
2671            }
2672        });
2673    }
2674
2675    loop {
2676        tokio::select! {
2677            _ = interval.tick() => {
2678                let tick_start = Instant::now();
2679
2680                // Detect a host suspend/resume gap once per iteration, in one place.
2681                // The watchdog and stuck-task recovery both read from this instead of
2682                // re-deriving suspend information from their own wall-clock deltas.
2683                if let Some(gap) = engine_suspend::checkpoint() {
2684                    tracing::info!(
2685                        gap_secs = gap.num_seconds(),
2686                        "host suspend/resume gap detected, discounting from stuck-task age checks"
2687                    );
2688                }
2689
2690                // Finish a pending shrink of the dispatch pool as running tasks
2691                // release their permits. Only logs when a permit was actually
2692                // taken back, so a long-running task does not log every tick.
2693                if review_in_force != config.max_parallel {
2694                    review_in_force =
2695                        resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2696                }
2697                if parallel_in_force != config.max_parallel {
2698                    let resized =
2699                        resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2700                    if resized != parallel_in_force {
2701                        parallel_in_force = resized;
2702                        tracing::info!(
2703                            in_force = parallel_in_force,
2704                            target = config.max_parallel,
2705                            "dispatch pool resized"
2706                        );
2707                    }
2708                }
2709
2710                // Tick weight recovery for rate-limited agents
2711                {
2712                    let mut rw = router.write().await;
2713                    rw.tick_weight_recovery();
2714                }
2715
2716                // Report the GitHub rate limit once per window, then tick anyway. Calls
2717                // into GitHub now fail fast instead of sleeping until the reset, so the
2718                // local phases keep moving and the watchdog does not see a stalled loop.
2719                if let Some(remaining) = GhHttp::is_rate_limited() {
2720                    let reset_at = chrono::Utc::now()
2721                        .timestamp()
2722                        .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2723                    // Tolerate jitter in the header so one window reports once
2724                    if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2725                        rate_limit_reported = Some(reset_at);
2726                        report_rate_limited(reset_at, Some(&store), Some(&transport)).await;
2727                    }
2728                } else if let Some(prev) = rate_limit_reported.take() {
2729                    tracing::info!(
2730                        limited_until = prev,
2731                        "GitHub API rate limit cleared"
2732                    );
2733                    let _ = store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2734                }
2735
2736                // Core tick: poll tasks for all projects
2737                tick::tick_all(
2738                    &project_engines,
2739                    &tmux,
2740                    &capture_for_tick,
2741                    &semaphore,
2742                    &config,
2743                    &router,
2744                    &weight_tx,
2745                    &dispatching,
2746                    Some(&transport),
2747                    &active_repos,
2748                    &auto_merge_in_flight,
2749                ).await;
2750
2751                // Drain pending weight signals after tick processing so the router
2752                // learns from outcomes produced by this iteration's task dispatches.
2753                while let Ok(signal) = weight_rx.try_recv() {
2754                    let mut rw = router.write().await;
2755                    match signal {
2756                        WeightSignal::RateLimited { ref agent } => {
2757                            rw.record_rate_limit(agent);
2758                        }
2759                        WeightSignal::Success { ref agent } => {
2760                            rw.record_success(agent);
2761                        }
2762                        WeightSignal::Failed { .. }
2763                        | WeightSignal::Rerouted
2764                        | WeightSignal::Blocked
2765                        | WeightSignal::None => {}
2766                    }
2767                }
2768
2769                // Periodic sync (less frequent) — spawned as a background task so it
2770                // cannot block the main tick loop. A stalled sync (hung HTTP call, slow
2771                // DB query, lock contention) previously parked all tokio workers for 12+
2772                // minutes because it ran inline. See issue #2574.
2773                if last_sync.elapsed() >= config.sync_interval {
2774                    if sync_in_progress.compare_exchange(
2775                        false,
2776                        true,
2777                        Ordering::AcqRel,
2778                        Ordering::Relaxed,
2779                    ).is_ok() {
2780                        spawn_sync_pass(
2781                            &project_engines,
2782                            &tmux,
2783                            &config,
2784                            &router,
2785                            &dispatching,
2786                            &auto_merge_in_flight,
2787                            &sync_in_progress,
2788                            "periodic",
2789                        );
2790                    } else {
2791                        tracing::debug!("sync tick still in progress, skipping");
2792                    }
2793                    // Records sync *schedule* time (not completion). This prevents
2794                    // re-entering the sync branch every tick while one is in flight.
2795                    // The sync_in_progress guard handles actual concurrency control.
2796                    last_sync = Instant::now();
2797                }
2798
2799                // Periodic webhook health check
2800                if webhook_enabled {
2801                    let health_check_interval = config.webhook_health_check_interval
2802                        .unwrap_or(Duration::from_secs(60));
2803
2804                    if last_webhook_health_check.elapsed() >= health_check_interval {
2805                        if let Some(port) = webhook_port {
2806                            let (health, failure_reason) =
2807                                github::check_webhook_health(port).await;
2808                            if health != webhook_healthy {
2809                                webhook_healthy = health;
2810                                if webhook_healthy {
2811                                    tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2812                                } else {
2813                                    tracing::warn!(
2814                                        port,
2815                                        orch_webhook_up = false,
2816                                        reason = failure_reason.as_deref().unwrap_or("unknown"),
2817                                        "webhook health check failed"
2818                                    );
2819                                }
2820                            }
2821                            // Persist updated status.
2822                            {
2823                                let to_save = {
2824                                    let mut s = webhook_status.lock().await;
2825                                    s.healthy = health;
2826                                    s.last_check_utc = Some(chrono::Utc::now());
2827                                    if health {
2828                                        s.last_failure_reason = None;
2829                                    } else if failure_reason.is_some() {
2830                                        s.last_failure_reason = failure_reason;
2831                                    }
2832                                    s.clone()
2833                                };
2834                                to_save.save().await;
2835                            }
2836                        }
2837                        last_webhook_health_check = Instant::now();
2838                    }
2839                }
2840
2841                // Periodic upgrade check — notify channels when a newer release is available.
2842                if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2843                    let store = Arc::clone(&store);
2844                    let channels = Arc::clone(&channel_registry);
2845                    let current = env!("ORCH_VERSION").to_string();
2846                    let check_interval = Duration::from_secs(config.upgrade_check_interval);
2847                    tokio::spawn(async move {
2848                        check_and_notify_upgrade(&store, &channels, &current, check_interval)
2849                            .await;
2850                    });
2851                    last_upgrade_check = Some(Instant::now());
2852                }
2853
2854                // Update watchdog timestamp + log tick duration.
2855                let tick_elapsed = tick_start.elapsed();
2856                if tick_elapsed.as_secs() > 30 {
2857                    tracing::warn!(
2858                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2859                        "slow tick"
2860                    );
2861                } else {
2862                    tracing::debug!(
2863                        elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2864                        "tick complete"
2865                    );
2866                }
2867                last_tick_epoch.store(
2868                    SystemTime::now()
2869                        .duration_since(std_time::UNIX_EPOCH)
2870                        .unwrap_or_default()
2871                        .as_secs(),
2872                    Ordering::Relaxed,
2873                );
2874            }
2875            // Webhook events trigger an immediate tick (bypass polling interval)
2876            () = webhook_notify.notified() => {
2877                if let Some(remaining) = GhHttp::is_rate_limited() {
2878                    tracing::warn!(
2879                        remaining_secs = remaining.as_secs(),
2880                        "GitHub API rate-limited, skipping webhook-triggered tick"
2881                    );
2882                } else {
2883                    tracing::info!("webhook event triggered immediate tick");
2884
2885                    tick::tick_all(
2886                        &project_engines,
2887                        &tmux,
2888                        &capture_for_tick,
2889                        &semaphore,
2890                        &config,
2891                        &router,
2892                        &weight_tx,
2893                        &dispatching,
2894                        Some(&transport),
2895                        &active_repos,
2896                        &auto_merge_in_flight,
2897                    ).await;
2898
2899                // Drain pending weight signals after webhook-triggered tick processing.
2900                while let Ok(signal) = weight_rx.try_recv() {
2901                    let mut rw = router.write().await;
2902                    match signal {
2903                        WeightSignal::RateLimited { ref agent } => {
2904                            rw.record_rate_limit(agent);
2905                        }
2906                        WeightSignal::Success { ref agent } => {
2907                            rw.record_success(agent);
2908                        }
2909                        WeightSignal::Failed { .. }
2910                        | WeightSignal::Rerouted
2911                        | WeightSignal::Blocked
2912                        | WeightSignal::None => {}
2913                    }
2914                }
2915
2916                // Also attempt to run an immediate sync (ingest/cleanup) in the
2917                // background so webhook-driven events (e.g. new issues) are
2918                // picked up without waiting for the periodic sync interval.
2919                // Reuse the same sync_in_progress guard used by the periodic
2920                // branch to avoid concurrent syncs.
2921                if sync_in_progress.compare_exchange(
2922                    false,
2923                    true,
2924                    Ordering::AcqRel,
2925                    Ordering::Relaxed,
2926                ).is_ok() {
2927                    spawn_sync_pass(
2928                        &project_engines,
2929                        &tmux,
2930                        &config,
2931                        &router,
2932                        &dispatching,
2933                        &auto_merge_in_flight,
2934                        &sync_in_progress,
2935                        "webhook",
2936                    );
2937                    // Record schedule time so periodic sync doesn't immediately re-run.
2938                    last_sync = Instant::now();
2939                } else {
2940                    tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2941                }
2942                }
2943
2944                // Also reset the interval so we don't get a redundant tick right after
2945                interval.reset();
2946            }
2947            result = config_rx.recv() => {
2948                // A notification only arrives for a file that parsed, so there is no
2949                // broken-config case to guard against here (see `config::reload_file`).
2950                let reload = match result {
2951                    Ok(path) => {
2952                        tracing::info!(path = %path.display(), "config file changed, reloading");
2953                        true
2954                    }
2955                    Err(RecvError::Lagged(n)) => {
2956                        tracing::warn!(missed = n, "config change receiver lagged, reloading");
2957                        true
2958                    }
2959                    Err(RecvError::Closed) => {
2960                        tracing::warn!("config change channel closed");
2961                        false
2962                    }
2963                };
2964
2965                if reload {
2966                    // Reload engine config
2967                    let new_config = EngineConfig::from_config();
2968                    log_engine_config_changes(&config, &new_config);
2969
2970                    let new_restart_only = restart_only_snapshot();
2971                    log_restart_only_changes(&restart_only, &new_restart_only);
2972                    restart_only = new_restart_only;
2973
2974                    let tick_changed = new_config.tick_interval != config.tick_interval;
2975                    config = new_config;
2976
2977                    // Reset tick interval if it changed
2978                    if tick_changed {
2979                        interval = time::interval(config.tick_interval);
2980                        tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2981                    }
2982
2983                    // Resize the dispatch pool. A shrink only takes the permits that are
2984                    // free right now; the rest are taken as running tasks finish.
2985                    if review_in_force != config.max_parallel {
2986                        review_in_force = resize_dispatch_pool(
2987                            &review_semaphore,
2988                            review_in_force,
2989                            config.max_parallel,
2990                        );
2991                    }
2992                    if parallel_in_force != config.max_parallel {
2993                        parallel_in_force = resize_dispatch_pool(
2994                            &semaphore,
2995                            parallel_in_force,
2996                            config.max_parallel,
2997                        );
2998                        tracing::info!(
2999                            in_force = parallel_in_force,
3000                            target = config.max_parallel,
3001                            "dispatch pool resized"
3002                        );
3003                    }
3004
3005                    // Reload router config (async-safe: use reload_async to avoid blocking the runtime)
3006                    {
3007                        let mut router_guard = router.write().await;
3008                        router_guard.reload_async().await;
3009                    }
3010
3011                    // Pick up projects added to `projects:`
3012                    sync_project_engines(
3013                        &store,
3014                        &mut project_engines,
3015                        &mut active_repos,
3016                        &event_bus,
3017                        &tmux,
3018                        &capture_for_tick,
3019                        &semaphore,
3020                        &review_semaphore,
3021                        &weight_tx,
3022                        &router,
3023                        &dispatching,
3024                    ).await;
3025
3026                    tracing::info!(
3027                        tick = ?config.tick_interval,
3028                        sync = ?config.sync_interval,
3029                        parallel = config.max_parallel,
3030                        projects = project_engines.len(),
3031                        "config reloaded"
3032                    );
3033                }
3034            }
3035            signal_name = async {
3036                tokio::select! {
3037                    _ = signal::ctrl_c() => "SIGINT",
3038                    _ = sigterm.recv() => "SIGTERM",
3039                    _ = sighup.recv() => "SIGHUP",
3040                }
3041            } => {
3042                tracing::info!(signal = signal_name, "beginning graceful shutdown");
3043
3044                // Wrap the entire shutdown sequence in a timeout to enforce the configured limit.
3045                // This prevents indefinite blocking if status updates or tmux operations hang.
3046                let shutdown_result = time::timeout(
3047                    config.graceful_shutdown_timeout,
3048                    async {
3049                        // Reset in_progress tasks to routed so they re-dispatch after restart.
3050                        // Also reset in_review tasks to needs_review — their review agent
3051                        // tmux sessions will be killed when the process exits.
3052                        let mut reset_count = 0u32;
3053                        let mut review_reset_count = 0u32;
3054                        for engine in &project_engines {
3055                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
3056                                for task in &tasks {
3057                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
3058                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
3059                                    } else {
3060                                        if let Ok(Some(store_id)) =
3061                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3062                                            && let Err(e) = engine
3063                                                .store
3064                                                .finalize_incomplete_runs(
3065                                                    store_id,
3066                                                    "aborted",
3067                                                    "graceful shutdown: reset in_progress task to routed",
3068                                                )
3069                                                .await
3070                                            {
3071                                                tracing::warn!(
3072                                                    task_id = task.id.0,
3073                                                    ?e,
3074                                                    "failed to finalize incomplete runs on shutdown"
3075                                                );
3076                                            }
3077                                        reset_count += 1;
3078                                    }
3079                                }
3080                            }
3081                            // Also reset internal in_progress tasks
3082                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
3083                                for task in &tasks {
3084                                    let task_id = format!("internal:{}", task.id);
3085                                    if let Err(e) = engine.task_manager.update_task_status(
3086                                        &ExternalId(task_id.clone()),
3087                                        Status::Routed,
3088                                    ).await {
3089                                        tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3090                                    } else {
3091                                        if let Err(e) = engine
3092                                            .store
3093                                            .finalize_incomplete_runs(
3094                                                task.id,
3095                                                "aborted",
3096                                                "graceful shutdown: reset in_progress task to routed",
3097                                            )
3098                                            .await
3099                                        {
3100                                            tracing::warn!(
3101                                                task_id,
3102                                                ?e,
3103                                                "failed to finalize incomplete internal runs on shutdown"
3104                                            );
3105                                        }
3106                                        reset_count += 1;
3107                                    }
3108                                }
3109                            }
3110                            // Reset in_review tasks — review agent sessions die on shutdown
3111                            if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3112                                for task in &tasks {
3113                                    if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3114                                        tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3115                                    } else {
3116                                        if let Ok(Some(store_id)) =
3117                                            engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3118                                            && let Err(e) = engine
3119                                                .store
3120                                                .finalize_incomplete_runs(
3121                                                    store_id,
3122                                                    "aborted",
3123                                                    "graceful shutdown: reset in_review task to needs_review",
3124                                                )
3125                                                .await
3126                                            {
3127                                                tracing::warn!(
3128                                                    task_id = task.id.0,
3129                                                    ?e,
3130                                                    "failed to finalize incomplete review runs on shutdown"
3131                                                );
3132                                            }
3133                                        review_reset_count += 1;
3134                                    }
3135                                }
3136                            }
3137                            // Also reset internal in_review tasks
3138                            if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3139                                for task in &tasks {
3140                                    let task_id = format!("internal:{}", task.id);
3141                                    if let Err(e) = engine.task_manager.update_task_status(
3142                                        &ExternalId(task_id.clone()),
3143                                        Status::NeedsReview,
3144                                    ).await {
3145                                        tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3146                                    } else {
3147                                        if let Err(e) = engine
3148                                            .store
3149                                            .finalize_incomplete_runs(
3150                                                task.id,
3151                                                "aborted",
3152                                                "graceful shutdown: reset in_review task to needs_review",
3153                                            )
3154                                            .await
3155                                        {
3156                                            tracing::warn!(
3157                                                task_id,
3158                                                ?e,
3159                                                "failed to finalize incomplete internal review runs on shutdown"
3160                                            );
3161                                        }
3162                                        review_reset_count += 1;
3163                                    }
3164                                }
3165                            }
3166                        }
3167                        if reset_count > 0 {
3168                            tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3169                        }
3170                        if review_reset_count > 0 {
3171                            tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3172                        }
3173
3174                        // Kill all orch-managed tmux sessions so stale sessions don't
3175                        // block dispatch after restart (session_exists check).
3176                        let mut killed_sessions = Vec::new();
3177                        if let Ok(sessions) = tmux.list_sessions().await {
3178                            for session in &sessions {
3179                                if session.name.starts_with("orch-") {
3180                                    match tmux.kill_session(&session.name).await { Err(e) => {
3181                                        tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3182                                    } _ => {
3183                                        killed_sessions.push(session.name.clone());
3184                                    }}
3185                                }
3186                            }
3187                            if !killed_sessions.is_empty() {
3188                                tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3189                            }
3190                        }
3191
3192                        // Wait for sessions to actually die before exiting.
3193                        // Prevents race where process exits before tmux finishes cleanup.
3194                        if !killed_sessions.is_empty() {
3195                            let still_alive = tmux
3196                                .wait_for_sessions_dead(
3197                                    &killed_sessions,
3198                                    Duration::from_millis(100),
3199                                    Duration::from_secs(5),
3200                                )
3201                                .await;
3202                            if still_alive > 0 {
3203                                tracing::warn!(
3204                                    still_alive,
3205                                    "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3206                                );
3207                            }
3208                            tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3209                        }
3210                    },
3211                ).await;
3212
3213                if shutdown_result.is_err() {
3214                    tracing::error!(
3215                        timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3216                        "graceful shutdown timed out, forcing exit"
3217                    );
3218                }
3219
3220                break;
3221            }
3222        }
3223    }
3224
3225    // Clean up event bus port file and service version file
3226    events::cleanup_port_file().await;
3227    local::cleanup().await;
3228    ui::cleanup().await;
3229    events::cleanup_version_file();
3230
3231    // transport and channels drop here at end of scope
3232    let _ = transport;
3233    tracing::info!("orch engine stopped");
3234    Ok(())
3235}
3236
3237#[cfg(test)]
3238mod tests {
3239    use std::env;
3240    use std::process;
3241    use std::slice;
3242    use std::sync::Arc;
3243
3244    use super::*;
3245    use crate::backends::ExternalBackend;
3246    use crate::backends::test_helpers::NoopBackend;
3247    use crate::channels::capture::CaptureService;
3248    use crate::channels::transport::Transport;
3249    use crate::engine::runner::git_ops;
3250    use crate::store::UpsertExternal;
3251    use crate::tmux::SessionGuard;
3252    use crate::tmux::TmuxManager;
3253
3254    async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3255        let id = store
3256            .upsert_external(&UpsertExternal {
3257                repo: "owner/repo",
3258                ext_id: number,
3259                title: "Under review",
3260                body: "",
3261                author: "",
3262                url: "",
3263                labels: &[],
3264                origin: "github",
3265            })
3266            .await
3267            .unwrap();
3268        store.update_status(id, TaskStatus::InReview).await.unwrap();
3269        id
3270    }
3271
3272    #[tokio::test]
3273    async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3274        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3275        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3276        let task_manager =
3277            TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3278        let tmux = TmuxManager::isolated();
3279        let orphaned = in_review_task(&store, "11").await;
3280        let live = in_review_task(&store, "12").await;
3281        let session = tmux
3282            .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3283            .await
3284            .unwrap();
3285        let _guard = SessionGuard(session);
3286
3287        let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3288
3289        assert_eq!(reset, 1);
3290        assert_eq!(
3291            store.get(orphaned).await.unwrap().status,
3292            TaskStatus::NeedsReview
3293        );
3294        assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3295    }
3296
3297    #[test]
3298    fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3299        let semaphore = Arc::new(Semaphore::new(4));
3300        let held: Vec<_> = (0..4)
3301            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3302            .collect();
3303        assert!(
3304            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3305            "pool is full"
3306        );
3307
3308        let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3309        assert_eq!(in_force, 8);
3310
3311        let grown: Vec<_> = (0..4)
3312            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3313            .collect();
3314        assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3315        assert!(
3316            Arc::clone(&semaphore).try_acquire_owned().is_err(),
3317            "pool is full"
3318        );
3319    }
3320
3321    #[test]
3322    fn shrinking_max_parallel_never_cancels_a_running_task() {
3323        let semaphore = Arc::new(Semaphore::new(8));
3324        let mut held: Vec<_> = (0..8)
3325            .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3326            .collect();
3327
3328        // Nothing is free, so all 8 tasks keep their permits and keep running.
3329        let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3330        assert_eq!(in_force, 8);
3331        assert_eq!(held.len(), 8);
3332
3333        // Three finish. Their permits are taken back instead of handed out again.
3334        held.truncate(5);
3335        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3336        assert_eq!(in_force, 5);
3337        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3338
3339        // One more finishes and the pool settles at the new limit.
3340        held.truncate(4);
3341        in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3342        assert_eq!(in_force, 4);
3343        assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3344
3345        // A slot freed after that is handed out, since 4 is now the real limit.
3346        held.truncate(3);
3347        assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3348    }
3349
3350    #[test]
3351    fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3352        let configured = vec![
3353            ("owner/one".to_string(), PathBuf::from("/srv/one")),
3354            ("owner/two".to_string(), PathBuf::from("/srv/two")),
3355        ];
3356        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3357
3358        let (added, removed) = diff_configured_projects(&configured, &active);
3359        assert_eq!(
3360            added,
3361            vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3362            "only the new project is started, so owner/one keeps its single set of subscribers"
3363        );
3364        assert!(removed.is_empty(), "{removed:?}");
3365    }
3366
3367    #[test]
3368    fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3369        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3370        let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3371            .into_iter()
3372            .collect();
3373
3374        let (added, removed) = diff_configured_projects(&configured, &active);
3375        assert!(added.is_empty(), "nothing new to start");
3376        assert_eq!(removed, vec!["owner/gone".to_string()]);
3377    }
3378
3379    #[test]
3380    fn an_unchanged_project_list_starts_nothing() {
3381        let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3382        let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3383
3384        assert_eq!(
3385            diff_configured_projects(&configured, &active),
3386            (vec![], vec![])
3387        );
3388    }
3389
3390    #[test]
3391    fn restart_only_keys_are_the_settings_read_once_at_startup() {
3392        // Guards the docs table: every key listed here is documented as needing
3393        // `orch service restart`, and nothing reloadable may be added to it.
3394        assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3395    }
3396
3397    #[test]
3398    fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3399        let before = vec![Some("bot_token: a\n".to_string()), None];
3400
3401        assert!(
3402            log_restart_only_changes(&before, &before.clone()).is_empty(),
3403            "a reload that did not touch these keys must stay quiet"
3404        );
3405
3406        // An edited token and a newly added webhook block are both changes
3407        let after = vec![
3408            Some("bot_token: b\n".to_string()),
3409            Some("port: 8080\n".to_string()),
3410        ];
3411        assert_eq!(
3412            log_restart_only_changes(&before, &after),
3413            ["channels", "webhook"]
3414        );
3415
3416        // Only the edited key is reported when the other is untouched
3417        let token_only = vec![Some("bot_token: b\n".to_string()), None];
3418        assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3419    }
3420
3421    #[test]
3422    fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3423        let semaphore = Arc::new(Semaphore::new(4));
3424        assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3425        assert_eq!(semaphore.available_permits(), 4);
3426    }
3427
3428    #[test]
3429    fn live_session_stuck_limit_follows_the_run_timeout() {
3430        let config = EngineConfig {
3431            stuck_timeout: 1800,
3432            run_timeouts: RunTimeouts {
3433                base: 1800,
3434                medium: 2700,
3435                complex: 3600,
3436            },
3437            ..EngineConfig::default()
3438        };
3439        assert_eq!(
3440            config.live_session_stuck_limit(Some("simple")),
3441            1800 + RUN_TIMEOUT_GRACE_SECS
3442        );
3443        assert_eq!(
3444            config.live_session_stuck_limit(Some("medium")),
3445            2700 + RUN_TIMEOUT_GRACE_SECS
3446        );
3447        assert_eq!(
3448            config.live_session_stuck_limit(Some("complex")),
3449            3600 + RUN_TIMEOUT_GRACE_SECS
3450        );
3451
3452        let long_stuck = EngineConfig {
3453            stuck_timeout: 7200,
3454            ..config
3455        };
3456        assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3457    }
3458
3459    #[test]
3460    fn engine_config_defaults() {
3461        let config = EngineConfig::default();
3462        assert_eq!(config.tick_interval, Duration::from_secs(10));
3463        assert_eq!(config.sync_interval, Duration::from_secs(45));
3464        assert_eq!(config.max_parallel, 4);
3465        assert_eq!(config.stuck_timeout, 1800);
3466        assert_eq!(config.no_session_stuck_timeout, 600);
3467        assert!(
3468            config.no_session_stuck_timeout < config.stuck_timeout,
3469            "no_session_stuck_timeout must be shorter than stuck_timeout"
3470        );
3471        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3472        assert_eq!(config.upgrade_check_interval, 3600);
3473    }
3474
3475    #[test]
3476    fn engine_config_from_config_uses_defaults_when_no_config() {
3477        let config = EngineConfig::from_config();
3478        assert_eq!(config.tick_interval, Duration::from_secs(10));
3479        assert_eq!(config.sync_interval, Duration::from_secs(45));
3480        // max_parallel may be overridden by user config; just check it's reasonable
3481        assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3482        assert_eq!(config.stuck_timeout, 1800);
3483        assert_eq!(config.no_session_stuck_timeout, 600);
3484        assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3485        assert_eq!(config.upgrade_check_interval, 3600);
3486    }
3487
3488    #[test]
3489    fn engine_config_no_session_timeout_is_under_15_minutes() {
3490        let config = EngineConfig::default();
3491        assert!(
3492            config.no_session_stuck_timeout <= 900,
3493            "no_session_stuck_timeout ({}) exceeds 15 minutes",
3494            config.no_session_stuck_timeout
3495        );
3496    }
3497
3498    #[test]
3499    fn upgrade_check_runs_immediately_when_never_checked() {
3500        assert!(should_run_upgrade_check(None, 3600));
3501    }
3502
3503    #[test]
3504    fn upgrade_check_waits_for_interval_after_first_run() {
3505        assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3506    }
3507
3508    #[test]
3509    fn upgrade_check_can_be_disabled() {
3510        assert!(!should_run_upgrade_check(None, 0));
3511    }
3512
3513    #[test]
3514    fn configured_agents_parses_yaml_block_array() {
3515        // config::get returns serde_norway::to_string for arrays, which is YAML block format
3516        let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3517        let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3518        assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3519    }
3520
3521    #[test]
3522    fn configured_agents_parses_json_inline_array() {
3523        let json_str = r#"["claude","codex","olm"]"#;
3524        let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3525        assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3526    }
3527
3528    #[test]
3529    fn configured_agents_fallback_returns_default_agents() {
3530        // When no config is present, configured_agents() returns DEFAULT_AGENTS
3531        let defaults = configured_agents();
3532        assert!(defaults.contains(&"claude".to_string()));
3533        assert!(defaults.contains(&"codex".to_string()));
3534        assert!(defaults.contains(&"opencode".to_string()));
3535    }
3536
3537    /// Integration test that requires a fully working tmux environment with
3538    /// reliable capture-pane timing. Runs locally but not in CI (GitHub Actions
3539    /// has tmux installed but capture-pane timing is unreliable).
3540    ///
3541    /// DO NOT REMOVE #[ignore] — this test was un-ignored in PR #608 which
3542    /// broke CI. The `tmux -V` skip check is insufficient because tmux CAN
3543    /// create sessions on CI runners, but capture-pane output is empty or
3544    /// delayed, causing the assertion to fail after a 10s timeout.
3545    #[tokio::test]
3546    #[ignore = "capture-pane timing is unreliable on CI runners"]
3547    async fn integration_channel_to_tmux_to_capture() {
3548        // Create transport and capture service
3549        let transport = Arc::new(Transport::new());
3550        let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3551        let tmux = TmuxManager::new();
3552
3553        let task_id = "testtask1".to_string();
3554        let session_name = format!("orch-test-{task_id}");
3555
3556        // Start a detached tmux session
3557        let _ = tmux
3558            .command()
3559            .args(["new-session", "-d", "-s", &session_name])
3560            .output()
3561            .await
3562            .expect("failed to start tmux session");
3563
3564        // Guard ensures session cleanup even on panic
3565        let _guard = SessionGuard(session_name.clone());
3566
3567        // Register session with capture service and transport binding
3568        capture
3569            .register_session("owner/repo", &task_id, &session_name)
3570            .await;
3571        transport
3572            .bind(
3573                "owner/repo",
3574                &task_id,
3575                &session_name,
3576                "telegram",
3577                "12345",
3578                None,
3579            )
3580            .await;
3581
3582        // Spawn capture.run() which exits when session is unregistered
3583        let capture_clone = Arc::clone(&capture);
3584        let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3585
3586        // Subscribe to transport output for this task
3587        let mut rx = transport
3588            .subscribe("owner/repo", &task_id)
3589            .await
3590            .expect("no subscription");
3591
3592        // Send a message into the tmux session via tmux send-keys (simulate channel input)
3593        let send = tmux
3594            .command()
3595            .args([
3596                "send-keys",
3597                "-t",
3598                &session_name,
3599                "echo from-channel",
3600                "Enter",
3601            ])
3602            .output()
3603            .await
3604            .expect("failed to send keys");
3605        assert!(send.status.success());
3606
3607        // Wait for an output chunk from capture -> transport
3608        let mut got = false;
3609        for _ in 0..20 {
3610            if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3611                && c.content.contains("from-channel")
3612            {
3613                got = true;
3614                break;
3615            }
3616        }
3617
3618        // Clean up: unregister session (tmux kill handled by SessionGuard on drop)
3619        capture.unregister_session("owner/repo", &task_id).await;
3620
3621        // Wait for capture.run to finish
3622        let _ = capture_handle.await;
3623
3624        assert!(got, "did not observe tmux output via capture/transport");
3625    }
3626
3627    // ── read_project_channel_config tests ─────────────────────────────────
3628
3629    #[test]
3630    fn read_project_channel_config_returns_defaults_when_no_file() {
3631        let dir = tempfile::tempdir().expect("tempdir");
3632        let cfg = read_project_channel_config(dir.path());
3633        assert!(cfg.telegram_topic_id.is_none());
3634        assert!(cfg.discord_channel_id.is_none());
3635    }
3636
3637    #[test]
3638    fn read_project_channel_config_parses_telegram_and_discord() {
3639        let dir = tempfile::tempdir().expect("tempdir");
3640        let orch_yml = dir.path().join(".orch.yml");
3641        fs::write(
3642            &orch_yml,
3643            "channels:\n  telegram:\n    topic_id: \"42\"\n  discord:\n    channel_id: \"9999\"\n",
3644        )
3645        .expect("write .orch.yml");
3646        let cfg = read_project_channel_config(dir.path());
3647        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3648        assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3649    }
3650
3651    #[test]
3652    fn read_project_channel_config_handles_partial_channels() {
3653        let dir = tempfile::tempdir().expect("tempdir");
3654        let orch_yml = dir.path().join(".orch.yml");
3655        fs::write(&orch_yml, "channels:\n  telegram:\n    topic_id: \"7\"\n")
3656            .expect("write .orch.yml");
3657        let cfg = read_project_channel_config(dir.path());
3658        assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3659        assert!(cfg.discord_channel_id.is_none());
3660    }
3661
3662    #[test]
3663    fn read_project_channel_config_handles_malformed_yaml() {
3664        let dir = tempfile::tempdir().expect("tempdir");
3665        let orch_yml = dir.path().join(".orch.yml");
3666        fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3667        let cfg = read_project_channel_config(dir.path());
3668        // Should return defaults (not panic) and emit a warn! log with the parse error.
3669        assert!(cfg.telegram_topic_id.is_none());
3670        assert!(cfg.discord_channel_id.is_none());
3671    }
3672
3673    #[test]
3674    #[cfg(unix)]
3675    fn read_project_channel_config_handles_unreadable_file() {
3676        let dir = tempfile::tempdir().expect("tempdir");
3677        // A directory fails to read for every user, root included, where chmod 000 does not
3678        fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3679        let cfg = read_project_channel_config(dir.path());
3680        // Should return defaults (not panic) and emit a warn! log with the read error.
3681        assert!(cfg.telegram_topic_id.is_none());
3682        assert!(cfg.discord_channel_id.is_none());
3683    }
3684
3685    fn git(dir: &Path, args: &[&str]) {
3686        let mut cmd = process::Command::new("git");
3687        cmd.current_dir(dir);
3688        if args.first() == Some(&"commit") {
3689            cmd.args(["-c", "commit.gpgsign=false"]);
3690        }
3691        let out = cmd
3692            .args(args)
3693            .output()
3694            .unwrap_or_else(|e| panic!("git {args:?} failed to start: {e}"));
3695        assert!(
3696            out.status.success(),
3697            "git {:?} failed:\nstdout: {}\nstderr: {}",
3698            args,
3699            String::from_utf8_lossy(&out.stdout),
3700            String::from_utf8_lossy(&out.stderr)
3701        );
3702    }
3703
3704    fn restore_orch_home(prev: Option<String>) {
3705        match prev {
3706            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3707            Some(v) => unsafe { env::set_var("ORCH_HOME", v) },
3708            // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3709            None => unsafe { env::remove_var("ORCH_HOME") },
3710        }
3711    }
3712
3713    struct StartupConflict {
3714        _tmp: tempfile::TempDir,
3715        _orch_home: tempfile::TempDir,
3716        prev_orch_home: Option<String>,
3717        project_dir: PathBuf,
3718        worktree_dir: PathBuf,
3719        branch: String,
3720        store: Arc<TaskStore>,
3721        store_id: i64,
3722        engine: ProjectEngine,
3723    }
3724
3725    impl Drop for StartupConflict {
3726        fn drop(&mut self) {
3727            restore_orch_home(self.prev_orch_home.take());
3728        }
3729    }
3730
3731    async fn startup_conflict_fixture(
3732        retries: i32,
3733        pr_number: Option<i32>,
3734        start_rebase: bool,
3735    ) -> StartupConflict {
3736        let orch_home = tempfile::tempdir().unwrap();
3737        let prev_orch_home = env::var("ORCH_HOME").ok();
3738        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3739        unsafe { env::set_var("ORCH_HOME", orch_home.path()) };
3740
3741        let tmp = tempfile::tempdir().unwrap();
3742        let origin = tmp.path().join("origin.git");
3743        fs::create_dir_all(&origin).unwrap();
3744        git(&origin, &["init", "--bare", "-b", "main"]);
3745        let project_dir = tmp.path().join("repo");
3746        git(
3747            tmp.path(),
3748            &[
3749                "clone",
3750                origin.to_str().unwrap(),
3751                project_dir.to_str().unwrap(),
3752            ],
3753        );
3754        git(&project_dir, &["config", "user.email", "t@t.com"]);
3755        git(&project_dir, &["config", "user.name", "T"]);
3756        fs::write(project_dir.join("f.txt"), "base\n").unwrap();
3757        git(&project_dir, &["add", "."]);
3758        git(&project_dir, &["commit", "-m", "base"]);
3759        git(&project_dir, &["push", "origin", "HEAD:main"]);
3760        git(&project_dir, &["remote", "set-head", "origin", "main"]);
3761
3762        let branch = "gh-issue-4087-conflict".to_string();
3763        git(&project_dir, &["checkout", "-b", &branch]);
3764        fs::write(project_dir.join("f.txt"), "agent\n").unwrap();
3765        git(&project_dir, &["commit", "-am", "agent"]);
3766        git(&project_dir, &["checkout", "main"]);
3767        fs::write(project_dir.join("f.txt"), "main\n").unwrap();
3768        git(&project_dir, &["commit", "-am", "main"]);
3769        git(&project_dir, &["push", "origin", "HEAD:main"]);
3770
3771        let wt_base = orch_home.path().join("worktrees").join("repo");
3772        fs::create_dir_all(&wt_base).unwrap();
3773        let worktree_dir = wt_base.join(&branch);
3774        git(
3775            &project_dir,
3776            &["worktree", "add", worktree_dir.to_str().unwrap(), &branch],
3777        );
3778        git(&worktree_dir, &["config", "user.email", "t@t.com"]);
3779        git(&worktree_dir, &["config", "user.name", "T"]);
3780        git(&worktree_dir, &["fetch", "origin"]);
3781        if start_rebase {
3782            let rebase = process::Command::new("git")
3783                .args(["rebase", "origin/main"])
3784                .current_dir(&worktree_dir)
3785                .output()
3786                .unwrap();
3787            assert!(!rebase.status.success(), "precondition: rebase conflicts");
3788        }
3789
3790        let store = Arc::new(TaskStore::open_memory().await.unwrap());
3791        let store_id = store
3792            .upsert_external(&UpsertExternal {
3793                repo: "owner/repo",
3794                ext_id: "4087",
3795                title: "Conflict",
3796                body: "",
3797                author: "",
3798                url: "",
3799                labels: &[],
3800                origin: "github",
3801            })
3802            .await
3803            .unwrap();
3804        store
3805            .update_status(store_id, TaskStatus::Routed)
3806            .await
3807            .unwrap();
3808        let mut fields = vec![
3809            ("branch", serde_json::json!(branch)),
3810            (
3811                "worktree",
3812                serde_json::json!(worktree_dir.to_string_lossy().as_ref()),
3813            ),
3814            ("merge_conflict_retries", serde_json::json!(retries)),
3815        ];
3816        if let Some(pr) = pr_number {
3817            fields.push(("pr_number", serde_json::json!(pr)));
3818        }
3819        store.set_fields(store_id, &fields).await.unwrap();
3820
3821        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3822        let engine = ProjectEngine {
3823            repo: "owner/repo".to_string(),
3824            project_dir: project_dir.clone(),
3825            backend: Arc::clone(&backend),
3826            task_manager: Arc::new(TaskManager::with_store(
3827                backend,
3828                Arc::clone(&store),
3829                "owner/repo".to_string(),
3830            )),
3831            runner: Arc::new(
3832                runner::TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store)),
3833            ),
3834            store: Arc::clone(&store),
3835        };
3836
3837        StartupConflict {
3838            _tmp: tmp,
3839            _orch_home: orch_home,
3840            prev_orch_home,
3841            project_dir,
3842            worktree_dir,
3843            branch,
3844            store,
3845            store_id,
3846            engine,
3847        }
3848    }
3849
3850    #[tokio::test]
3851    #[serial_test::serial(orch_home)]
3852    async fn startup_keeps_a_content_conflict_when_the_task_retried_a_merge() {
3853        let fx = startup_conflict_fixture(2, Some(4043), false).await;
3854        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3855            .await
3856            .unwrap();
3857
3858        assert!(fx.worktree_dir.exists(), "worktree must stay");
3859        assert!(
3860            git_ops::rebase_in_progress(&fx.worktree_dir).await,
3861            "the content conflict must stay for the agent"
3862        );
3863        let task = fx.store.get(fx.store_id).await.unwrap();
3864        assert_eq!(task.status, TaskStatus::Routed);
3865        assert_eq!(task.branch, fx.branch);
3866        assert!(!task.worktree.is_empty());
3867    }
3868
3869    #[tokio::test]
3870    #[serial_test::serial(orch_home)]
3871    async fn startup_keeps_an_in_progress_rebase_when_the_task_has_a_pr() {
3872        let fx = startup_conflict_fixture(0, Some(12), true).await;
3873        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3874            .await
3875            .unwrap();
3876
3877        assert!(fx.worktree_dir.exists(), "worktree must stay");
3878        assert!(
3879            git_ops::rebase_in_progress(&fx.worktree_dir).await,
3880            "an in-progress rebase for a PR task must stay"
3881        );
3882        let task = fx.store.get(fx.store_id).await.unwrap();
3883        assert_eq!(task.status, TaskStatus::Routed);
3884        assert_eq!(task.branch, fx.branch);
3885    }
3886
3887    #[tokio::test]
3888    #[serial_test::serial(orch_home)]
3889    async fn startup_resets_a_conflict_without_retries_or_a_pr() {
3890        let fx = startup_conflict_fixture(0, None, false).await;
3891        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3892            .await
3893            .unwrap();
3894
3895        assert!(
3896            !fx.worktree_dir.exists(),
3897            "worktree is removed when there is nothing to keep"
3898        );
3899        let task = fx.store.get(fx.store_id).await.unwrap();
3900        assert_eq!(task.status, TaskStatus::New);
3901    }
3902
3903    #[tokio::test]
3904    #[serial_test::serial(orch_home)]
3905    async fn startup_reset_keeps_branch_when_rescue_cannot_push() {
3906        let fx = startup_conflict_fixture(0, None, false).await;
3907        git(
3908            &fx.project_dir,
3909            &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3910        );
3911        reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3912            .await
3913            .unwrap();
3914
3915        let task = fx.store.get(fx.store_id).await.unwrap();
3916        assert_eq!(task.status, TaskStatus::New);
3917        assert_eq!(
3918            task.branch, fx.branch,
3919            "the task still names the local branch cleanup kept"
3920        );
3921        assert!(
3922            fx.worktree_dir.exists(),
3923            "rescue failure keeps the worktree on disk"
3924        );
3925    }
3926}