1pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{remove_worktree_and_branch, remove_worktree_and_branch_for};
58use crate::engine::router::{Router, RouterConfig};
59use crate::engine::runner::worktree::{
60 abort_worktree_rebase, list_project_worktrees, rebase_worktree_on_origin_main,
61 task_id_from_worktree_name, validate_worktree_gitdir,
62};
63use crate::engine::tasks::TaskManager;
64use crate::github::http::{GhHttp, rate_limit_metrics};
65use crate::repo_context::REPO_CONTEXT;
66use crate::store::{TaskStatus, TaskStore};
67use crate::tmux::TmuxManager;
68use runner::WeightSignal;
69use crate::backends::Status;
71use crate::backends::github::GitHubBackend;
72use crate::channels::github;
73use crate::channels::github::is_transient_bind_error;
74use crate::channels::github::webhook_backoff_delay;
75use crate::channels::notification;
76use crate::channels::notification::TaskNotification;
77use crate::channels::transport;
78use crate::cluster;
79use crate::cluster::local;
80use crate::engine::cooldown as engine_cooldown;
81use crate::engine::review as engine_review;
82use crate::engine::runner::git_ops;
83use crate::engine::runner::worktree;
84use crate::engine::suspend as engine_suspend;
85use crate::engine::suspend::SuspendDetector;
86use crate::engine::tasks as engine_tasks;
87use crate::github::app;
88use crate::home;
89use crate::secret_ref;
90use crate::store;
91use crate::webhook_status::WebhookStatus;
92use futures::future;
93use std::collections::HashMap;
94use std::collections::HashSet;
95use std::fmt;
96use std::fmt::Debug;
97use std::fmt::Formatter;
98use std::fs;
99use std::path::Path;
100use std::path::PathBuf;
101use std::sync::Arc;
102use std::sync::atomic::AtomicBool;
103use std::sync::atomic::AtomicU64;
104use std::sync::atomic::Ordering;
105use std::time as std_time;
106use std::time::Duration;
107use std::time::Instant;
108use std::time::SystemTime;
109use tokio::process::Command;
110use tokio::signal;
111use tokio::signal::unix;
112use tokio::signal::unix::SignalKind;
113use tokio::sync::Mutex;
114use tokio::sync::broadcast::Sender;
115use tokio::sync::broadcast::error::RecvError;
116use tokio::sync::mpsc::Receiver;
117use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
118use tokio::task;
119use tokio::time;
120
121pub fn orch_footer() -> &'static str {
122 "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
123}
124
125pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
129 match model {
130 Some(m) => format!(
131 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
132 ),
133 None => format!(
134 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
135 ),
136 }
137}
138
139type EngineRef = (
141 String,
142 Arc<dyn ExternalBackend>,
143 Arc<TaskManager>,
144 Option<Arc<TaskStore>>,
145);
146
147#[derive(Clone)]
152pub struct ProjectEngine {
153 pub repo: String,
154 pub project_dir: PathBuf,
155 pub backend: Arc<dyn ExternalBackend>,
156 pub task_manager: Arc<TaskManager>,
157 pub runner: Arc<runner::TaskRunner>,
158 pub store: Arc<TaskStore>,
159}
160
161impl Debug for ProjectEngine {
162 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
163 f.debug_struct("ProjectEngine")
164 .field("repo", &self.repo)
165 .field("project_dir", &self.project_dir)
166 .finish_non_exhaustive()
167 }
168}
169
170pub(crate) async fn reset_orphaned_reviews(
173 task_manager: &TaskManager,
174 store: &Arc<TaskStore>,
175 repo: &str,
176 tmux: &TmuxManager,
177) -> usize {
178 let mut ids: Vec<ExternalId> = store
180 .list_by_status(repo, TaskStatus::InReview)
181 .await
182 .unwrap_or_default()
183 .iter()
184 .map(|t| engine_tasks::store_task_to_external(t).id)
185 .collect();
186 if let Ok(legacy) = task_manager
187 .list_internal_by_status(TaskStatus::InReview)
188 .await
189 {
190 ids.extend(legacy.into_iter().map(|t| t.id));
191 }
192 ids.sort_by(|a, b| a.0.cmp(&b.0));
193 ids.dedup_by(|a, b| a.0 == b.0);
194
195 let mut reset = 0;
196 for id in &ids {
197 let session = tmux.session_name(repo, &format!("{}-review", id.0));
198
199 if tmux.session_exists(&session).await {
200 continue;
201 }
202
203 if let Err(e) = task_manager
204 .update_task_status(id, Status::NeedsReview)
205 .await
206 {
207 tracing::warn!(
208 task_id = id.0,
209 ?e,
210 "failed to reset an orphaned review at start"
211 );
212 continue;
213 }
214
215 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
216 let reason = "startup: review session gone, reset to needs_review";
217
218 if let Err(e) = store
219 .finalize_incomplete_runs(store_id, "aborted", reason)
220 .await
221 {
222 tracing::warn!(
223 task_id = id.0,
224 ?e,
225 "failed to finalize the orphaned review's runs"
226 );
227 }
228 }
229
230 reset += 1;
231 }
232
233 reset
234}
235
236fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
239 let url = config::get_global("relay.url").unwrap_or_default();
241 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
242 return;
243 };
244
245 if url.is_empty() {
246 return;
247 }
248
249 let token_ref = config::get_global("relay.token").unwrap_or_default();
250 let notify = Arc::clone(notify);
251 tokio::spawn(async move {
252 let mut warned = false;
253 let token = loop {
254 let reference = token_ref.clone();
255 let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
256 .await
257 .map_err(anyhow::Error::from)
258 .and_then(|r| r);
259
260 match read {
261 Ok(token) if !token.is_empty() => break token,
262 Ok(_) => {
263 tracing::warn!("relay.token is empty, the relay client stays off");
264 return;
265 }
266 Err(e) if !warned => {
267 tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
268 warned = true;
269 }
270 Err(_) => {}
271 }
272
273 time::sleep(Duration::from_secs(300)).await;
274 };
275
276 relay::run(url, token, store, notify).await;
277 });
278}
279
280#[expect(
286 clippy::too_many_arguments,
287 reason = "the spawned task takes its own handle to each piece of engine state"
288)]
289fn spawn_sync_pass(
290 project_engines: &[ProjectEngine],
291 tmux: &Arc<TmuxManager>,
292 engine_config: &EngineConfig,
293 router: &Arc<RwLock<Router>>,
294 dispatching: &Arc<dashmap::DashMap<String, String>>,
295 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
296 sync_in_progress: &Arc<AtomicBool>,
297 trigger: &'static str,
298) {
299 let engines = project_engines.to_vec();
300 let tmux = Arc::clone(tmux);
301 let engine_config = engine_config.clone();
302 let router = Arc::clone(router);
303 let dispatching = Arc::clone(dispatching);
304 let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
305 let guard = Arc::clone(sync_in_progress);
306
307 tokio::spawn(async move {
308 let started = Instant::now();
309 let mut passes = Vec::with_capacity(engines.len());
310 for engine in &engines {
311 passes.push(REPO_CONTEXT.scope(
312 engine.repo.clone(),
313 sync_one_project(
314 engine,
315 &tmux,
316 &engine_config,
317 &router,
318 &dispatching,
319 &auto_merge_in_flight,
320 ),
321 ));
322 }
323 future::join_all(passes).await;
324
325 sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
326 tracing::info!(
327 trigger,
328 elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
329 "sync tick complete"
330 );
331 guard.store(false, Ordering::Release);
332 });
333}
334
335async fn sync_one_project(
336 engine: &ProjectEngine,
337 tmux: &Arc<TmuxManager>,
338 engine_config: &EngineConfig,
339 router: &Arc<RwLock<Router>>,
340 dispatching: &Arc<dashmap::DashMap<String, String>>,
341 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
342) {
343 if let Err(e) = Box::pin(sync::sync_tick(
344 &engine.backend,
345 tmux,
346 &engine.repo,
347 engine_config,
348 router,
349 &engine.task_manager,
350 &engine.store,
351 dispatching,
352 auto_merge_in_flight,
353 ))
354 .await
355 {
356 tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
357 }
358}
359
360const SLOW_PHASE_GROUP_MS: u128 = 1_000;
362
363pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
369 let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
370 if total_ms >= SLOW_PHASE_GROUP_MS {
371 tracing::info!(group, repo, total_ms, ?phases, "phase timings");
372 } else {
373 tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
374 }
375}
376
377pub fn configured_agents() -> Vec<String> {
389 if let Ok(agents_str) = config::get("agents")
390 && !agents_str.is_empty()
391 && agents_str != "[]"
392 {
393 if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
396 && !agents_arr.is_empty()
397 {
398 return agents_arr;
399 }
400 if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
402 && !agents_arr.is_empty()
403 {
404 return agents_arr;
405 }
406 }
407 router::config::DEFAULT_AGENTS
408 .iter()
409 .map(ToString::to_string)
410 .collect()
411}
412
413pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
419pub struct RunTimeouts {
420 pub base: u64,
421 pub medium: u64,
422 pub complex: u64,
423}
424
425impl Default for RunTimeouts {
426 fn default() -> Self {
427 Self {
428 base: 1800,
429 medium: 1800,
430 complex: 1800,
431 }
432 }
433}
434
435impl RunTimeouts {
436 pub fn from_config() -> Self {
439 let read =
440 |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
441 let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
442 Self {
443 base,
444 medium: read("workflow.timeout_by_complexity.medium")
445 .unwrap_or(base)
446 .max(base),
447 complex: read("workflow.timeout_by_complexity.complex")
448 .unwrap_or(base)
449 .max(base),
450 }
451 }
452
453 pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
454 match complexity {
455 Some("complex") => self.complex,
456 Some("medium") => self.medium,
457 _ => self.base,
458 }
459 }
460}
461
462#[derive(Clone, Debug)]
464pub struct EngineConfig {
465 pub tick_interval: Duration,
467 pub sync_interval: Duration,
469 pub webhook_health_check_interval: Option<Duration>,
471 pub max_parallel: usize,
473 pub stuck_timeout: u64,
475 pub no_session_stuck_timeout: u64,
478 pub in_review_no_session_stuck_timeout: u64,
485 pub run_timeouts: RunTimeouts,
487 pub auto_create_followup_on_changes: bool,
489 pub graceful_shutdown_timeout: Duration,
491 pub silence_grace_period: u64,
494 pub silence_cooldown: u64,
496 pub upgrade_check_interval: u64,
499 pub startup_failure_escalation_secs: u64,
504}
505
506impl Default for EngineConfig {
507 fn default() -> Self {
508 Self {
509 tick_interval: Duration::from_secs(10),
510 sync_interval: Duration::from_secs(45),
511 webhook_health_check_interval: Some(Duration::from_secs(60)),
512 max_parallel: 4,
513 stuck_timeout: 1800,
514 no_session_stuck_timeout: 600,
515 in_review_no_session_stuck_timeout: 1920,
516 run_timeouts: RunTimeouts::default(),
517 auto_create_followup_on_changes: true,
518 graceful_shutdown_timeout: Duration::from_secs(600),
519 silence_grace_period: 300,
520 silence_cooldown: 3600,
521 upgrade_check_interval: 3600,
522 startup_failure_escalation_secs: 3600,
523 }
524 }
525}
526
527impl EngineConfig {
528 pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
531 self.stuck_timeout.max(
532 self.run_timeouts
533 .for_complexity(complexity)
534 .saturating_add(RUN_TIMEOUT_GRACE_SECS),
535 )
536 }
537
538 pub fn from_config() -> Self {
540 let mut config = Self::default();
541
542 if let Ok(val) = config::get("engine.tick_interval") {
543 if let Ok(secs) = val.parse::<u64>() {
544 config.tick_interval = Duration::from_secs(secs);
545 } else {
546 tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
547 }
548 }
549
550 if let Ok(val) = config::get("engine.sync_interval") {
551 if let Ok(secs) = val.parse::<u64>() {
552 config.sync_interval = Duration::from_secs(secs);
553 } else {
554 tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
555 }
556 }
557
558 if let Ok(val) = config::get("engine.max_parallel") {
559 if let Ok(n) = val.parse::<usize>() {
560 config.max_parallel = n;
561 } else {
562 tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
563 }
564 }
565
566 if let Ok(val) = config::get("engine.stuck_timeout") {
567 if let Ok(secs) = val.parse::<u64>() {
568 config.stuck_timeout = secs;
569 } else {
570 tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
571 }
572 }
573
574 if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
575 if let Ok(secs) = val.parse::<u64>() {
576 config.no_session_stuck_timeout = secs;
577 } else {
578 tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
579 }
580 }
581
582 let mut in_review_from_config = false;
583 if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
584 if let Ok(secs) = val.parse::<u64>() {
585 config.in_review_no_session_stuck_timeout = secs;
586 in_review_from_config = true;
587 } else {
588 tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
589 }
590 }
591
592 let review_timeout = engine_review::review_timeout_seconds();
596 let floored = engine_review::no_session_review_floor(
597 config.in_review_no_session_stuck_timeout,
598 review_timeout,
599 );
600 if floored != config.in_review_no_session_stuck_timeout {
601 if in_review_from_config {
602 tracing::warn!(
603 configured_secs = config.in_review_no_session_stuck_timeout,
604 review_timeout_secs = review_timeout,
605 raised_to_secs = floored,
606 "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
607 );
608 } else {
609 tracing::debug!(
610 review_timeout_secs = review_timeout,
611 raised_to_secs = floored,
612 "raised the default in_review no-session limit over a long review timeout"
613 );
614 }
615 config.in_review_no_session_stuck_timeout = floored;
616 }
617
618 config.run_timeouts = RunTimeouts::from_config();
619
620 if let Ok(val) = config::get("engine.webhook_health_check_interval") {
621 if let Ok(secs) = val.parse::<u64>() {
622 if secs > 0 {
623 config.webhook_health_check_interval = Some(Duration::from_secs(secs));
624 } else {
625 config.webhook_health_check_interval = None;
626 }
627 } else {
628 tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
629 }
630 }
631
632 if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
633 config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
634 }
635
636 if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
637 if let Ok(secs) = val.parse::<u64>() {
638 config.graceful_shutdown_timeout = Duration::from_secs(secs);
639 } else {
640 tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
641 }
642 }
643
644 if let Ok(val) = config::get("engine.silence_grace_period") {
645 if let Ok(secs) = val.parse::<u64>() {
646 config.silence_grace_period = secs;
647 } else {
648 tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
649 }
650 }
651
652 if let Ok(val) = config::get("engine.silence_cooldown") {
653 if let Ok(secs) = val.parse::<u64>() {
654 config.silence_cooldown = secs;
655 } else {
656 tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
657 }
658 }
659
660 if let Ok(val) = config::get("engine.upgrade_check_interval") {
661 if let Ok(secs) = val.parse::<u64>() {
662 config.upgrade_check_interval = secs;
663 } else {
664 tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
665 }
666 }
667
668 if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
669 if let Ok(secs) = val.parse::<u64>() {
670 config.startup_failure_escalation_secs = secs;
671 } else {
672 tracing::warn!(
673 key = "engine.startup_failure_escalation_secs",
674 value = %val,
675 default_secs = config.startup_failure_escalation_secs,
676 "invalid value for engine.startup_failure_escalation_secs, using default"
677 );
678 }
679 }
680
681 config
682 }
683}
684
685pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
688
689pub fn format_rate_limited_until(reset_at: i64) -> String {
691 chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
692 || reset_at.to_string(),
693 |t| {
694 t.with_timezone(&chrono::Local)
695 .format("%H:%M:%S %Z")
696 .to_string()
697 },
698 )
699}
700
701async fn report_rate_limited(
704 reset_at: i64,
705 store: Option<&Arc<TaskStore>>,
706 transport: Option<&Arc<Transport>>,
707) {
708 let m = rate_limit_metrics();
709 let until = format_rate_limited_until(reset_at);
710 tracing::warn!(
711 reset_at,
712 rate_limit_hits = m.hits,
713 skipped_calls = m.skipped_calls,
714 five_xx_hits = m.five_xx_hits,
715 rest_remaining = ?m.rest_remaining,
716 graphql_remaining = ?m.graphql_remaining,
717 "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
718 );
719
720 if let Some(store) = store
721 && let Err(e) = store
722 .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
723 .await
724 {
725 tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
726 }
727
728 if let Some(transport) = transport {
729 transport.push_notification(TaskNotification {
730 task_id: "github".to_string(),
731 title: "GitHub rate limit".to_string(),
732 status: "blocked".to_string(),
733 agent: "orch".to_string(),
734 duration_seconds: 0.0,
735 summary: format!(
736 "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
737 ),
738 repo: None,
739 notify_target: None,
740 pr_number: None,
741 });
742 }
743}
744
745fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
751 if target > in_force {
752 semaphore.add_permits(target - in_force);
753 return target;
754 }
755
756 if target < in_force {
757 return in_force - semaphore.forget_permits(in_force - target);
758 }
759
760 in_force
761}
762
763const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
766
767fn restart_only_snapshot() -> Vec<Option<String>> {
769 RESTART_ONLY_KEYS
770 .iter()
771 .map(|key| config::get(key).ok())
772 .collect()
773}
774
775fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
778 let changed: Vec<&'static str> = RESTART_ONLY_KEYS
779 .iter()
780 .zip(old)
781 .zip(new)
782 .filter(|((_, old), new)| old != new)
783 .map(|((key, _), _)| *key)
784 .collect();
785
786 for key in &changed {
787 tracing::info!(key = key, "config changed, takes effect after restart");
788 }
789
790 changed
791}
792
793fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
796 macro_rules! log_changed {
797 ($($key:literal => $field:ident),+ $(,)?) => {
798 $(if old.$field != new.$field {
799 tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
800 })+
801 };
802 }
803
804 log_changed!(
805 "engine.tick_interval" => tick_interval,
806 "engine.sync_interval" => sync_interval,
807 "engine.webhook_health_check_interval" => webhook_health_check_interval,
808 "engine.max_parallel" => max_parallel,
809 "engine.stuck_timeout" => stuck_timeout,
810 "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
811 "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
812 "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
813 "engine.silence_grace_period" => silence_grace_period,
814 "engine.silence_cooldown" => silence_cooldown,
815 "engine.upgrade_check_interval" => upgrade_check_interval,
816 "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
817 "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
818 );
819}
820
821fn should_run_upgrade_check(
822 last_upgrade_check: Option<Instant>,
823 upgrade_check_interval: u64,
824) -> bool {
825 if upgrade_check_interval == 0 {
826 return false;
827 }
828
829 last_upgrade_check
830 .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
831}
832
833async fn connect_project_engine(
840 repo: String,
841 project_dir: PathBuf,
842 store: Arc<TaskStore>,
843 event_tx: Option<Sender<events::TaskEvent>>,
844) -> anyhow::Result<ProjectEngine> {
845 let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
846
847 match backend.health_check().await {
851 Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
852 Err(e) => tracing::warn!(
853 repo = %repo,
854 error = %e,
855 "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
856 ),
857 }
858
859 let task_manager = Arc::new(match event_tx {
860 Some(tx) => {
861 TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
862 }
863 None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
864 });
865 let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
866
867 Ok(ProjectEngine {
868 repo,
869 project_dir,
870 backend,
871 task_manager,
872 runner,
873 store,
874 })
875}
876
877fn diff_configured_projects(
884 configured: &[(String, PathBuf)],
885 active_repos: &HashSet<String>,
886) -> (Vec<(String, PathBuf)>, Vec<String>) {
887 let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
888
889 let added = configured
890 .iter()
891 .filter(|(repo, _)| !active_repos.contains(repo))
892 .cloned()
893 .collect();
894 let removed = active_repos
895 .iter()
896 .filter(|repo| !configured_repos.contains(repo.as_str()))
897 .cloned()
898 .collect();
899
900 (added, removed)
901}
902
903#[expect(
909 clippy::too_many_arguments,
910 reason = "a new project engine needs every shared engine handle"
911)]
912async fn sync_project_engines(
913 project_engines: &mut Vec<ProjectEngine>,
914 active_repos: &mut HashSet<String>,
915 event_bus: &events::EventBus,
916 tmux: &Arc<TmuxManager>,
917 capture: &Arc<CaptureService>,
918 semaphore: &Arc<Semaphore>,
919 review_semaphore: &Arc<Semaphore>,
920 weight_tx: &mpsc::Sender<WeightSignal>,
921 router: &Arc<RwLock<Router>>,
922 dispatching: &Arc<dashmap::DashMap<String, String>>,
923) {
924 let configured = match config::get_projects_with_paths() {
925 Ok(p) if !p.is_empty() => p,
926 Ok(_) => return,
927 Err(e) => {
928 tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
929 return;
930 }
931 };
932 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
933 return;
934 };
935
936 let (added, removed) = diff_configured_projects(&configured, active_repos);
937 for repo in removed {
938 tracing::info!(
939 repo = %repo,
940 "project removed from config, takes effect after restart"
941 );
942 }
943
944 for (repo, project_dir) in added {
945 config::watch(&project_dir.join(".orch.yml"));
946 let engine = match connect_project_engine(
947 repo.clone(),
948 project_dir,
949 Arc::clone(&store),
950 Some(event_bus.sender()),
951 )
952 .await
953 {
954 Ok(engine) => engine,
955 Err(e) => {
956 tracing::warn!(
959 repo = %repo,
960 err = %e,
961 "project added to config but its backend could not be built, retrying on the next config change"
962 );
963 continue;
964 }
965 };
966
967 subscribers::dispatch::spawn(
968 event_bus.subscribe(),
969 Arc::clone(&engine.backend),
970 Arc::clone(tmux),
971 Arc::clone(&engine.runner),
972 Arc::clone(capture),
973 Arc::clone(semaphore),
974 Arc::clone(&engine.task_manager),
975 weight_tx.clone(),
976 Arc::clone(router),
977 Arc::clone(dispatching),
978 Arc::clone(&engine.store),
979 engine.repo.clone(),
980 );
981 subscribers::review::spawn(
982 event_bus.subscribe(),
983 Arc::clone(&engine.backend),
984 Arc::clone(tmux),
985 Arc::clone(review_semaphore),
986 Arc::clone(&engine.task_manager),
987 Arc::clone(router),
988 Arc::clone(dispatching),
989 Arc::clone(&engine.store),
990 engine.repo.clone(),
991 );
992 subscribers::unblock::spawn(
993 event_bus.subscribe(),
994 Arc::clone(&engine.backend),
995 Arc::clone(&engine.task_manager),
996 Arc::clone(&engine.store),
997 engine.repo.clone(),
998 );
999
1000 active_repos.insert(repo.clone());
1001 project_engines.push(engine);
1002 tracing::info!(repo = %repo, "project added from config, now syncing");
1003 }
1004}
1005
1006async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1010 let projects = config::get_projects_with_paths()?;
1011 let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1012 tracing::info!(repos = ?repos, "loading projects from config");
1013
1014 if projects.is_empty() {
1015 let config_path = home::config_path().map_or_else(
1016 |_| "~/.orch/config.yml".to_string(),
1017 |p| p.display().to_string(),
1018 );
1019 anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1020 }
1021
1022 let mut engines = Vec::new();
1023
1024 let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1027 if cluster::enabled() {
1028 cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1029 local::serve((*store).clone()).await?;
1030 }
1031
1032 engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1035
1036 for (repo, project_dir) in projects {
1037 tracing::info!(repo = %repo, "initializing project engine");
1038 config::watch(&project_dir.join(".orch.yml"));
1039
1040 engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1041 }
1042
1043 Ok(engines)
1044}
1045
1046async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1047 for engine in project_engines {
1048 let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1049 let repo_root_path = PathBuf::from(&repo_root);
1050 let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1051 let worktrees = list_project_worktrees(&repo_root_path).await?;
1052
1053 if let Err(e) = git_ops::git()
1056 .await
1057 .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1058 .output_with_context()
1059 .await
1060 {
1061 tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1062 }
1063
1064 let mut orphans_removed: u32 = 0;
1065 let mut invalid_removed: u32 = 0;
1066 let mut cleaned: u32 = 0;
1067 let mut valid_kept: u32 = 0;
1068
1069 for worktree_dir in worktrees {
1070 let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1071 continue;
1072 };
1073
1074 if !validate_worktree_gitdir(&worktree_dir).await {
1077 tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1078 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1079 .await;
1080 invalid_removed += 1;
1081 continue;
1082 }
1083
1084 let Some(task_id) = task_id_from_worktree_name(name) else {
1085 tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1086 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1087 .await;
1088 orphans_removed += 1;
1089 continue;
1090 };
1091
1092 let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1093 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1094 remove_worktree_and_branch(
1095 &task_id,
1096 &worktree_dir,
1097 Some(name),
1098 &repo_root_path,
1099 false,
1100 )
1101 .await;
1102 orphans_removed += 1;
1103 continue;
1104 };
1105
1106 let task = engine.store.get(store_id).await?;
1107 let branch_name = if task.branch.is_empty() {
1108 name
1109 } else {
1110 task.branch.as_str()
1111 };
1112
1113 match task.status {
1114 TaskStatus::New
1115 | TaskStatus::Routed
1116 | TaskStatus::InProgress
1117 | TaskStatus::NeedsReview
1118 | TaskStatus::InReview => {
1119 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1120 match rebase_worktree_on_origin_main(&worktree_dir, &default_branch).await {
1121 Err(e) => {
1122 tracing::warn!(repo = %engine.repo, task_id = %task_id, err = %e, "startup rebase failed, resetting task");
1127 abort_worktree_rebase(&worktree_dir).await;
1128 let keep_remote = task.pr_number.is_some();
1129 remove_worktree_and_branch(
1130 &task_id,
1131 &worktree_dir,
1132 Some(branch_name),
1133 &repo_root_path,
1134 keep_remote,
1135 )
1136 .await;
1137 match engine.store.resolve_task_id(&engine.repo, &task_id).await {
1138 Ok(Some(reset_id)) => {
1139 if let Err(e) = engine.store.reset_to_new(reset_id).await {
1140 tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1141 }
1142 }
1143 Ok(None) => {
1144 tracing::warn!(
1145 task_id,
1146 "task not found in store during reconciliation"
1147 );
1148 }
1149 Err(e) => {
1150 tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1151 }
1152 }
1153 if let Err(e) = engine
1154 .task_manager
1155 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1156 .await
1157 {
1158 tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1159 }
1160 cleaned += 1;
1161 }
1162 _ => {
1163 valid_kept += 1;
1164 }
1165 }
1166 }
1167 TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1170 valid_kept += 1;
1171 }
1172 TaskStatus::Done => {
1173 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1174 remove_worktree_and_branch_for(
1175 &task_id,
1176 &worktree_dir,
1177 Some(branch_name),
1178 &repo_root_path,
1179 false,
1180 task.pr_number.is_some(),
1181 )
1182 .await;
1183 cleaned += 1;
1184 }
1185 }
1186 }
1187
1188 tracing::info!(
1189 repo = %engine.repo,
1190 orphans_removed,
1191 invalid_removed,
1192 cleaned,
1193 valid_kept,
1194 "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1195 );
1196
1197 let prune = Command::new("git")
1200 .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1201 .output_with_context()
1202 .await;
1203 match prune {
1204 Ok(output) if output.status.success() => {
1205 tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1206 }
1207 Ok(output) => {
1208 let stderr = String::from_utf8_lossy(&output.stderr);
1209 tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1210 }
1211 Err(e) => {
1212 tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1213 }
1214 }
1215 }
1216
1217 Ok(())
1218}
1219
1220async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1228 for engine in project_engines {
1229 let tasks = match engine.store.list_external_tasks_with_estimates().await {
1230 Ok(t) => t,
1231 Err(e) => {
1232 tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1233 continue;
1234 }
1235 };
1236
1237 if tasks.is_empty() {
1238 continue;
1239 }
1240
1241 let mut synced: u32 = 0;
1242 let mut failed: u32 = 0;
1243 for (external_id, estimate) in &tasks {
1244 match engine
1245 .backend
1246 .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1247 .await
1248 {
1249 Ok(()) => synced += 1,
1250 Err(e) => {
1251 tracing::debug!(
1252 repo = %engine.repo,
1253 task_id = %external_id,
1254 estimate,
1255 err = %e,
1256 "startup estimate reconciliation: sync failed"
1257 );
1258 failed += 1;
1259 }
1260 }
1261 }
1262
1263 if synced > 0 || failed > 0 {
1264 tracing::info!(
1265 repo = %engine.repo,
1266 synced,
1267 failed,
1268 "startup estimate reconciliation complete"
1269 );
1270 }
1271 }
1272}
1273
1274fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1276 let config_path = project_dir.join(".orch.yml");
1277 if !config_path.exists() {
1278 return ProjectChannelConfig::default();
1279 }
1280 let content = match fs::read_to_string(&config_path) {
1281 Ok(c) => c,
1282 Err(e) => {
1283 tracing::warn!(
1284 path = %config_path.display(),
1285 error = %e,
1286 "failed to read .orch.yml; channel config will use defaults"
1287 );
1288 return ProjectChannelConfig::default();
1289 }
1290 };
1291 let val: serde_norway::Value = match serde_norway::from_str(&content) {
1292 Ok(v) => v,
1293 Err(e) => {
1294 tracing::warn!(
1295 path = %config_path.display(),
1296 error = %e,
1297 "failed to parse .orch.yml; channel config will use defaults"
1298 );
1299 return ProjectChannelConfig::default();
1300 }
1301 };
1302 let Some(channels) = val.get("channels") else {
1303 return ProjectChannelConfig::default();
1304 };
1305 let get_str = |section: &str, key: &str| -> Option<String> {
1306 channels
1307 .get(section)
1308 .and_then(|s| s.get(key))
1309 .and_then(|v| v.as_str())
1310 .map(String::from)
1311 };
1312 ProjectChannelConfig {
1313 telegram_topic_id: get_str("telegram", "topic_id"),
1314 discord_channel_id: get_str("discord", "channel_id"),
1315 }
1316}
1317
1318async fn fetch_latest_release_version() -> Option<String> {
1323 let output = Command::new("gh")
1324 .args([
1325 "api",
1326 "repos/gabrielkoerich/orch/releases/latest",
1327 "--jq",
1328 ".tag_name",
1329 ])
1330 .output()
1331 .await
1332 .ok()?;
1333 if !output.status.success() {
1334 return None;
1335 }
1336 let tag = String::from_utf8(output.stdout).ok()?;
1337 let version = tag.trim().trim_start_matches('v').to_string();
1338 if version.is_empty() {
1339 None
1340 } else {
1341 Some(version)
1342 }
1343}
1344
1345async fn check_and_notify_upgrade(
1354 store: &Arc<TaskStore>,
1355 channels: &Arc<ChannelRegistry>,
1356 current_version: &str,
1357 check_interval: Duration,
1358) {
1359 let last_check_key = "upgrade:last_check_at";
1360 let last_notified_key = "upgrade:last_notified_version";
1361
1362 let now = chrono::Utc::now().timestamp();
1364 if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1365 && let Ok(last_check) = last_check_str.parse::<i64>()
1366 && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1367 {
1368 return;
1369 }
1370
1371 let _ = store.kv_set(last_check_key, &now.to_string()).await;
1373
1374 let Some(latest) = fetch_latest_release_version().await else {
1375 tracing::debug!("upgrade check: failed to fetch latest release version");
1376 return;
1377 };
1378
1379 if latest == current_version {
1380 tracing::debug!(current = %current_version, "orch is up to date");
1381 return;
1382 }
1383
1384 tracing::warn!(
1385 current_version = %current_version,
1386 latest_version = %latest,
1387 "orch upgrade available"
1388 );
1389
1390 if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1395 && last_notified == latest
1396 {
1397 tracing::debug!(latest = %latest, "upgrade already notified");
1398 return;
1399 }
1400
1401 let msg_telegram = format!(
1402 "⚠️ <b>Orch Upgrade Available</b>\n\n\
1403 Service: <code>{current_version}</code>\n\
1404 Latest: <code>{latest}</code>\n\n\
1405 Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1406 );
1407
1408 let msg_discord = format!(
1409 "⚠️ **Orch Upgrade Available**\n\n\
1410 Service: `{current_version}`\n\
1411 Latest: `{latest}`\n\n\
1412 Run: `brew update && brew upgrade orch && brew services restart orch`"
1413 );
1414
1415 let msg_slack = format!(
1416 "⚠️ *Orch Upgrade Available*\n\n\
1417 Service: `{current_version}`\n\
1418 Latest: `{latest}`\n\n\
1419 Run: `brew update && brew upgrade orch && brew services restart orch`"
1420 );
1421
1422 for channel in channels.iter() {
1423 let (body, metadata, topic_id) = match channel.name() {
1424 "telegram" => (
1425 msg_telegram.clone(),
1426 serde_json::json!({"preformatted_html": true}),
1427 None,
1428 ),
1429 "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1430 "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1431 _ => continue,
1432 };
1433
1434 let msg = OutgoingMessage {
1435 thread_id: "upgrade".to_string(),
1436 body,
1437 reply_to: None,
1438 metadata,
1439 topic_id,
1440 };
1441
1442 if let Err(e) = channel.send(&msg).await {
1443 tracing::warn!(
1444 channel = channel.name(),
1445 ?e,
1446 "failed to send upgrade notification"
1447 );
1448 }
1449 }
1450
1451 let _ = store.kv_set(last_notified_key, &latest).await;
1453}
1454
1455async fn send_startup_escalation_notification(elapsed_secs: u64) {
1463 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1464 let duration = notification::format_duration(elapsed_secs as f64);
1465
1466 let msg_telegram = format!(
1467 "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1468 The engine has been retrying backend connections for {duration}\n\
1469 without success. All task processing, routing, and scheduled\n\
1470 jobs are paused. Check service logs with `orch log -f`."
1471 );
1472
1473 let msg_discord = format!(
1474 "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1475 The engine has been retrying backend connections for {duration}\n\
1476 without success. All task processing, routing, and scheduled\n\
1477 jobs are paused. Check service logs with `orch log -f`."
1478 );
1479
1480 let msg_slack = format!(
1481 "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1482 The engine has been retrying backend connections for {duration}\n\
1483 without success. All task processing, routing, and scheduled\n\
1484 jobs are paused. Check service logs with `orch log -f`."
1485 );
1486
1487 let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1489 let tg_chat_id = config::get("channels.telegram.chat_id");
1490 if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1491 && !token.is_empty()
1492 && !chat_id.is_empty()
1493 && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1494 {
1495 let msg = OutgoingMessage {
1496 thread_id: "startup_escalation".to_string(),
1497 body: msg_telegram.clone(),
1498 reply_to: None,
1499 metadata: serde_json::json!({ "preformatted_html": true }),
1500 topic_id: config::get("channels.telegram.general_topic_id").ok(),
1501 };
1502 match channel.send(&msg).await {
1503 Err(e) => {
1504 tracing::warn!(?e, "failed to send startup escalation via telegram");
1505 }
1506 _ => {
1507 tracing::info!("startup escalation notification sent to telegram");
1508 }
1509 }
1510 }
1511
1512 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1514 let channel_id = config::get("channels.discord.channel_id")
1515 .or_else(|_| config::get("channels.discord.general_channel_id"));
1516 if !token.is_empty()
1517 && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1518 {
1519 let shard_id = config::get("channels.discord.shard_id")
1520 .ok()
1521 .and_then(|v| v.parse::<u64>().ok())
1522 .unwrap_or(0);
1523 let shard_count = config::get("channels.discord.shard_count")
1524 .ok()
1525 .and_then(|v| v.parse::<u64>().ok())
1526 .unwrap_or(1);
1527 if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1528 let msg = OutgoingMessage {
1529 thread_id: "startup_escalation".to_string(),
1530 body: msg_discord.clone(),
1531 reply_to: None,
1532 metadata: serde_json::Value::Null,
1533 topic_id: None,
1534 };
1535 match gateway.send(&msg).await {
1536 Err(e) => {
1537 tracing::warn!(?e, "failed to send startup escalation via discord");
1538 }
1539 _ => {
1540 tracing::info!("startup escalation notification sent to discord");
1541 }
1542 }
1543 }
1544 }
1545 }
1546
1547 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1549 let channel_id = config::get("channels.slack.channel_id");
1550 if !token.is_empty()
1551 && let Ok(cid) = channel_id
1552 && let Ok(channel) = SlackChannel::new(token, Some(cid))
1553 {
1554 let msg = OutgoingMessage {
1555 thread_id: "startup_escalation".to_string(),
1556 body: msg_slack.clone(),
1557 reply_to: None,
1558 metadata: serde_json::Value::Null,
1559 topic_id: None,
1560 };
1561 match channel.send(&msg).await {
1562 Err(e) => {
1563 tracing::warn!(?e, "failed to send startup escalation via slack");
1564 }
1565 _ => {
1566 tracing::info!("startup escalation notification sent to slack");
1567 }
1568 }
1569 }
1570 }
1571}
1572
1573pub async fn serve() -> anyhow::Result<()> {
1577 let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1578 if killed > 0 {
1579 tracing::info!(count = killed, "terminated stale orch serve processes");
1580 }
1581
1582 tracing::info!("orch engine starting");
1583
1584 if let Err(e) = home::init_dirs().await {
1588 tracing::warn!("failed to initialize orch directories: {e}");
1589 }
1590
1591 if let Err(e) = config::warm_cache().await {
1594 tracing::warn!("failed to warm config cache: {e}");
1595 }
1596
1597 let mut config = EngineConfig::from_config();
1598
1599 tracing::info!("internal database ready");
1600
1601 let startup_escalation_secs = config.startup_failure_escalation_secs;
1610 let mut project_engines = {
1611 let mut delay_secs = 5u64;
1612 let mut attempt = 0u32;
1613 let mut escalated = false;
1614 let start = Instant::now();
1615 loop {
1616 attempt += 1;
1617 match init_project_engines().await {
1618 Ok(engines) => break engines,
1619 Err(e) => {
1620 let elapsed = start.elapsed().as_secs();
1621
1622 if startup_escalation_secs > 0
1628 && !escalated
1629 && elapsed >= startup_escalation_secs
1630 {
1631 escalated = true;
1632 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1633 let outage = notification::format_duration(elapsed as f64);
1634 tracing::error!(
1635 delay_secs,
1636 attempt,
1637 elapsed_secs = elapsed,
1638 threshold_secs = startup_escalation_secs,
1639 "project backends unavailable for {outage} — escalating startup failure: {e}",
1640 );
1641 send_startup_escalation_notification(elapsed).await;
1645 } else {
1646 tracing::warn!(
1647 delay_secs,
1648 attempt,
1649 "project backends unavailable, retrying: {e}"
1650 );
1651 }
1652
1653 time::sleep(Duration::from_secs(delay_secs)).await;
1654 delay_secs = (delay_secs * 2).min(120);
1655 }
1656 }
1657 }
1658 };
1659
1660 tracing::info!(
1661 project_count = project_engines.len(),
1662 "initialized project engines"
1663 );
1664
1665 let mut active_repos: HashSet<String> =
1669 project_engines.iter().map(|e| e.repo.clone()).collect();
1670
1671 let event_bus = events::EventBus::new(256);
1673 if let Err(e) = event_bus.start_ws_server().await {
1674 tracing::warn!(
1675 ?e,
1676 "failed to start event websocket server, continuing without it"
1677 );
1678 }
1679
1680 {
1684 let current = env!("ORCH_VERSION").to_string();
1685 tokio::spawn(async move {
1686 if let Some(latest) = fetch_latest_release_version().await
1687 && latest != current
1688 {
1689 tracing::warn!(
1690 current_version = %current,
1691 latest_version = %latest,
1692 "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1693 );
1694 }
1695 });
1696 }
1697
1698 for engine in &mut project_engines {
1700 engine.task_manager = Arc::new(TaskManager::with_events(
1701 Arc::clone(&engine.backend),
1702 Arc::clone(&engine.store),
1703 engine.repo.clone(),
1704 event_bus.sender(),
1705 ));
1706 }
1707
1708 let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1710 if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1711 tracing::warn!(err = %e, "startup worktree reconciliation failed");
1712 }
1713
1714 if leads {
1715 let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1719 tokio::spawn(async move {
1720 reconcile_startup_estimates(&engines_snapshot).await;
1721 });
1722 }
1723
1724 let global_channel_config = GlobalChannelConfig {
1726 telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1727 discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1728 control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1729 control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1730 };
1731 let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1732 .iter()
1733 .map(|e| {
1734 let cfg = read_project_channel_config(&e.project_dir);
1735 (e.repo.clone(), cfg)
1736 })
1737 .collect();
1738 let channel_router = Arc::new(ChannelRouter::new(
1739 &global_channel_config,
1740 &project_channel_configs,
1741 ));
1742 tracing::info!(
1743 projects = ?channel_router.projects(),
1744 "channel router initialized"
1745 );
1746
1747 let tmux = Arc::new(TmuxManager::new());
1749 tick::startup_cleanup(&tmux).await;
1752
1753 let transport = Arc::new(Transport::new());
1755
1756 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1758 let capture_for_tick = Arc::clone(&capture);
1759 tokio::spawn(async move {
1760 capture.start().await;
1761 });
1762
1763 let mut channel_registry = ChannelRegistry::new();
1765
1766 if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1768 && !token.is_empty()
1769 {
1770 let chat_id = config::get("channels.telegram.chat_id").ok();
1771 match TelegramChannel::new(token, chat_id) {
1772 Ok(telegram) => {
1773 if let Err(e) = telegram.health_check().await {
1774 tracing::warn!(?e, "telegram channel health check failed, skipping");
1775 } else {
1776 channel_registry.register(Box::new(telegram));
1777 tracing::info!("telegram channel registered");
1778 }
1779 }
1780 Err(e) => {
1781 tracing::warn!(?e, "failed to create telegram channel, skipping");
1782 }
1783 }
1784 }
1785
1786 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1788 && !token.is_empty()
1789 {
1790 let channel_id = config::get("channels.discord.channel_id").ok();
1791 let shard_id = config::get("channels.discord.shard_id")
1792 .ok()
1793 .and_then(|v| v.parse::<u64>().ok())
1794 .unwrap_or(0);
1795 let shard_count = config::get("channels.discord.shard_count")
1796 .ok()
1797 .and_then(|v| v.parse::<u64>().ok())
1798 .unwrap_or(1);
1799 match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1800 Ok(discord) => {
1801 if let Err(e) = discord.health_check().await {
1802 tracing::warn!(?e, "discord gateway health check failed, skipping");
1803 } else {
1804 channel_registry.register(Box::new(discord));
1805 tracing::info!(shard_id, shard_count, "discord gateway registered");
1806 }
1807 }
1808 Err(e) => {
1809 tracing::warn!(?e, "failed to create discord gateway, skipping");
1810 }
1811 }
1812 }
1813
1814 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1816 && !token.is_empty()
1817 {
1818 let channel_id = config::get("channels.slack.channel_id").ok();
1819 match SlackChannel::new(token, channel_id) {
1820 Ok(slack) => {
1821 if let Err(e) = slack.health_check().await {
1822 tracing::warn!(?e, "slack channel health check failed, skipping");
1823 } else {
1824 channel_registry.register(Box::new(slack));
1825 tracing::info!("slack channel registered");
1826 }
1827 }
1828 Err(e) => {
1829 tracing::warn!(?e, "failed to create slack channel, skipping");
1830 }
1831 }
1832 }
1833
1834 let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1836 channel_registry.register(Box::new(tmux_channel));
1837 tracing::info!("tmux channel registered");
1838
1839 let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1841 for channel in channel_registry.iter() {
1842 match channel.start().await {
1843 Ok(rx) => {
1844 tracing::info!(channel = channel.name(), "channel started");
1845 channel_receivers.push(rx);
1846 }
1847 Err(e) => {
1848 tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1849 }
1850 }
1851 }
1852
1853 let channel_registry = Arc::new(channel_registry);
1855
1856 let transport_for_messages = Arc::clone(&transport);
1858 let tmux_for_messages = Arc::clone(&tmux);
1859 let capture_for_messages = Arc::clone(&capture_for_tick);
1860 let channels_for_messages = Arc::clone(&channel_registry);
1861 let engine_refs: Vec<EngineRef> = project_engines
1863 .iter()
1864 .map(|e| {
1865 (
1866 e.repo.clone(),
1867 Arc::clone(&e.backend),
1868 Arc::clone(&e.task_manager),
1869 Some(Arc::clone(&e.store)),
1870 )
1871 })
1872 .collect();
1873 let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1875 let router_for_messages = Arc::clone(&channel_router);
1876 for mut rx in channel_receivers {
1877 let transport = Arc::clone(&transport_for_messages);
1878 let tmux = Arc::clone(&tmux_for_messages);
1879 let capture = Arc::clone(&capture_for_messages);
1880 let channels = Arc::clone(&channels_for_messages);
1881 let engine_refs = engine_refs.clone();
1882 let ch_router = Arc::clone(&router_for_messages);
1883 let picks = Arc::clone(&pending_picks);
1884 tokio::spawn(async move {
1885 while let Some(msg) = rx.recv().await {
1886 tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1887 channel_handler::handle_channel_message(
1888 msg,
1889 &transport,
1890 &tmux,
1891 &capture,
1892 &channels,
1893 &engine_refs,
1894 &ch_router,
1895 &picks,
1896 )
1897 .await;
1898 }
1899 });
1900 }
1901
1902 {
1906 let restart_channels = Arc::clone(&channel_registry);
1907 let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1908 if let Some(store) = restart_store {
1909 tokio::spawn(async move {
1910 time::sleep(Duration::from_secs(5)).await;
1911 match store.kv_get("control:restart_pending").await {
1912 Ok(Some(conv_key)) => {
1913 if let Some((ch_name, thread_id, topic_id)) =
1914 transport::parse_conversation_key(&conv_key)
1915 {
1916 for ch in restart_channels.iter() {
1917 if ch.name() == ch_name {
1918 let msg = OutgoingMessage {
1919 thread_id: thread_id.to_string(),
1920 body: "orch restarted successfully.".to_string(),
1921 reply_to: None,
1922 metadata: serde_json::json!({}),
1923 topic_id: topic_id.map(ToString::to_string),
1924 };
1925 if let Err(e) = ch.send(&msg).await {
1926 tracing::warn!(
1927 channel = ch_name,
1928 error = %e,
1929 "failed to send restart confirmation"
1930 );
1931 }
1932 break;
1933 }
1934 }
1935 }
1936 if let Err(e) = store.kv_delete("control:restart_pending").await {
1937 tracing::warn!(error = %e, "failed to clear control:restart_pending");
1938 }
1939 }
1940 Ok(None) => {} Err(e) => {
1942 tracing::warn!(error = %e, "failed to read control:restart_pending");
1943 }
1944 }
1945 });
1946 }
1947 }
1948
1949 {
1960 let mut notification_rx = transport.subscribe_notifications();
1961 let channels = Arc::clone(&channel_registry);
1962 let notif_router = Arc::clone(&channel_router);
1963 let notif_store: Option<Arc<TaskStore>> =
1965 project_engines.first().map(|e| Arc::clone(&e.store));
1966 tokio::spawn(async move {
1967 loop {
1968 match notification_rx.recv().await {
1969 Ok(notification) => {
1970 let level = NotificationLevel::from_config();
1971 if !level.should_notify(¬ification.status) {
1972 tracing::debug!(
1973 task_id = %notification.task_id,
1974 status = %notification.status,
1975 "notification suppressed by level={:?}",
1976 level
1977 );
1978 continue;
1979 }
1980
1981 tracing::info!(
1982 task_id = %notification.task_id,
1983 status = %notification.status,
1984 repo = ?notification.repo,
1985 "routing notification to channels"
1986 );
1987
1988 let mut routed = false;
1990
1991 if let Some(ref target) = notification.notify_target {
1994 let telegram = channels.iter().find(|c| c.name() == "telegram");
1995 if let Some(channel) = telegram {
1996 let body = notification.format_telegram();
1997 let metadata = serde_json::json!({
1998 "preformatted_html": true,
1999 "chat_id_override": target
2000 });
2001 let msg = OutgoingMessage {
2002 thread_id: notification.task_id.clone(),
2003 body,
2004 reply_to: None,
2005 metadata,
2006 topic_id: None,
2007 };
2008 match channel.send(&msg).await {
2009 Err(e) => {
2010 tracing::warn!(
2011 task_id = %notification.task_id,
2012 target,
2013 ?e,
2014 "failed to send to notify_target"
2015 );
2016 }
2017 _ => {
2018 routed = true;
2019 }
2020 }
2021 }
2022 }
2023
2024 if routed {
2025 } else if let Some(repo) = notification.repo.as_deref() {
2027 let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2033
2034 for channel in channels.iter() {
2036 let ch_name = channel.name();
2037 if let Some(topic_id) =
2038 notif_router.target_for_project(repo, ch_name)
2039 {
2040 let body = match ch_name {
2041 "telegram" => notification.format_telegram(),
2042 "discord" => notification.format_discord(),
2043 "slack" => notification.format_slack(),
2044 _ => continue,
2045 };
2046 let metadata = if ch_name == "telegram" {
2047 serde_json::json!({"preformatted_html": true})
2048 } else {
2049 serde_json::json!({})
2050 };
2051 let resolved_topic = Some(topic_id.to_string());
2052 let msg = OutgoingMessage {
2053 thread_id: notification.task_id.clone(),
2054 body,
2055 reply_to: None,
2056 metadata,
2057 topic_id: resolved_topic.clone(),
2058 };
2059 if let Err(e) = channel.send(&msg).await {
2060 tracing::warn!(
2061 channel = ch_name,
2062 task_id = %notification.task_id,
2063 ?e,
2064 "failed to send to dedicated channel"
2065 );
2066 } else {
2067 sent.insert((ch_name.to_string(), resolved_topic));
2068 routed = true;
2069 }
2070 }
2071 }
2072
2073 if let Some(store) = ¬if_store {
2075 match store.list_subscribers_for_repo(repo).await {
2076 Ok(subscribers) => {
2077 for (ch_name, thread_id, topic_id) in subscribers {
2078 let channel =
2079 channels.iter().find(|c| c.name() == ch_name);
2080 let Some(channel) = channel else {
2081 continue;
2082 };
2083 let resolved_topic = if topic_id.is_empty() {
2084 None
2085 } else {
2086 Some(topic_id.clone())
2087 };
2088 if sent.contains(&(
2091 ch_name.clone(),
2092 resolved_topic.clone(),
2093 )) {
2094 tracing::debug!(
2095 channel = %ch_name,
2096 task_id = %notification.task_id,
2097 topic_id = ?resolved_topic,
2098 "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2099 );
2100 continue;
2101 }
2102 let body = notification.format_with_project(&ch_name);
2103 let metadata = if ch_name == "telegram" {
2104 serde_json::json!({"preformatted_html": true})
2105 } else {
2106 serde_json::json!({})
2107 };
2108 let msg = OutgoingMessage {
2109 thread_id: thread_id.clone(),
2110 body,
2111 reply_to: None,
2112 metadata,
2113 topic_id: resolved_topic.clone(),
2114 };
2115 if let Err(e) = channel.send(&msg).await {
2116 tracing::warn!(
2117 channel = %ch_name,
2118 task_id = %notification.task_id,
2119 ?e,
2120 "failed to send to subscribed channel"
2121 );
2122 } else {
2123 sent.insert((ch_name.clone(), resolved_topic));
2124 routed = true;
2125 }
2126 }
2127 }
2128 Err(e) => {
2129 tracing::warn!(
2130 repo,
2131 ?e,
2132 "failed to list subscribers for repo"
2133 );
2134 }
2135 }
2136 }
2137 }
2138
2139 if !routed {
2141 for channel in channels.iter() {
2142 let (body, should_send, is_telegram) = match channel.name() {
2143 "telegram" => (notification.format_telegram(), true, true),
2144 "discord" => (notification.format_discord(), true, false),
2145 "slack" => (notification.format_slack(), true, false),
2146 _ => (String::new(), false, false),
2149 };
2150
2151 if !should_send {
2152 continue;
2153 }
2154
2155 let metadata = if is_telegram {
2156 serde_json::json!({"preformatted_html": true})
2157 } else {
2158 serde_json::json!({})
2159 };
2160 let msg = OutgoingMessage {
2161 thread_id: notification.task_id.clone(),
2162 body,
2163 reply_to: None,
2164 metadata,
2165 topic_id: None,
2166 };
2167
2168 if let Err(e) = channel.send(&msg).await {
2169 tracing::warn!(
2170 channel = channel.name(),
2171 task_id = %notification.task_id,
2172 ?e,
2173 "failed to send notification"
2174 );
2175 }
2176 }
2177 }
2178 }
2179 Err(RecvError::Lagged(n)) => {
2180 tracing::warn!(missed = n, "notification receiver lagged");
2181 }
2182 Err(RecvError::Closed) => {
2183 tracing::debug!("notification channel closed");
2184 break;
2185 }
2186 }
2187 }
2188 });
2189 tracing::info!("notification dispatcher started");
2190 }
2191
2192 let webhook_notify = Arc::new(Notify::new());
2194
2195 let webhook_port: Option<u16>;
2198 let mut webhook_healthy: bool;
2199 let mut last_webhook_health_check = Instant::now();
2200 let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2202 let webhook_secret = if webhook_enabled {
2204 secret_ref::config_secret("webhook.secret")
2205 .await
2206 .ok()
2207 .filter(|s| !s.trim().is_empty())
2208 } else {
2209 None
2210 };
2211 if webhook_enabled && webhook_secret.is_none() {
2212 tracing::error!(
2213 "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2214 );
2215 }
2216 let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2217 let webhook_secret = webhook_secret.unwrap_or_default();
2218
2219 let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2221
2222 if webhook_enabled {
2223 const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2227 let port: u16 = config::get("webhook.port")
2228 .ok()
2229 .and_then(|v| v.parse().ok())
2230 .unwrap_or(8080);
2231 webhook_port = Some(port);
2232 let secret = webhook_secret.clone();
2233 let webhook_repo = project_engines
2234 .first()
2235 .map(|e| e.repo.clone())
2236 .unwrap_or_default();
2237
2238 let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2239
2240 let to_save = {
2242 let mut s = webhook_status.lock().await;
2243 s.configured = true;
2244 s.port = Some(port);
2245 s.healthy = false;
2246 s.fallback_mode = false;
2247 s.clone()
2248 };
2249 to_save.save().await;
2250
2251 let status_for_spawn = Arc::clone(&webhook_status);
2252 tokio::spawn(async move {
2253 let mut attempt = 0u32;
2254 loop {
2255 attempt += 1;
2256 tracing::info!(attempt, port, "attempting to start webhook server");
2257 match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2258 .await
2259 {
2260 Ok(()) => {
2261 tracing::info!(attempt, "webhook server exited cleanly");
2262 break;
2263 }
2264 Err(e) => {
2265 let reason = e.to_string();
2266 tracing::error!(attempt, %reason, "webhook server failed");
2267 if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2268 let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2269 "max attempts reached"
2270 } else {
2271 "non-transient error"
2272 };
2273 tracing::error!(
2274 attempt,
2275 %reason,
2276 kind,
2277 orch_webhook_in_fallback = true,
2278 "webhook server giving up, switching to polling fallback"
2279 );
2280 let to_save = {
2281 let mut s = status_for_spawn.lock().await;
2282 s.fallback_mode = true;
2283 s.healthy = false;
2284 s.last_failure_reason = Some(reason);
2285 s.startup_attempts = attempt;
2286 s.clone()
2287 };
2288 to_save.save().await;
2289 break;
2290 }
2291 let delay = webhook_backoff_delay(attempt);
2292 tracing::warn!(
2293 attempt,
2294 delay_ms = delay.as_millis(),
2295 %reason,
2296 "webhook bind failed (transient), retrying with backoff"
2297 );
2298 {
2299 let to_save = {
2300 let mut s = status_for_spawn.lock().await;
2301 s.startup_attempts = attempt;
2302 s.last_failure_reason = Some(reason);
2303 s.clone()
2304 };
2305 to_save.save().await;
2306 }
2307 time::sleep(delay).await;
2308 }
2309 }
2310 }
2311 });
2312
2313 let notify = Arc::clone(&webhook_notify);
2315 let transport_for_webhook = Arc::clone(&transport);
2316 tokio::spawn(async move {
2317 while let Some(msg) = rx.recv().await {
2318 tracing::info!(
2319 channel = %msg.channel,
2320 thread = %msg.thread_id,
2321 event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2322 action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2323 "webhook event received, triggering immediate tick"
2324 );
2325 let route = transport_for_webhook.route(&msg).await;
2326 tracing::info!(
2327 channel = %msg.channel,
2328 route = ?route,
2329 "webhook event routed"
2330 );
2331 notify.notify_one();
2333 }
2334 });
2335
2336 webhook_healthy = true;
2337 tracing::info!(port, orch_webhook_up = true, "webhook server started");
2338 } else {
2339 webhook_port = None;
2340 webhook_healthy = false;
2341 {
2342 let to_save = {
2343 let mut s = webhook_status.lock().await;
2344 s.configured = false;
2345 s.fallback_mode = true;
2346 s.clone()
2347 };
2348 to_save.save().await;
2349 }
2350 tracing::info!(
2351 orch_webhook_in_fallback = true,
2352 "webhook server disabled, using polling fallback mode"
2353 );
2354 }
2355
2356 start_relay_client(&project_engines, &webhook_notify);
2357 app::start_loader();
2358
2359 let router = Arc::new(RwLock::new(
2363 match task::spawn_blocking(Router::from_config).await {
2364 Ok(r) => r,
2365 Err(e) => {
2366 tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2367 task::spawn_blocking(|| Router::new(RouterConfig::default()))
2368 .await
2369 .unwrap_or_else(|e2| {
2370 panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2371 })
2372 }
2373 },
2374 ));
2375 {
2376 let r = router.read().await;
2377 tracing::info!(
2378 mode = %r.config.mode,
2379 agents = ?r.available_agents,
2380 fallback = %r.config.fallback_executor,
2381 "router initialized"
2382 );
2383 }
2384
2385 let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2389 let mut parallel_in_force = config.max_parallel;
2390 let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2393 let mut review_in_force = config.max_parallel;
2394
2395 let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2402
2403 let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2407
2408 let mut config_rx = config::subscribe();
2410
2411 let mut last_sync = Instant::now();
2413
2414 let mut last_upgrade_check: Option<Instant> = None;
2416
2417 let mut rate_limit_reported: Option<i64> = None;
2420
2421 let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2423
2424 for engine in project_engines.iter().filter(|_| leads) {
2426 let reset =
2427 reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2428
2429 if reset > 0 {
2430 tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2431 }
2432 }
2433
2434 for engine in &project_engines {
2436 subscribers::dispatch::spawn(
2437 event_bus.subscribe(),
2438 Arc::clone(&engine.backend),
2439 Arc::clone(&tmux),
2440 Arc::clone(&engine.runner),
2441 Arc::clone(&capture_for_tick),
2442 Arc::clone(&semaphore),
2443 Arc::clone(&engine.task_manager),
2444 weight_tx.clone(),
2445 Arc::clone(&router),
2446 Arc::clone(&dispatching),
2447 Arc::clone(&engine.store),
2448 engine.repo.clone(),
2449 );
2450 }
2451 tracing::info!("dispatch subscriber started");
2452
2453 for engine in &project_engines {
2455 subscribers::review::spawn(
2456 event_bus.subscribe(),
2457 Arc::clone(&engine.backend),
2458 Arc::clone(&tmux),
2459 Arc::clone(&review_semaphore),
2460 Arc::clone(&engine.task_manager),
2461 Arc::clone(&router),
2462 Arc::clone(&dispatching),
2463 Arc::clone(&engine.store),
2464 engine.repo.clone(),
2465 );
2466 }
2467 tracing::info!("review subscriber started");
2468
2469 for engine in &project_engines {
2471 subscribers::unblock::spawn(
2472 event_bus.subscribe(),
2473 Arc::clone(&engine.backend),
2474 Arc::clone(&engine.task_manager),
2475 Arc::clone(&engine.store),
2476 engine.repo.clone(),
2477 );
2478 }
2479 tracing::info!("unblock subscriber started");
2480
2481 subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2484 tracing::info!("notify subscriber started");
2485
2486 tracing::info!(
2488 tick = ?config.tick_interval,
2489 sync = ?config.sync_interval,
2490 parallel = config.max_parallel,
2491 "entering main loop"
2492 );
2493 let mut restart_only = restart_only_snapshot();
2496
2497 let mut interval = time::interval(config.tick_interval);
2498
2499 let mut sigterm = unix::signal(SignalKind::terminate())?;
2501 let mut sighup = unix::signal(SignalKind::hangup())?;
2502
2503 let sync_in_progress = Arc::new(AtomicBool::new(false));
2505
2506 let last_tick_epoch = Arc::new(AtomicU64::new(
2509 SystemTime::now()
2510 .duration_since(std_time::UNIX_EPOCH)
2511 .unwrap_or_default()
2512 .as_secs(),
2513 ));
2514 {
2515 let last_tick_epoch = Arc::clone(&last_tick_epoch);
2516 let tick_interval_secs = config.tick_interval.as_secs();
2517 tokio::spawn(async move {
2518 let mut watchdog_interval = time::interval(Duration::from_secs(30));
2520 let mut suspend_detector = SuspendDetector::new();
2525 loop {
2526 watchdog_interval.tick().await;
2527 let own_gap = suspend_detector.check();
2528 let last = last_tick_epoch.load(Ordering::Relaxed);
2529 let now = SystemTime::now()
2530 .duration_since(std_time::UNIX_EPOCH)
2531 .unwrap_or_default()
2532 .as_secs();
2533 let stale_secs = now.saturating_sub(last);
2534 let threshold = (tick_interval_secs * 6).max(60);
2536 if stale_secs > threshold {
2537 let shared_gap =
2544 engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2545 let gap = match (own_gap, shared_gap) {
2546 (Some(a), Some(b)) => Some(a.max(b)),
2547 (Some(a), None) => Some(a),
2548 (None, Some(b)) => Some(b),
2549 (None, None) => None,
2550 };
2551 if let Some(gap) = gap {
2552 let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2553 if gap_secs >= stale_secs {
2554 tracing::info!(
2555 stale_secs,
2556 threshold,
2557 suspend_gap_secs = gap_secs,
2558 "tick loop stale window explained by host suspend/resume, not a stall"
2559 );
2560 } else {
2561 tracing::error!(
2562 stale_secs,
2563 threshold,
2564 suspend_gap_secs = gap_secs,
2565 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2566 stale_secs,
2567 threshold,
2568 gap_secs,
2569 );
2570 }
2571 } else {
2572 tracing::error!(
2573 stale_secs,
2574 threshold,
2575 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2576 stale_secs,
2577 threshold,
2578 );
2579 }
2580 }
2581 }
2582 });
2583 }
2584
2585 loop {
2586 tokio::select! {
2587 _ = interval.tick() => {
2588 let tick_start = Instant::now();
2589
2590 if let Some(gap) = engine_suspend::checkpoint() {
2594 tracing::info!(
2595 gap_secs = gap.num_seconds(),
2596 "host suspend/resume gap detected, discounting from stuck-task age checks"
2597 );
2598 }
2599
2600 if review_in_force != config.max_parallel {
2604 review_in_force =
2605 resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2606 }
2607 if parallel_in_force != config.max_parallel {
2608 let resized =
2609 resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2610 if resized != parallel_in_force {
2611 parallel_in_force = resized;
2612 tracing::info!(
2613 in_force = parallel_in_force,
2614 target = config.max_parallel,
2615 "dispatch pool resized"
2616 );
2617 }
2618 }
2619
2620 {
2622 let mut rw = router.write().await;
2623 rw.tick_weight_recovery();
2624 }
2625
2626 if let Some(remaining) = GhHttp::is_rate_limited() {
2630 let reset_at = chrono::Utc::now()
2631 .timestamp()
2632 .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2633 if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2635 rate_limit_reported = Some(reset_at);
2636 report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2637 }
2638 } else if let Some(prev) = rate_limit_reported.take() {
2639 tracing::info!(
2640 limited_until = prev,
2641 "GitHub API rate limit cleared"
2642 );
2643 if let Some(engine) = project_engines.first() {
2644 let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2645 }
2646 }
2647
2648 tick::tick_all(
2650 &project_engines,
2651 &tmux,
2652 &capture_for_tick,
2653 &semaphore,
2654 &config,
2655 &router,
2656 &weight_tx,
2657 &dispatching,
2658 Some(&transport),
2659 &active_repos,
2660 &auto_merge_in_flight,
2661 ).await;
2662
2663 while let Ok(signal) = weight_rx.try_recv() {
2666 let mut rw = router.write().await;
2667 match signal {
2668 WeightSignal::RateLimited { ref agent } => {
2669 rw.record_rate_limit(agent);
2670 }
2671 WeightSignal::Success { ref agent } => {
2672 rw.record_success(agent);
2673 }
2674 WeightSignal::Failed { .. }
2675 | WeightSignal::Rerouted
2676 | WeightSignal::Blocked
2677 | WeightSignal::None => {}
2678 }
2679 }
2680
2681 if last_sync.elapsed() >= config.sync_interval {
2686 if sync_in_progress.compare_exchange(
2687 false,
2688 true,
2689 Ordering::AcqRel,
2690 Ordering::Relaxed,
2691 ).is_ok() {
2692 spawn_sync_pass(
2693 &project_engines,
2694 &tmux,
2695 &config,
2696 &router,
2697 &dispatching,
2698 &auto_merge_in_flight,
2699 &sync_in_progress,
2700 "periodic",
2701 );
2702 } else {
2703 tracing::debug!("sync tick still in progress, skipping");
2704 }
2705 last_sync = Instant::now();
2709 }
2710
2711 if webhook_enabled {
2713 let health_check_interval = config.webhook_health_check_interval
2714 .unwrap_or(Duration::from_secs(60));
2715
2716 if last_webhook_health_check.elapsed() >= health_check_interval {
2717 if let Some(port) = webhook_port {
2718 let (health, failure_reason) =
2719 github::check_webhook_health(port).await;
2720 if health != webhook_healthy {
2721 webhook_healthy = health;
2722 if webhook_healthy {
2723 tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2724 } else {
2725 tracing::warn!(
2726 port,
2727 orch_webhook_up = false,
2728 reason = failure_reason.as_deref().unwrap_or("unknown"),
2729 "webhook health check failed"
2730 );
2731 }
2732 }
2733 {
2735 let to_save = {
2736 let mut s = webhook_status.lock().await;
2737 s.healthy = health;
2738 s.last_check_utc = Some(chrono::Utc::now());
2739 if health {
2740 s.last_failure_reason = None;
2741 } else if failure_reason.is_some() {
2742 s.last_failure_reason = failure_reason;
2743 }
2744 s.clone()
2745 };
2746 to_save.save().await;
2747 }
2748 }
2749 last_webhook_health_check = Instant::now();
2750 }
2751 }
2752
2753 if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2755 if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2756 let channels = Arc::clone(&channel_registry);
2757 let current = env!("ORCH_VERSION").to_string();
2758 let check_interval =
2759 Duration::from_secs(config.upgrade_check_interval);
2760 tokio::spawn(async move {
2761 check_and_notify_upgrade(&store, &channels, ¤t, check_interval)
2762 .await;
2763 });
2764 }
2765 last_upgrade_check = Some(Instant::now());
2766 }
2767
2768 let tick_elapsed = tick_start.elapsed();
2770 if tick_elapsed.as_secs() > 30 {
2771 tracing::warn!(
2772 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2773 "slow tick"
2774 );
2775 } else {
2776 tracing::debug!(
2777 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2778 "tick complete"
2779 );
2780 }
2781 last_tick_epoch.store(
2782 SystemTime::now()
2783 .duration_since(std_time::UNIX_EPOCH)
2784 .unwrap_or_default()
2785 .as_secs(),
2786 Ordering::Relaxed,
2787 );
2788 }
2789 () = webhook_notify.notified() => {
2791 if let Some(remaining) = GhHttp::is_rate_limited() {
2792 tracing::warn!(
2793 remaining_secs = remaining.as_secs(),
2794 "GitHub API rate-limited, skipping webhook-triggered tick"
2795 );
2796 } else {
2797 tracing::info!("webhook event triggered immediate tick");
2798
2799 tick::tick_all(
2800 &project_engines,
2801 &tmux,
2802 &capture_for_tick,
2803 &semaphore,
2804 &config,
2805 &router,
2806 &weight_tx,
2807 &dispatching,
2808 Some(&transport),
2809 &active_repos,
2810 &auto_merge_in_flight,
2811 ).await;
2812
2813 while let Ok(signal) = weight_rx.try_recv() {
2815 let mut rw = router.write().await;
2816 match signal {
2817 WeightSignal::RateLimited { ref agent } => {
2818 rw.record_rate_limit(agent);
2819 }
2820 WeightSignal::Success { ref agent } => {
2821 rw.record_success(agent);
2822 }
2823 WeightSignal::Failed { .. }
2824 | WeightSignal::Rerouted
2825 | WeightSignal::Blocked
2826 | WeightSignal::None => {}
2827 }
2828 }
2829
2830 if sync_in_progress.compare_exchange(
2836 false,
2837 true,
2838 Ordering::AcqRel,
2839 Ordering::Relaxed,
2840 ).is_ok() {
2841 spawn_sync_pass(
2842 &project_engines,
2843 &tmux,
2844 &config,
2845 &router,
2846 &dispatching,
2847 &auto_merge_in_flight,
2848 &sync_in_progress,
2849 "webhook",
2850 );
2851 last_sync = Instant::now();
2853 } else {
2854 tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2855 }
2856 }
2857
2858 interval.reset();
2860 }
2861 result = config_rx.recv() => {
2862 let reload = match result {
2865 Ok(path) => {
2866 tracing::info!(path = %path.display(), "config file changed, reloading");
2867 true
2868 }
2869 Err(RecvError::Lagged(n)) => {
2870 tracing::warn!(missed = n, "config change receiver lagged, reloading");
2871 true
2872 }
2873 Err(RecvError::Closed) => {
2874 tracing::warn!("config change channel closed");
2875 false
2876 }
2877 };
2878
2879 if reload {
2880 let new_config = EngineConfig::from_config();
2882 log_engine_config_changes(&config, &new_config);
2883
2884 let new_restart_only = restart_only_snapshot();
2885 log_restart_only_changes(&restart_only, &new_restart_only);
2886 restart_only = new_restart_only;
2887
2888 let tick_changed = new_config.tick_interval != config.tick_interval;
2889 config = new_config;
2890
2891 if tick_changed {
2893 interval = time::interval(config.tick_interval);
2894 tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2895 }
2896
2897 if review_in_force != config.max_parallel {
2900 review_in_force = resize_dispatch_pool(
2901 &review_semaphore,
2902 review_in_force,
2903 config.max_parallel,
2904 );
2905 }
2906 if parallel_in_force != config.max_parallel {
2907 parallel_in_force = resize_dispatch_pool(
2908 &semaphore,
2909 parallel_in_force,
2910 config.max_parallel,
2911 );
2912 tracing::info!(
2913 in_force = parallel_in_force,
2914 target = config.max_parallel,
2915 "dispatch pool resized"
2916 );
2917 }
2918
2919 {
2921 let mut router_guard = router.write().await;
2922 router_guard.reload_async().await;
2923 }
2924
2925 sync_project_engines(
2927 &mut project_engines,
2928 &mut active_repos,
2929 &event_bus,
2930 &tmux,
2931 &capture_for_tick,
2932 &semaphore,
2933 &review_semaphore,
2934 &weight_tx,
2935 &router,
2936 &dispatching,
2937 ).await;
2938
2939 tracing::info!(
2940 tick = ?config.tick_interval,
2941 sync = ?config.sync_interval,
2942 parallel = config.max_parallel,
2943 projects = project_engines.len(),
2944 "config reloaded"
2945 );
2946 }
2947 }
2948 signal_name = async {
2949 tokio::select! {
2950 _ = signal::ctrl_c() => "SIGINT",
2951 _ = sigterm.recv() => "SIGTERM",
2952 _ = sighup.recv() => "SIGHUP",
2953 }
2954 } => {
2955 tracing::info!(signal = signal_name, "beginning graceful shutdown");
2956
2957 let shutdown_result = time::timeout(
2960 config.graceful_shutdown_timeout,
2961 async {
2962 let mut reset_count = 0u32;
2966 let mut review_reset_count = 0u32;
2967 for engine in &project_engines {
2968 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
2969 for task in &tasks {
2970 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
2971 tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
2972 } else {
2973 if let Ok(Some(store_id)) =
2974 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
2975 && let Err(e) = engine
2976 .store
2977 .finalize_incomplete_runs(
2978 store_id,
2979 "aborted",
2980 "graceful shutdown: reset in_progress task to routed",
2981 )
2982 .await
2983 {
2984 tracing::warn!(
2985 task_id = task.id.0,
2986 ?e,
2987 "failed to finalize incomplete runs on shutdown"
2988 );
2989 }
2990 reset_count += 1;
2991 }
2992 }
2993 }
2994 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
2996 for task in &tasks {
2997 let task_id = format!("internal:{}", task.id);
2998 if let Err(e) = engine.task_manager.update_task_status(
2999 &ExternalId(task_id.clone()),
3000 Status::Routed,
3001 ).await {
3002 tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3003 } else {
3004 if let Err(e) = engine
3005 .store
3006 .finalize_incomplete_runs(
3007 task.id,
3008 "aborted",
3009 "graceful shutdown: reset in_progress task to routed",
3010 )
3011 .await
3012 {
3013 tracing::warn!(
3014 task_id,
3015 ?e,
3016 "failed to finalize incomplete internal runs on shutdown"
3017 );
3018 }
3019 reset_count += 1;
3020 }
3021 }
3022 }
3023 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3025 for task in &tasks {
3026 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3027 tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3028 } else {
3029 if let Ok(Some(store_id)) =
3030 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3031 && let Err(e) = engine
3032 .store
3033 .finalize_incomplete_runs(
3034 store_id,
3035 "aborted",
3036 "graceful shutdown: reset in_review task to needs_review",
3037 )
3038 .await
3039 {
3040 tracing::warn!(
3041 task_id = task.id.0,
3042 ?e,
3043 "failed to finalize incomplete review runs on shutdown"
3044 );
3045 }
3046 review_reset_count += 1;
3047 }
3048 }
3049 }
3050 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3052 for task in &tasks {
3053 let task_id = format!("internal:{}", task.id);
3054 if let Err(e) = engine.task_manager.update_task_status(
3055 &ExternalId(task_id.clone()),
3056 Status::NeedsReview,
3057 ).await {
3058 tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3059 } else {
3060 if let Err(e) = engine
3061 .store
3062 .finalize_incomplete_runs(
3063 task.id,
3064 "aborted",
3065 "graceful shutdown: reset in_review task to needs_review",
3066 )
3067 .await
3068 {
3069 tracing::warn!(
3070 task_id,
3071 ?e,
3072 "failed to finalize incomplete internal review runs on shutdown"
3073 );
3074 }
3075 review_reset_count += 1;
3076 }
3077 }
3078 }
3079 }
3080 if reset_count > 0 {
3081 tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3082 }
3083 if review_reset_count > 0 {
3084 tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3085 }
3086
3087 let mut killed_sessions = Vec::new();
3090 if let Ok(sessions) = tmux.list_sessions().await {
3091 for session in &sessions {
3092 if session.name.starts_with("orch-") {
3093 match tmux.kill_session(&session.name).await { Err(e) => {
3094 tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3095 } _ => {
3096 killed_sessions.push(session.name.clone());
3097 }}
3098 }
3099 }
3100 if !killed_sessions.is_empty() {
3101 tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3102 }
3103 }
3104
3105 if !killed_sessions.is_empty() {
3108 let still_alive = tmux
3109 .wait_for_sessions_dead(
3110 &killed_sessions,
3111 Duration::from_millis(100),
3112 Duration::from_secs(5),
3113 )
3114 .await;
3115 if still_alive > 0 {
3116 tracing::warn!(
3117 still_alive,
3118 "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3119 );
3120 }
3121 tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3122 }
3123 },
3124 ).await;
3125
3126 if shutdown_result.is_err() {
3127 tracing::error!(
3128 timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3129 "graceful shutdown timed out, forcing exit"
3130 );
3131 }
3132
3133 break;
3134 }
3135 }
3136 }
3137
3138 events::cleanup_port_file().await;
3140 local::cleanup().await;
3141 events::cleanup_version_file();
3142
3143 let _ = transport;
3145 tracing::info!("orch engine stopped");
3146 Ok(())
3147}
3148
3149#[cfg(test)]
3150mod tests {
3151 use std::sync::Arc;
3152
3153 use super::*;
3154 use crate::backends::test_helpers::NoopBackend;
3155 use crate::channels::capture::CaptureService;
3156 use crate::channels::transport::Transport;
3157 use crate::tmux::SessionGuard;
3158 use crate::tmux::TmuxManager;
3159
3160 async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3161 let id = store
3162 .upsert_external(&crate::store::UpsertExternal {
3163 repo: "owner/repo",
3164 ext_id: number,
3165 title: "Under review",
3166 body: "",
3167 author: "",
3168 url: "",
3169 labels: &[],
3170 origin: "github",
3171 })
3172 .await
3173 .unwrap();
3174 store.update_status(id, TaskStatus::InReview).await.unwrap();
3175 id
3176 }
3177
3178 #[tokio::test]
3179 async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3180 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3181 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3182 let task_manager =
3183 TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3184 let tmux = TmuxManager::isolated();
3185 let orphaned = in_review_task(&store, "11").await;
3186 let live = in_review_task(&store, "12").await;
3187 let session = tmux
3188 .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3189 .await
3190 .unwrap();
3191 let _guard = SessionGuard(session);
3192
3193 let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3194
3195 assert_eq!(reset, 1);
3196 assert_eq!(
3197 store.get(orphaned).await.unwrap().status,
3198 TaskStatus::NeedsReview
3199 );
3200 assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3201 }
3202
3203 #[test]
3204 fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3205 let semaphore = Arc::new(Semaphore::new(4));
3206 let held: Vec<_> = (0..4)
3207 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3208 .collect();
3209 assert!(
3210 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3211 "pool is full"
3212 );
3213
3214 let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3215 assert_eq!(in_force, 8);
3216
3217 let grown: Vec<_> = (0..4)
3218 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3219 .collect();
3220 assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3221 assert!(
3222 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3223 "pool is full"
3224 );
3225 }
3226
3227 #[test]
3228 fn shrinking_max_parallel_never_cancels_a_running_task() {
3229 let semaphore = Arc::new(Semaphore::new(8));
3230 let mut held: Vec<_> = (0..8)
3231 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3232 .collect();
3233
3234 let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3236 assert_eq!(in_force, 8);
3237 assert_eq!(held.len(), 8);
3238
3239 held.truncate(5);
3241 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3242 assert_eq!(in_force, 5);
3243 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3244
3245 held.truncate(4);
3247 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3248 assert_eq!(in_force, 4);
3249 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3250
3251 held.truncate(3);
3253 assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3254 }
3255
3256 #[test]
3257 fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3258 let configured = vec![
3259 ("owner/one".to_string(), PathBuf::from("/srv/one")),
3260 ("owner/two".to_string(), PathBuf::from("/srv/two")),
3261 ];
3262 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3263
3264 let (added, removed) = diff_configured_projects(&configured, &active);
3265 assert_eq!(
3266 added,
3267 vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3268 "only the new project is started, so owner/one keeps its single set of subscribers"
3269 );
3270 assert!(removed.is_empty(), "{removed:?}");
3271 }
3272
3273 #[test]
3274 fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3275 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3276 let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3277 .into_iter()
3278 .collect();
3279
3280 let (added, removed) = diff_configured_projects(&configured, &active);
3281 assert!(added.is_empty(), "nothing new to start");
3282 assert_eq!(removed, vec!["owner/gone".to_string()]);
3283 }
3284
3285 #[test]
3286 fn an_unchanged_project_list_starts_nothing() {
3287 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3288 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3289
3290 assert_eq!(
3291 diff_configured_projects(&configured, &active),
3292 (vec![], vec![])
3293 );
3294 }
3295
3296 #[test]
3297 fn restart_only_keys_are_the_settings_read_once_at_startup() {
3298 assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3301 }
3302
3303 #[test]
3304 fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3305 let before = vec![Some("bot_token: a\n".to_string()), None];
3306
3307 assert!(
3308 log_restart_only_changes(&before, &before.clone()).is_empty(),
3309 "a reload that did not touch these keys must stay quiet"
3310 );
3311
3312 let after = vec![
3314 Some("bot_token: b\n".to_string()),
3315 Some("port: 8080\n".to_string()),
3316 ];
3317 assert_eq!(
3318 log_restart_only_changes(&before, &after),
3319 ["channels", "webhook"]
3320 );
3321
3322 let token_only = vec![Some("bot_token: b\n".to_string()), None];
3324 assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3325 }
3326
3327 #[test]
3328 fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3329 let semaphore = Arc::new(Semaphore::new(4));
3330 assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3331 assert_eq!(semaphore.available_permits(), 4);
3332 }
3333
3334 #[test]
3335 fn live_session_stuck_limit_follows_the_run_timeout() {
3336 let config = EngineConfig {
3337 stuck_timeout: 1800,
3338 run_timeouts: RunTimeouts {
3339 base: 1800,
3340 medium: 2700,
3341 complex: 3600,
3342 },
3343 ..EngineConfig::default()
3344 };
3345 assert_eq!(
3346 config.live_session_stuck_limit(Some("simple")),
3347 1800 + RUN_TIMEOUT_GRACE_SECS
3348 );
3349 assert_eq!(
3350 config.live_session_stuck_limit(Some("medium")),
3351 2700 + RUN_TIMEOUT_GRACE_SECS
3352 );
3353 assert_eq!(
3354 config.live_session_stuck_limit(Some("complex")),
3355 3600 + RUN_TIMEOUT_GRACE_SECS
3356 );
3357
3358 let long_stuck = EngineConfig {
3359 stuck_timeout: 7200,
3360 ..config
3361 };
3362 assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3363 }
3364
3365 #[test]
3366 fn engine_config_defaults() {
3367 let config = EngineConfig::default();
3368 assert_eq!(config.tick_interval, Duration::from_secs(10));
3369 assert_eq!(config.sync_interval, Duration::from_secs(45));
3370 assert_eq!(config.max_parallel, 4);
3371 assert_eq!(config.stuck_timeout, 1800);
3372 assert_eq!(config.no_session_stuck_timeout, 600);
3373 assert!(
3374 config.no_session_stuck_timeout < config.stuck_timeout,
3375 "no_session_stuck_timeout must be shorter than stuck_timeout"
3376 );
3377 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3378 assert_eq!(config.upgrade_check_interval, 3600);
3379 }
3380
3381 #[test]
3382 fn engine_config_from_config_uses_defaults_when_no_config() {
3383 let config = EngineConfig::from_config();
3384 assert_eq!(config.tick_interval, Duration::from_secs(10));
3385 assert_eq!(config.sync_interval, Duration::from_secs(45));
3386 assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3388 assert_eq!(config.stuck_timeout, 1800);
3389 assert_eq!(config.no_session_stuck_timeout, 600);
3390 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3391 assert_eq!(config.upgrade_check_interval, 3600);
3392 }
3393
3394 #[test]
3395 fn engine_config_no_session_timeout_is_under_15_minutes() {
3396 let config = EngineConfig::default();
3397 assert!(
3398 config.no_session_stuck_timeout <= 900,
3399 "no_session_stuck_timeout ({}) exceeds 15 minutes",
3400 config.no_session_stuck_timeout
3401 );
3402 }
3403
3404 #[test]
3405 fn upgrade_check_runs_immediately_when_never_checked() {
3406 assert!(should_run_upgrade_check(None, 3600));
3407 }
3408
3409 #[test]
3410 fn upgrade_check_waits_for_interval_after_first_run() {
3411 assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3412 }
3413
3414 #[test]
3415 fn upgrade_check_can_be_disabled() {
3416 assert!(!should_run_upgrade_check(None, 0));
3417 }
3418
3419 #[test]
3420 fn configured_agents_parses_yaml_block_array() {
3421 let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3423 let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3424 assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3425 }
3426
3427 #[test]
3428 fn configured_agents_parses_json_inline_array() {
3429 let json_str = r#"["claude","codex","olm"]"#;
3430 let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3431 assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3432 }
3433
3434 #[test]
3435 fn configured_agents_fallback_returns_default_agents() {
3436 let defaults = configured_agents();
3438 assert!(defaults.contains(&"claude".to_string()));
3439 assert!(defaults.contains(&"codex".to_string()));
3440 assert!(defaults.contains(&"opencode".to_string()));
3441 }
3442
3443 #[tokio::test]
3452 #[ignore = "capture-pane timing is unreliable on CI runners"]
3453 async fn integration_channel_to_tmux_to_capture() {
3454 let transport = Arc::new(Transport::new());
3456 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3457 let tmux = TmuxManager::new();
3458
3459 let task_id = "testtask1".to_string();
3460 let session_name = format!("orch-test-{task_id}");
3461
3462 let _ = tmux
3464 .command()
3465 .args(["new-session", "-d", "-s", &session_name])
3466 .output()
3467 .await
3468 .expect("failed to start tmux session");
3469
3470 let _guard = SessionGuard(session_name.clone());
3472
3473 capture
3475 .register_session("owner/repo", &task_id, &session_name)
3476 .await;
3477 transport
3478 .bind(
3479 "owner/repo",
3480 &task_id,
3481 &session_name,
3482 "telegram",
3483 "12345",
3484 None,
3485 )
3486 .await;
3487
3488 let capture_clone = Arc::clone(&capture);
3490 let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3491
3492 let mut rx = transport
3494 .subscribe("owner/repo", &task_id)
3495 .await
3496 .expect("no subscription");
3497
3498 let send = tmux
3500 .command()
3501 .args([
3502 "send-keys",
3503 "-t",
3504 &session_name,
3505 "echo from-channel",
3506 "Enter",
3507 ])
3508 .output()
3509 .await
3510 .expect("failed to send keys");
3511 assert!(send.status.success());
3512
3513 let mut got = false;
3515 for _ in 0..20 {
3516 if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3517 && c.content.contains("from-channel")
3518 {
3519 got = true;
3520 break;
3521 }
3522 }
3523
3524 capture.unregister_session("owner/repo", &task_id).await;
3526
3527 let _ = capture_handle.await;
3529
3530 assert!(got, "did not observe tmux output via capture/transport");
3531 }
3532
3533 #[test]
3536 fn read_project_channel_config_returns_defaults_when_no_file() {
3537 let dir = tempfile::tempdir().expect("tempdir");
3538 let cfg = read_project_channel_config(dir.path());
3539 assert!(cfg.telegram_topic_id.is_none());
3540 assert!(cfg.discord_channel_id.is_none());
3541 }
3542
3543 #[test]
3544 fn read_project_channel_config_parses_telegram_and_discord() {
3545 let dir = tempfile::tempdir().expect("tempdir");
3546 let orch_yml = dir.path().join(".orch.yml");
3547 fs::write(
3548 &orch_yml,
3549 "channels:\n telegram:\n topic_id: \"42\"\n discord:\n channel_id: \"9999\"\n",
3550 )
3551 .expect("write .orch.yml");
3552 let cfg = read_project_channel_config(dir.path());
3553 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3554 assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3555 }
3556
3557 #[test]
3558 fn read_project_channel_config_handles_partial_channels() {
3559 let dir = tempfile::tempdir().expect("tempdir");
3560 let orch_yml = dir.path().join(".orch.yml");
3561 fs::write(&orch_yml, "channels:\n telegram:\n topic_id: \"7\"\n")
3562 .expect("write .orch.yml");
3563 let cfg = read_project_channel_config(dir.path());
3564 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3565 assert!(cfg.discord_channel_id.is_none());
3566 }
3567
3568 #[test]
3569 fn read_project_channel_config_handles_malformed_yaml() {
3570 let dir = tempfile::tempdir().expect("tempdir");
3571 let orch_yml = dir.path().join(".orch.yml");
3572 fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3573 let cfg = read_project_channel_config(dir.path());
3574 assert!(cfg.telegram_topic_id.is_none());
3576 assert!(cfg.discord_channel_id.is_none());
3577 }
3578
3579 #[test]
3580 #[cfg(unix)]
3581 fn read_project_channel_config_handles_unreadable_file() {
3582 let dir = tempfile::tempdir().expect("tempdir");
3583 fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3585 let cfg = read_project_channel_config(dir.path());
3586 assert!(cfg.telegram_topic_id.is_none());
3588 assert!(cfg.discord_channel_id.is_none());
3589 }
3590}