1pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod relay;
27pub mod review;
28pub mod review_gate;
29pub mod review_poll;
30pub mod review_shipcheck;
31pub mod router;
32pub mod runner;
33pub mod subscribers;
34pub mod suspend;
35pub mod sync;
36pub mod tasks;
37pub mod tick;
38
39use crate::backends::{ExternalBackend, ExternalId};
44use crate::channels::capture::CaptureService;
45use crate::channels::discord_ws::DiscordGateway;
46use crate::channels::github::start_webhook_server;
47use crate::channels::notification::NotificationLevel;
48use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
49use crate::channels::slack::SlackChannel;
50use crate::channels::telegram::TelegramChannel;
51use crate::channels::tmux::TmuxChannel;
52use crate::channels::transport::Transport;
53use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
54use crate::cmd::CommandErrorContext;
55use crate::config;
56use crate::engine::cleanup::{remove_worktree_and_branch, remove_worktree_and_branch_for};
57use crate::engine::router::{Router, RouterConfig};
58use crate::engine::runner::worktree::{
59 abort_worktree_rebase, list_project_worktrees, rebase_worktree_on_origin_main,
60 task_id_from_worktree_name, validate_worktree_gitdir,
61};
62use crate::engine::tasks::TaskManager;
63use crate::github::http::{GhHttp, rate_limit_metrics};
64use crate::repo_context::REPO_CONTEXT;
65use crate::store::{TaskStatus, TaskStore};
66use crate::tmux::TmuxManager;
67use runner::WeightSignal;
68use crate::backends::Status;
70use crate::backends::github::GitHubBackend;
71use crate::channels::github;
72use crate::channels::github::is_transient_bind_error;
73use crate::channels::github::webhook_backoff_delay;
74use crate::channels::notification;
75use crate::channels::notification::TaskNotification;
76use crate::channels::transport;
77use crate::cluster;
78use crate::cluster::local;
79use crate::engine::cooldown as engine_cooldown;
80use crate::engine::review as engine_review;
81use crate::engine::runner::git_ops;
82use crate::engine::runner::worktree;
83use crate::engine::suspend as engine_suspend;
84use crate::engine::suspend::SuspendDetector;
85use crate::engine::tasks as engine_tasks;
86use crate::github::app;
87use crate::home;
88use crate::secret_ref;
89use crate::store;
90use crate::webhook_status::WebhookStatus;
91use futures::future;
92use std::collections::HashMap;
93use std::collections::HashSet;
94use std::fmt;
95use std::fmt::Debug;
96use std::fmt::Formatter;
97use std::fs;
98use std::path::Path;
99use std::path::PathBuf;
100use std::sync::Arc;
101use std::sync::atomic::AtomicBool;
102use std::sync::atomic::AtomicU64;
103use std::sync::atomic::Ordering;
104use std::time as std_time;
105use std::time::Duration;
106use std::time::Instant;
107use std::time::SystemTime;
108use tokio::process::Command;
109use tokio::signal;
110use tokio::signal::unix;
111use tokio::signal::unix::SignalKind;
112use tokio::sync::Mutex;
113use tokio::sync::broadcast::Sender;
114use tokio::sync::broadcast::error::RecvError;
115use tokio::sync::mpsc::Receiver;
116use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
117use tokio::task;
118use tokio::time;
119
120pub fn orch_footer() -> &'static str {
121 "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
122}
123
124pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
128 match model {
129 Some(m) => format!(
130 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
131 ),
132 None => format!(
133 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
134 ),
135 }
136}
137
138type EngineRef = (
140 String,
141 Arc<dyn ExternalBackend>,
142 Arc<TaskManager>,
143 Option<Arc<TaskStore>>,
144);
145
146#[derive(Clone)]
151pub struct ProjectEngine {
152 pub repo: String,
153 pub project_dir: PathBuf,
154 pub backend: Arc<dyn ExternalBackend>,
155 pub task_manager: Arc<TaskManager>,
156 pub runner: Arc<runner::TaskRunner>,
157 pub store: Arc<TaskStore>,
158}
159
160impl Debug for ProjectEngine {
161 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
162 f.debug_struct("ProjectEngine")
163 .field("repo", &self.repo)
164 .field("project_dir", &self.project_dir)
165 .finish_non_exhaustive()
166 }
167}
168
169pub(crate) async fn reset_orphaned_reviews(
172 task_manager: &TaskManager,
173 store: &Arc<TaskStore>,
174 repo: &str,
175 tmux: &TmuxManager,
176) -> usize {
177 let mut ids: Vec<ExternalId> = store
179 .list_by_status(repo, TaskStatus::InReview)
180 .await
181 .unwrap_or_default()
182 .iter()
183 .map(|t| engine_tasks::store_task_to_external(t).id)
184 .collect();
185 if let Ok(legacy) = task_manager
186 .list_internal_by_status(TaskStatus::InReview)
187 .await
188 {
189 ids.extend(legacy.into_iter().map(|t| t.id));
190 }
191 ids.sort_by(|a, b| a.0.cmp(&b.0));
192 ids.dedup_by(|a, b| a.0 == b.0);
193
194 let mut reset = 0;
195 for id in &ids {
196 let session = tmux.session_name(repo, &format!("{}-review", id.0));
197
198 if tmux.session_exists(&session).await {
199 continue;
200 }
201
202 if let Err(e) = task_manager
203 .update_task_status(id, Status::NeedsReview)
204 .await
205 {
206 tracing::warn!(
207 task_id = id.0,
208 ?e,
209 "failed to reset an orphaned review at start"
210 );
211 continue;
212 }
213
214 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
215 let reason = "startup: review session gone, reset to needs_review";
216
217 if let Err(e) = store
218 .finalize_incomplete_runs(store_id, "aborted", reason)
219 .await
220 {
221 tracing::warn!(
222 task_id = id.0,
223 ?e,
224 "failed to finalize the orphaned review's runs"
225 );
226 }
227 }
228
229 reset += 1;
230 }
231
232 reset
233}
234
235fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
238 let url = config::get_global("relay.url").unwrap_or_default();
240 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
241 return;
242 };
243
244 if url.is_empty() {
245 return;
246 }
247
248 let token_ref = config::get_global("relay.token").unwrap_or_default();
249 let notify = Arc::clone(notify);
250 tokio::spawn(async move {
251 let mut warned = false;
252 let token = loop {
253 let reference = token_ref.clone();
254 let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
255 .await
256 .map_err(anyhow::Error::from)
257 .and_then(|r| r);
258
259 match read {
260 Ok(token) if !token.is_empty() => break token,
261 Ok(_) => {
262 tracing::warn!("relay.token is empty, the relay client stays off");
263 return;
264 }
265 Err(e) if !warned => {
266 tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
267 warned = true;
268 }
269 Err(_) => {}
270 }
271
272 time::sleep(Duration::from_secs(300)).await;
273 };
274
275 relay::run(url, token, store, notify).await;
276 });
277}
278
279#[expect(
285 clippy::too_many_arguments,
286 reason = "the spawned task takes its own handle to each piece of engine state"
287)]
288fn spawn_sync_pass(
289 project_engines: &[ProjectEngine],
290 tmux: &Arc<TmuxManager>,
291 engine_config: &EngineConfig,
292 router: &Arc<RwLock<Router>>,
293 dispatching: &Arc<dashmap::DashMap<String, String>>,
294 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
295 sync_in_progress: &Arc<AtomicBool>,
296 trigger: &'static str,
297) {
298 let engines = project_engines.to_vec();
299 let tmux = Arc::clone(tmux);
300 let engine_config = engine_config.clone();
301 let router = Arc::clone(router);
302 let dispatching = Arc::clone(dispatching);
303 let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
304 let guard = Arc::clone(sync_in_progress);
305
306 tokio::spawn(async move {
307 let started = Instant::now();
308 let mut passes = Vec::with_capacity(engines.len());
309 for engine in &engines {
310 passes.push(REPO_CONTEXT.scope(
311 engine.repo.clone(),
312 sync_one_project(
313 engine,
314 &tmux,
315 &engine_config,
316 &router,
317 &dispatching,
318 &auto_merge_in_flight,
319 ),
320 ));
321 }
322 future::join_all(passes).await;
323
324 sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
325 tracing::info!(
326 trigger,
327 elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
328 "sync tick complete"
329 );
330 guard.store(false, Ordering::Release);
331 });
332}
333
334async fn sync_one_project(
335 engine: &ProjectEngine,
336 tmux: &Arc<TmuxManager>,
337 engine_config: &EngineConfig,
338 router: &Arc<RwLock<Router>>,
339 dispatching: &Arc<dashmap::DashMap<String, String>>,
340 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
341) {
342 if let Err(e) = Box::pin(sync::sync_tick(
343 &engine.backend,
344 tmux,
345 &engine.repo,
346 engine_config,
347 router,
348 &engine.task_manager,
349 &engine.store,
350 dispatching,
351 auto_merge_in_flight,
352 ))
353 .await
354 {
355 tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
356 }
357}
358
359const SLOW_PHASE_GROUP_MS: u128 = 1_000;
361
362pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
368 let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
369 if total_ms >= SLOW_PHASE_GROUP_MS {
370 tracing::info!(group, repo, total_ms, ?phases, "phase timings");
371 } else {
372 tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
373 }
374}
375
376pub fn configured_agents() -> Vec<String> {
388 if let Ok(agents_str) = config::get("agents")
389 && !agents_str.is_empty()
390 && agents_str != "[]"
391 {
392 if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
395 && !agents_arr.is_empty()
396 {
397 return agents_arr;
398 }
399 if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
401 && !agents_arr.is_empty()
402 {
403 return agents_arr;
404 }
405 }
406 router::config::DEFAULT_AGENTS
407 .iter()
408 .map(ToString::to_string)
409 .collect()
410}
411
412pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
415
416#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418pub struct RunTimeouts {
419 pub base: u64,
420 pub medium: u64,
421 pub complex: u64,
422}
423
424impl Default for RunTimeouts {
425 fn default() -> Self {
426 Self {
427 base: 1800,
428 medium: 1800,
429 complex: 1800,
430 }
431 }
432}
433
434impl RunTimeouts {
435 pub fn from_config() -> Self {
438 let read =
439 |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
440 let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
441 Self {
442 base,
443 medium: read("workflow.timeout_by_complexity.medium")
444 .unwrap_or(base)
445 .max(base),
446 complex: read("workflow.timeout_by_complexity.complex")
447 .unwrap_or(base)
448 .max(base),
449 }
450 }
451
452 pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
453 match complexity {
454 Some("complex") => self.complex,
455 Some("medium") => self.medium,
456 _ => self.base,
457 }
458 }
459}
460
461#[derive(Clone, Debug)]
463pub struct EngineConfig {
464 pub tick_interval: Duration,
466 pub sync_interval: Duration,
468 pub webhook_health_check_interval: Option<Duration>,
470 pub max_parallel: usize,
472 pub stuck_timeout: u64,
474 pub no_session_stuck_timeout: u64,
477 pub in_review_no_session_stuck_timeout: u64,
484 pub run_timeouts: RunTimeouts,
486 pub auto_create_followup_on_changes: bool,
488 pub graceful_shutdown_timeout: Duration,
490 pub silence_grace_period: u64,
493 pub silence_cooldown: u64,
495 pub upgrade_check_interval: u64,
498 pub startup_failure_escalation_secs: u64,
503}
504
505impl Default for EngineConfig {
506 fn default() -> Self {
507 Self {
508 tick_interval: Duration::from_secs(10),
509 sync_interval: Duration::from_secs(45),
510 webhook_health_check_interval: Some(Duration::from_secs(60)),
511 max_parallel: 4,
512 stuck_timeout: 1800,
513 no_session_stuck_timeout: 600,
514 in_review_no_session_stuck_timeout: 1920,
515 run_timeouts: RunTimeouts::default(),
516 auto_create_followup_on_changes: true,
517 graceful_shutdown_timeout: Duration::from_secs(600),
518 silence_grace_period: 300,
519 silence_cooldown: 3600,
520 upgrade_check_interval: 3600,
521 startup_failure_escalation_secs: 3600,
522 }
523 }
524}
525
526impl EngineConfig {
527 pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
530 self.stuck_timeout.max(
531 self.run_timeouts
532 .for_complexity(complexity)
533 .saturating_add(RUN_TIMEOUT_GRACE_SECS),
534 )
535 }
536
537 pub fn from_config() -> Self {
539 let mut config = Self::default();
540
541 if let Ok(val) = config::get("engine.tick_interval") {
542 if let Ok(secs) = val.parse::<u64>() {
543 config.tick_interval = Duration::from_secs(secs);
544 } else {
545 tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
546 }
547 }
548
549 if let Ok(val) = config::get("engine.sync_interval") {
550 if let Ok(secs) = val.parse::<u64>() {
551 config.sync_interval = Duration::from_secs(secs);
552 } else {
553 tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
554 }
555 }
556
557 if let Ok(val) = config::get("engine.max_parallel") {
558 if let Ok(n) = val.parse::<usize>() {
559 config.max_parallel = n;
560 } else {
561 tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
562 }
563 }
564
565 if let Ok(val) = config::get("engine.stuck_timeout") {
566 if let Ok(secs) = val.parse::<u64>() {
567 config.stuck_timeout = secs;
568 } else {
569 tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
570 }
571 }
572
573 if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
574 if let Ok(secs) = val.parse::<u64>() {
575 config.no_session_stuck_timeout = secs;
576 } else {
577 tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
578 }
579 }
580
581 let mut in_review_from_config = false;
582 if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
583 if let Ok(secs) = val.parse::<u64>() {
584 config.in_review_no_session_stuck_timeout = secs;
585 in_review_from_config = true;
586 } else {
587 tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
588 }
589 }
590
591 let review_timeout = engine_review::review_timeout_seconds();
595 let floored = engine_review::no_session_review_floor(
596 config.in_review_no_session_stuck_timeout,
597 review_timeout,
598 );
599 if floored != config.in_review_no_session_stuck_timeout {
600 if in_review_from_config {
601 tracing::warn!(
602 configured_secs = config.in_review_no_session_stuck_timeout,
603 review_timeout_secs = review_timeout,
604 raised_to_secs = floored,
605 "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
606 );
607 } else {
608 tracing::debug!(
609 review_timeout_secs = review_timeout,
610 raised_to_secs = floored,
611 "raised the default in_review no-session limit over a long review timeout"
612 );
613 }
614 config.in_review_no_session_stuck_timeout = floored;
615 }
616
617 config.run_timeouts = RunTimeouts::from_config();
618
619 if let Ok(val) = config::get("engine.webhook_health_check_interval") {
620 if let Ok(secs) = val.parse::<u64>() {
621 if secs > 0 {
622 config.webhook_health_check_interval = Some(Duration::from_secs(secs));
623 } else {
624 config.webhook_health_check_interval = None;
625 }
626 } else {
627 tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
628 }
629 }
630
631 if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
632 config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
633 }
634
635 if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
636 if let Ok(secs) = val.parse::<u64>() {
637 config.graceful_shutdown_timeout = Duration::from_secs(secs);
638 } else {
639 tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
640 }
641 }
642
643 if let Ok(val) = config::get("engine.silence_grace_period") {
644 if let Ok(secs) = val.parse::<u64>() {
645 config.silence_grace_period = secs;
646 } else {
647 tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
648 }
649 }
650
651 if let Ok(val) = config::get("engine.silence_cooldown") {
652 if let Ok(secs) = val.parse::<u64>() {
653 config.silence_cooldown = secs;
654 } else {
655 tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
656 }
657 }
658
659 if let Ok(val) = config::get("engine.upgrade_check_interval") {
660 if let Ok(secs) = val.parse::<u64>() {
661 config.upgrade_check_interval = secs;
662 } else {
663 tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
664 }
665 }
666
667 if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
668 if let Ok(secs) = val.parse::<u64>() {
669 config.startup_failure_escalation_secs = secs;
670 } else {
671 tracing::warn!(
672 key = "engine.startup_failure_escalation_secs",
673 value = %val,
674 default_secs = config.startup_failure_escalation_secs,
675 "invalid value for engine.startup_failure_escalation_secs, using default"
676 );
677 }
678 }
679
680 config
681 }
682}
683
684pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
687
688pub fn format_rate_limited_until(reset_at: i64) -> String {
690 chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
691 || reset_at.to_string(),
692 |t| {
693 t.with_timezone(&chrono::Local)
694 .format("%H:%M:%S %Z")
695 .to_string()
696 },
697 )
698}
699
700async fn report_rate_limited(
703 reset_at: i64,
704 store: Option<&Arc<TaskStore>>,
705 transport: Option<&Arc<Transport>>,
706) {
707 let m = rate_limit_metrics();
708 let until = format_rate_limited_until(reset_at);
709 tracing::warn!(
710 reset_at,
711 rate_limit_hits = m.hits,
712 skipped_calls = m.skipped_calls,
713 five_xx_hits = m.five_xx_hits,
714 rest_remaining = ?m.rest_remaining,
715 graphql_remaining = ?m.graphql_remaining,
716 "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
717 );
718
719 if let Some(store) = store
720 && let Err(e) = store
721 .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
722 .await
723 {
724 tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
725 }
726
727 if let Some(transport) = transport {
728 transport.push_notification(TaskNotification {
729 task_id: "github".to_string(),
730 title: "GitHub rate limit".to_string(),
731 status: "blocked".to_string(),
732 agent: "orch".to_string(),
733 duration_seconds: 0.0,
734 summary: format!(
735 "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
736 ),
737 repo: None,
738 notify_target: None,
739 pr_number: None,
740 });
741 }
742}
743
744fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
750 if target > in_force {
751 semaphore.add_permits(target - in_force);
752 return target;
753 }
754
755 if target < in_force {
756 return in_force - semaphore.forget_permits(in_force - target);
757 }
758
759 in_force
760}
761
762const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
765
766fn restart_only_snapshot() -> Vec<Option<String>> {
768 RESTART_ONLY_KEYS
769 .iter()
770 .map(|key| config::get(key).ok())
771 .collect()
772}
773
774fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
777 let changed: Vec<&'static str> = RESTART_ONLY_KEYS
778 .iter()
779 .zip(old)
780 .zip(new)
781 .filter(|((_, old), new)| old != new)
782 .map(|((key, _), _)| *key)
783 .collect();
784
785 for key in &changed {
786 tracing::info!(key = key, "config changed, takes effect after restart");
787 }
788
789 changed
790}
791
792fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
795 macro_rules! log_changed {
796 ($($key:literal => $field:ident),+ $(,)?) => {
797 $(if old.$field != new.$field {
798 tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
799 })+
800 };
801 }
802
803 log_changed!(
804 "engine.tick_interval" => tick_interval,
805 "engine.sync_interval" => sync_interval,
806 "engine.webhook_health_check_interval" => webhook_health_check_interval,
807 "engine.max_parallel" => max_parallel,
808 "engine.stuck_timeout" => stuck_timeout,
809 "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
810 "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
811 "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
812 "engine.silence_grace_period" => silence_grace_period,
813 "engine.silence_cooldown" => silence_cooldown,
814 "engine.upgrade_check_interval" => upgrade_check_interval,
815 "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
816 "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
817 );
818}
819
820fn should_run_upgrade_check(
821 last_upgrade_check: Option<Instant>,
822 upgrade_check_interval: u64,
823) -> bool {
824 if upgrade_check_interval == 0 {
825 return false;
826 }
827
828 last_upgrade_check
829 .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
830}
831
832async fn connect_project_engine(
839 repo: String,
840 project_dir: PathBuf,
841 store: Arc<TaskStore>,
842 event_tx: Option<Sender<events::TaskEvent>>,
843) -> anyhow::Result<ProjectEngine> {
844 let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
845
846 match backend.health_check().await {
850 Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
851 Err(e) => tracing::warn!(
852 repo = %repo,
853 error = %e,
854 "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
855 ),
856 }
857
858 let task_manager = Arc::new(match event_tx {
859 Some(tx) => {
860 TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
861 }
862 None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
863 });
864 let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
865
866 Ok(ProjectEngine {
867 repo,
868 project_dir,
869 backend,
870 task_manager,
871 runner,
872 store,
873 })
874}
875
876fn diff_configured_projects(
883 configured: &[(String, PathBuf)],
884 active_repos: &HashSet<String>,
885) -> (Vec<(String, PathBuf)>, Vec<String>) {
886 let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
887
888 let added = configured
889 .iter()
890 .filter(|(repo, _)| !active_repos.contains(repo))
891 .cloned()
892 .collect();
893 let removed = active_repos
894 .iter()
895 .filter(|repo| !configured_repos.contains(repo.as_str()))
896 .cloned()
897 .collect();
898
899 (added, removed)
900}
901
902#[expect(
908 clippy::too_many_arguments,
909 reason = "a new project engine needs every shared engine handle"
910)]
911async fn sync_project_engines(
912 project_engines: &mut Vec<ProjectEngine>,
913 active_repos: &mut HashSet<String>,
914 event_bus: &events::EventBus,
915 tmux: &Arc<TmuxManager>,
916 capture: &Arc<CaptureService>,
917 semaphore: &Arc<Semaphore>,
918 review_semaphore: &Arc<Semaphore>,
919 weight_tx: &mpsc::Sender<WeightSignal>,
920 router: &Arc<RwLock<Router>>,
921 dispatching: &Arc<dashmap::DashMap<String, String>>,
922) {
923 let configured = match config::get_projects_with_paths() {
924 Ok(p) if !p.is_empty() => p,
925 Ok(_) => return,
926 Err(e) => {
927 tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
928 return;
929 }
930 };
931 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
932 return;
933 };
934
935 let (added, removed) = diff_configured_projects(&configured, active_repos);
936 for repo in removed {
937 tracing::info!(
938 repo = %repo,
939 "project removed from config, takes effect after restart"
940 );
941 }
942
943 for (repo, project_dir) in added {
944 config::watch(&project_dir.join(".orch.yml"));
945 let engine = match connect_project_engine(
946 repo.clone(),
947 project_dir,
948 Arc::clone(&store),
949 Some(event_bus.sender()),
950 )
951 .await
952 {
953 Ok(engine) => engine,
954 Err(e) => {
955 tracing::warn!(
958 repo = %repo,
959 err = %e,
960 "project added to config but its backend could not be built, retrying on the next config change"
961 );
962 continue;
963 }
964 };
965
966 subscribers::dispatch::spawn(
967 event_bus.subscribe(),
968 Arc::clone(&engine.backend),
969 Arc::clone(tmux),
970 Arc::clone(&engine.runner),
971 Arc::clone(capture),
972 Arc::clone(semaphore),
973 Arc::clone(&engine.task_manager),
974 weight_tx.clone(),
975 Arc::clone(router),
976 Arc::clone(dispatching),
977 Arc::clone(&engine.store),
978 engine.repo.clone(),
979 );
980 subscribers::review::spawn(
981 event_bus.subscribe(),
982 Arc::clone(&engine.backend),
983 Arc::clone(tmux),
984 Arc::clone(review_semaphore),
985 Arc::clone(&engine.task_manager),
986 Arc::clone(router),
987 Arc::clone(dispatching),
988 Arc::clone(&engine.store),
989 engine.repo.clone(),
990 );
991 subscribers::unblock::spawn(
992 event_bus.subscribe(),
993 Arc::clone(&engine.backend),
994 Arc::clone(&engine.task_manager),
995 Arc::clone(&engine.store),
996 engine.repo.clone(),
997 );
998
999 active_repos.insert(repo.clone());
1000 project_engines.push(engine);
1001 tracing::info!(repo = %repo, "project added from config, now syncing");
1002 }
1003}
1004
1005async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1009 let projects = config::get_projects_with_paths()?;
1010 let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1011 tracing::info!(repos = ?repos, "loading projects from config");
1012
1013 if projects.is_empty() {
1014 let config_path = home::config_path().map_or_else(
1015 |_| "~/.orch/config.yml".to_string(),
1016 |p| p.display().to_string(),
1017 );
1018 anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1019 }
1020
1021 let mut engines = Vec::new();
1022
1023 let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1026 if cluster::enabled() {
1027 cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1028 local::serve((*store).clone()).await?;
1029 }
1030
1031 engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1034
1035 for (repo, project_dir) in projects {
1036 tracing::info!(repo = %repo, "initializing project engine");
1037 config::watch(&project_dir.join(".orch.yml"));
1038
1039 engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1040 }
1041
1042 Ok(engines)
1043}
1044
1045async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1046 for engine in project_engines {
1047 let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1048 let repo_root_path = PathBuf::from(&repo_root);
1049 let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1050 let worktrees = list_project_worktrees(&repo_root_path).await?;
1051
1052 if let Err(e) = git_ops::git()
1055 .await
1056 .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1057 .output_with_context()
1058 .await
1059 {
1060 tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1061 }
1062
1063 let mut orphans_removed: u32 = 0;
1064 let mut invalid_removed: u32 = 0;
1065 let mut cleaned: u32 = 0;
1066 let mut valid_kept: u32 = 0;
1067
1068 for worktree_dir in worktrees {
1069 let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1070 continue;
1071 };
1072
1073 if !validate_worktree_gitdir(&worktree_dir).await {
1076 tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1077 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1078 .await;
1079 invalid_removed += 1;
1080 continue;
1081 }
1082
1083 let Some(task_id) = task_id_from_worktree_name(name) else {
1084 tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1085 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1086 .await;
1087 orphans_removed += 1;
1088 continue;
1089 };
1090
1091 let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1092 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1093 remove_worktree_and_branch(
1094 &task_id,
1095 &worktree_dir,
1096 Some(name),
1097 &repo_root_path,
1098 false,
1099 )
1100 .await;
1101 orphans_removed += 1;
1102 continue;
1103 };
1104
1105 let task = engine.store.get(store_id).await?;
1106 let branch_name = if task.branch.is_empty() {
1107 name
1108 } else {
1109 task.branch.as_str()
1110 };
1111
1112 match task.status {
1113 TaskStatus::New
1114 | TaskStatus::Routed
1115 | TaskStatus::InProgress
1116 | TaskStatus::NeedsReview
1117 | TaskStatus::InReview => {
1118 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1119 match rebase_worktree_on_origin_main(&worktree_dir, &default_branch).await {
1120 Err(e) => {
1121 tracing::warn!(repo = %engine.repo, task_id = %task_id, err = %e, "startup rebase failed, resetting task");
1126 abort_worktree_rebase(&worktree_dir).await;
1127 let keep_remote = task.pr_number.is_some();
1128 remove_worktree_and_branch(
1129 &task_id,
1130 &worktree_dir,
1131 Some(branch_name),
1132 &repo_root_path,
1133 keep_remote,
1134 )
1135 .await;
1136 match engine.store.resolve_task_id(&engine.repo, &task_id).await {
1137 Ok(Some(reset_id)) => {
1138 if let Err(e) = engine.store.reset_to_new(reset_id).await {
1139 tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1140 }
1141 }
1142 Ok(None) => {
1143 tracing::warn!(
1144 task_id,
1145 "task not found in store during reconciliation"
1146 );
1147 }
1148 Err(e) => {
1149 tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1150 }
1151 }
1152 if let Err(e) = engine
1153 .task_manager
1154 .update_task_status(&ExternalId(task_id.clone()), Status::New)
1155 .await
1156 {
1157 tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1158 }
1159 cleaned += 1;
1160 }
1161 _ => {
1162 valid_kept += 1;
1163 }
1164 }
1165 }
1166 TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1169 valid_kept += 1;
1170 }
1171 TaskStatus::Done => {
1172 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1173 remove_worktree_and_branch_for(
1174 &task_id,
1175 &worktree_dir,
1176 Some(branch_name),
1177 &repo_root_path,
1178 false,
1179 task.pr_number.is_some(),
1180 )
1181 .await;
1182 cleaned += 1;
1183 }
1184 }
1185 }
1186
1187 tracing::info!(
1188 repo = %engine.repo,
1189 orphans_removed,
1190 invalid_removed,
1191 cleaned,
1192 valid_kept,
1193 "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1194 );
1195
1196 let prune = Command::new("git")
1199 .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1200 .output_with_context()
1201 .await;
1202 match prune {
1203 Ok(output) if output.status.success() => {
1204 tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1205 }
1206 Ok(output) => {
1207 let stderr = String::from_utf8_lossy(&output.stderr);
1208 tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1209 }
1210 Err(e) => {
1211 tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1212 }
1213 }
1214 }
1215
1216 Ok(())
1217}
1218
1219async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1227 for engine in project_engines {
1228 let tasks = match engine.store.list_external_tasks_with_estimates().await {
1229 Ok(t) => t,
1230 Err(e) => {
1231 tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1232 continue;
1233 }
1234 };
1235
1236 if tasks.is_empty() {
1237 continue;
1238 }
1239
1240 let mut synced: u32 = 0;
1241 let mut failed: u32 = 0;
1242 for (external_id, estimate) in &tasks {
1243 match engine
1244 .backend
1245 .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1246 .await
1247 {
1248 Ok(()) => synced += 1,
1249 Err(e) => {
1250 tracing::debug!(
1251 repo = %engine.repo,
1252 task_id = %external_id,
1253 estimate,
1254 err = %e,
1255 "startup estimate reconciliation: sync failed"
1256 );
1257 failed += 1;
1258 }
1259 }
1260 }
1261
1262 if synced > 0 || failed > 0 {
1263 tracing::info!(
1264 repo = %engine.repo,
1265 synced,
1266 failed,
1267 "startup estimate reconciliation complete"
1268 );
1269 }
1270 }
1271}
1272
1273fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1275 let config_path = project_dir.join(".orch.yml");
1276 if !config_path.exists() {
1277 return ProjectChannelConfig::default();
1278 }
1279 let content = match fs::read_to_string(&config_path) {
1280 Ok(c) => c,
1281 Err(e) => {
1282 tracing::warn!(
1283 path = %config_path.display(),
1284 error = %e,
1285 "failed to read .orch.yml; channel config will use defaults"
1286 );
1287 return ProjectChannelConfig::default();
1288 }
1289 };
1290 let val: serde_norway::Value = match serde_norway::from_str(&content) {
1291 Ok(v) => v,
1292 Err(e) => {
1293 tracing::warn!(
1294 path = %config_path.display(),
1295 error = %e,
1296 "failed to parse .orch.yml; channel config will use defaults"
1297 );
1298 return ProjectChannelConfig::default();
1299 }
1300 };
1301 let Some(channels) = val.get("channels") else {
1302 return ProjectChannelConfig::default();
1303 };
1304 let get_str = |section: &str, key: &str| -> Option<String> {
1305 channels
1306 .get(section)
1307 .and_then(|s| s.get(key))
1308 .and_then(|v| v.as_str())
1309 .map(String::from)
1310 };
1311 ProjectChannelConfig {
1312 telegram_topic_id: get_str("telegram", "topic_id"),
1313 discord_channel_id: get_str("discord", "channel_id"),
1314 }
1315}
1316
1317async fn fetch_latest_release_version() -> Option<String> {
1322 let output = Command::new("gh")
1323 .args([
1324 "api",
1325 "repos/gabrielkoerich/orch/releases/latest",
1326 "--jq",
1327 ".tag_name",
1328 ])
1329 .output()
1330 .await
1331 .ok()?;
1332 if !output.status.success() {
1333 return None;
1334 }
1335 let tag = String::from_utf8(output.stdout).ok()?;
1336 let version = tag.trim().trim_start_matches('v').to_string();
1337 if version.is_empty() {
1338 None
1339 } else {
1340 Some(version)
1341 }
1342}
1343
1344async fn check_and_notify_upgrade(
1353 store: &Arc<TaskStore>,
1354 channels: &Arc<ChannelRegistry>,
1355 current_version: &str,
1356 check_interval: Duration,
1357) {
1358 let last_check_key = "upgrade:last_check_at";
1359 let last_notified_key = "upgrade:last_notified_version";
1360
1361 let now = chrono::Utc::now().timestamp();
1363 if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1364 && let Ok(last_check) = last_check_str.parse::<i64>()
1365 && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1366 {
1367 return;
1368 }
1369
1370 let _ = store.kv_set(last_check_key, &now.to_string()).await;
1372
1373 let Some(latest) = fetch_latest_release_version().await else {
1374 tracing::debug!("upgrade check: failed to fetch latest release version");
1375 return;
1376 };
1377
1378 if latest == current_version {
1379 tracing::debug!(current = %current_version, "orch is up to date");
1380 return;
1381 }
1382
1383 tracing::warn!(
1384 current_version = %current_version,
1385 latest_version = %latest,
1386 "orch upgrade available"
1387 );
1388
1389 if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1394 && last_notified == latest
1395 {
1396 tracing::debug!(latest = %latest, "upgrade already notified");
1397 return;
1398 }
1399
1400 let msg_telegram = format!(
1401 "⚠️ <b>Orch Upgrade Available</b>\n\n\
1402 Service: <code>{current_version}</code>\n\
1403 Latest: <code>{latest}</code>\n\n\
1404 Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1405 );
1406
1407 let msg_discord = format!(
1408 "⚠️ **Orch Upgrade Available**\n\n\
1409 Service: `{current_version}`\n\
1410 Latest: `{latest}`\n\n\
1411 Run: `brew update && brew upgrade orch && brew services restart orch`"
1412 );
1413
1414 let msg_slack = format!(
1415 "⚠️ *Orch Upgrade Available*\n\n\
1416 Service: `{current_version}`\n\
1417 Latest: `{latest}`\n\n\
1418 Run: `brew update && brew upgrade orch && brew services restart orch`"
1419 );
1420
1421 for channel in channels.iter() {
1422 let (body, metadata, topic_id) = match channel.name() {
1423 "telegram" => (
1424 msg_telegram.clone(),
1425 serde_json::json!({"preformatted_html": true}),
1426 None,
1427 ),
1428 "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1429 "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1430 _ => continue,
1431 };
1432
1433 let msg = OutgoingMessage {
1434 thread_id: "upgrade".to_string(),
1435 body,
1436 reply_to: None,
1437 metadata,
1438 topic_id,
1439 };
1440
1441 if let Err(e) = channel.send(&msg).await {
1442 tracing::warn!(
1443 channel = channel.name(),
1444 ?e,
1445 "failed to send upgrade notification"
1446 );
1447 }
1448 }
1449
1450 let _ = store.kv_set(last_notified_key, &latest).await;
1452}
1453
1454async fn send_startup_escalation_notification(elapsed_secs: u64) {
1462 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1463 let duration = notification::format_duration(elapsed_secs as f64);
1464
1465 let msg_telegram = format!(
1466 "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1467 The engine has been retrying backend connections for {duration}\n\
1468 without success. All task processing, routing, and scheduled\n\
1469 jobs are paused. Check service logs with `orch log -f`."
1470 );
1471
1472 let msg_discord = format!(
1473 "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1474 The engine has been retrying backend connections for {duration}\n\
1475 without success. All task processing, routing, and scheduled\n\
1476 jobs are paused. Check service logs with `orch log -f`."
1477 );
1478
1479 let msg_slack = format!(
1480 "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1481 The engine has been retrying backend connections for {duration}\n\
1482 without success. All task processing, routing, and scheduled\n\
1483 jobs are paused. Check service logs with `orch log -f`."
1484 );
1485
1486 let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1488 let tg_chat_id = config::get("channels.telegram.chat_id");
1489 if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1490 && !token.is_empty()
1491 && !chat_id.is_empty()
1492 && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1493 {
1494 let msg = OutgoingMessage {
1495 thread_id: "startup_escalation".to_string(),
1496 body: msg_telegram.clone(),
1497 reply_to: None,
1498 metadata: serde_json::json!({ "preformatted_html": true }),
1499 topic_id: config::get("channels.telegram.general_topic_id").ok(),
1500 };
1501 match channel.send(&msg).await {
1502 Err(e) => {
1503 tracing::warn!(?e, "failed to send startup escalation via telegram");
1504 }
1505 _ => {
1506 tracing::info!("startup escalation notification sent to telegram");
1507 }
1508 }
1509 }
1510
1511 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1513 let channel_id = config::get("channels.discord.channel_id")
1514 .or_else(|_| config::get("channels.discord.general_channel_id"));
1515 if !token.is_empty()
1516 && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1517 {
1518 let shard_id = config::get("channels.discord.shard_id")
1519 .ok()
1520 .and_then(|v| v.parse::<u64>().ok())
1521 .unwrap_or(0);
1522 let shard_count = config::get("channels.discord.shard_count")
1523 .ok()
1524 .and_then(|v| v.parse::<u64>().ok())
1525 .unwrap_or(1);
1526 if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1527 let msg = OutgoingMessage {
1528 thread_id: "startup_escalation".to_string(),
1529 body: msg_discord.clone(),
1530 reply_to: None,
1531 metadata: serde_json::Value::Null,
1532 topic_id: None,
1533 };
1534 match gateway.send(&msg).await {
1535 Err(e) => {
1536 tracing::warn!(?e, "failed to send startup escalation via discord");
1537 }
1538 _ => {
1539 tracing::info!("startup escalation notification sent to discord");
1540 }
1541 }
1542 }
1543 }
1544 }
1545
1546 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1548 let channel_id = config::get("channels.slack.channel_id");
1549 if !token.is_empty()
1550 && let Ok(cid) = channel_id
1551 && let Ok(channel) = SlackChannel::new(token, Some(cid))
1552 {
1553 let msg = OutgoingMessage {
1554 thread_id: "startup_escalation".to_string(),
1555 body: msg_slack.clone(),
1556 reply_to: None,
1557 metadata: serde_json::Value::Null,
1558 topic_id: None,
1559 };
1560 match channel.send(&msg).await {
1561 Err(e) => {
1562 tracing::warn!(?e, "failed to send startup escalation via slack");
1563 }
1564 _ => {
1565 tracing::info!("startup escalation notification sent to slack");
1566 }
1567 }
1568 }
1569 }
1570}
1571
1572pub async fn serve() -> anyhow::Result<()> {
1576 let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1577 if killed > 0 {
1578 tracing::info!(count = killed, "terminated stale orch serve processes");
1579 }
1580
1581 tracing::info!("orch engine starting");
1582
1583 if let Err(e) = home::init_dirs().await {
1587 tracing::warn!("failed to initialize orch directories: {e}");
1588 }
1589
1590 if let Err(e) = config::warm_cache().await {
1593 tracing::warn!("failed to warm config cache: {e}");
1594 }
1595
1596 let mut config = EngineConfig::from_config();
1597
1598 tracing::info!("internal database ready");
1599
1600 let startup_escalation_secs = config.startup_failure_escalation_secs;
1609 let mut project_engines = {
1610 let mut delay_secs = 5u64;
1611 let mut attempt = 0u32;
1612 let mut escalated = false;
1613 let start = Instant::now();
1614 loop {
1615 attempt += 1;
1616 match init_project_engines().await {
1617 Ok(engines) => break engines,
1618 Err(e) => {
1619 let elapsed = start.elapsed().as_secs();
1620
1621 if startup_escalation_secs > 0
1627 && !escalated
1628 && elapsed >= startup_escalation_secs
1629 {
1630 escalated = true;
1631 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1632 let outage = notification::format_duration(elapsed as f64);
1633 tracing::error!(
1634 delay_secs,
1635 attempt,
1636 elapsed_secs = elapsed,
1637 threshold_secs = startup_escalation_secs,
1638 "project backends unavailable for {outage} — escalating startup failure: {e}",
1639 );
1640 send_startup_escalation_notification(elapsed).await;
1644 } else {
1645 tracing::warn!(
1646 delay_secs,
1647 attempt,
1648 "project backends unavailable, retrying: {e}"
1649 );
1650 }
1651
1652 time::sleep(Duration::from_secs(delay_secs)).await;
1653 delay_secs = (delay_secs * 2).min(120);
1654 }
1655 }
1656 }
1657 };
1658
1659 tracing::info!(
1660 project_count = project_engines.len(),
1661 "initialized project engines"
1662 );
1663
1664 let mut active_repos: HashSet<String> =
1668 project_engines.iter().map(|e| e.repo.clone()).collect();
1669
1670 let event_bus = events::EventBus::new(256);
1672 if let Err(e) = event_bus.start_ws_server().await {
1673 tracing::warn!(
1674 ?e,
1675 "failed to start event websocket server, continuing without it"
1676 );
1677 }
1678
1679 {
1683 let current = env!("ORCH_VERSION").to_string();
1684 tokio::spawn(async move {
1685 if let Some(latest) = fetch_latest_release_version().await
1686 && latest != current
1687 {
1688 tracing::warn!(
1689 current_version = %current,
1690 latest_version = %latest,
1691 "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1692 );
1693 }
1694 });
1695 }
1696
1697 for engine in &mut project_engines {
1699 engine.task_manager = Arc::new(TaskManager::with_events(
1700 Arc::clone(&engine.backend),
1701 Arc::clone(&engine.store),
1702 engine.repo.clone(),
1703 event_bus.sender(),
1704 ));
1705 }
1706
1707 let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1709 if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1710 tracing::warn!(err = %e, "startup worktree reconciliation failed");
1711 }
1712
1713 if leads {
1714 let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1718 tokio::spawn(async move {
1719 reconcile_startup_estimates(&engines_snapshot).await;
1720 });
1721 }
1722
1723 let global_channel_config = GlobalChannelConfig {
1725 telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1726 discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1727 control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1728 control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1729 };
1730 let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1731 .iter()
1732 .map(|e| {
1733 let cfg = read_project_channel_config(&e.project_dir);
1734 (e.repo.clone(), cfg)
1735 })
1736 .collect();
1737 let channel_router = Arc::new(ChannelRouter::new(
1738 &global_channel_config,
1739 &project_channel_configs,
1740 ));
1741 tracing::info!(
1742 projects = ?channel_router.projects(),
1743 "channel router initialized"
1744 );
1745
1746 let tmux = Arc::new(TmuxManager::new());
1748 tick::startup_cleanup(&tmux).await;
1751
1752 let transport = Arc::new(Transport::new());
1754
1755 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1757 let capture_for_tick = Arc::clone(&capture);
1758 tokio::spawn(async move {
1759 capture.start().await;
1760 });
1761
1762 let mut channel_registry = ChannelRegistry::new();
1764
1765 if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1767 && !token.is_empty()
1768 {
1769 let chat_id = config::get("channels.telegram.chat_id").ok();
1770 match TelegramChannel::new(token, chat_id) {
1771 Ok(telegram) => {
1772 if let Err(e) = telegram.health_check().await {
1773 tracing::warn!(?e, "telegram channel health check failed, skipping");
1774 } else {
1775 channel_registry.register(Box::new(telegram));
1776 tracing::info!("telegram channel registered");
1777 }
1778 }
1779 Err(e) => {
1780 tracing::warn!(?e, "failed to create telegram channel, skipping");
1781 }
1782 }
1783 }
1784
1785 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1787 && !token.is_empty()
1788 {
1789 let channel_id = config::get("channels.discord.channel_id").ok();
1790 let shard_id = config::get("channels.discord.shard_id")
1791 .ok()
1792 .and_then(|v| v.parse::<u64>().ok())
1793 .unwrap_or(0);
1794 let shard_count = config::get("channels.discord.shard_count")
1795 .ok()
1796 .and_then(|v| v.parse::<u64>().ok())
1797 .unwrap_or(1);
1798 match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1799 Ok(discord) => {
1800 if let Err(e) = discord.health_check().await {
1801 tracing::warn!(?e, "discord gateway health check failed, skipping");
1802 } else {
1803 channel_registry.register(Box::new(discord));
1804 tracing::info!(shard_id, shard_count, "discord gateway registered");
1805 }
1806 }
1807 Err(e) => {
1808 tracing::warn!(?e, "failed to create discord gateway, skipping");
1809 }
1810 }
1811 }
1812
1813 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1815 && !token.is_empty()
1816 {
1817 let channel_id = config::get("channels.slack.channel_id").ok();
1818 match SlackChannel::new(token, channel_id) {
1819 Ok(slack) => {
1820 if let Err(e) = slack.health_check().await {
1821 tracing::warn!(?e, "slack channel health check failed, skipping");
1822 } else {
1823 channel_registry.register(Box::new(slack));
1824 tracing::info!("slack channel registered");
1825 }
1826 }
1827 Err(e) => {
1828 tracing::warn!(?e, "failed to create slack channel, skipping");
1829 }
1830 }
1831 }
1832
1833 let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1835 channel_registry.register(Box::new(tmux_channel));
1836 tracing::info!("tmux channel registered");
1837
1838 let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1840 for channel in channel_registry.iter() {
1841 match channel.start().await {
1842 Ok(rx) => {
1843 tracing::info!(channel = channel.name(), "channel started");
1844 channel_receivers.push(rx);
1845 }
1846 Err(e) => {
1847 tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1848 }
1849 }
1850 }
1851
1852 let channel_registry = Arc::new(channel_registry);
1854
1855 let transport_for_messages = Arc::clone(&transport);
1857 let tmux_for_messages = Arc::clone(&tmux);
1858 let capture_for_messages = Arc::clone(&capture_for_tick);
1859 let channels_for_messages = Arc::clone(&channel_registry);
1860 let engine_refs: Vec<EngineRef> = project_engines
1862 .iter()
1863 .map(|e| {
1864 (
1865 e.repo.clone(),
1866 Arc::clone(&e.backend),
1867 Arc::clone(&e.task_manager),
1868 Some(Arc::clone(&e.store)),
1869 )
1870 })
1871 .collect();
1872 let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1874 let router_for_messages = Arc::clone(&channel_router);
1875 for mut rx in channel_receivers {
1876 let transport = Arc::clone(&transport_for_messages);
1877 let tmux = Arc::clone(&tmux_for_messages);
1878 let capture = Arc::clone(&capture_for_messages);
1879 let channels = Arc::clone(&channels_for_messages);
1880 let engine_refs = engine_refs.clone();
1881 let ch_router = Arc::clone(&router_for_messages);
1882 let picks = Arc::clone(&pending_picks);
1883 tokio::spawn(async move {
1884 while let Some(msg) = rx.recv().await {
1885 tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1886 channel_handler::handle_channel_message(
1887 msg,
1888 &transport,
1889 &tmux,
1890 &capture,
1891 &channels,
1892 &engine_refs,
1893 &ch_router,
1894 &picks,
1895 )
1896 .await;
1897 }
1898 });
1899 }
1900
1901 {
1905 let restart_channels = Arc::clone(&channel_registry);
1906 let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1907 if let Some(store) = restart_store {
1908 tokio::spawn(async move {
1909 time::sleep(Duration::from_secs(5)).await;
1910 match store.kv_get("control:restart_pending").await {
1911 Ok(Some(conv_key)) => {
1912 if let Some((ch_name, thread_id, topic_id)) =
1913 transport::parse_conversation_key(&conv_key)
1914 {
1915 for ch in restart_channels.iter() {
1916 if ch.name() == ch_name {
1917 let msg = OutgoingMessage {
1918 thread_id: thread_id.to_string(),
1919 body: "orch restarted successfully.".to_string(),
1920 reply_to: None,
1921 metadata: serde_json::json!({}),
1922 topic_id: topic_id.map(ToString::to_string),
1923 };
1924 if let Err(e) = ch.send(&msg).await {
1925 tracing::warn!(
1926 channel = ch_name,
1927 error = %e,
1928 "failed to send restart confirmation"
1929 );
1930 }
1931 break;
1932 }
1933 }
1934 }
1935 if let Err(e) = store.kv_delete("control:restart_pending").await {
1936 tracing::warn!(error = %e, "failed to clear control:restart_pending");
1937 }
1938 }
1939 Ok(None) => {} Err(e) => {
1941 tracing::warn!(error = %e, "failed to read control:restart_pending");
1942 }
1943 }
1944 });
1945 }
1946 }
1947
1948 {
1959 let mut notification_rx = transport.subscribe_notifications();
1960 let channels = Arc::clone(&channel_registry);
1961 let notif_router = Arc::clone(&channel_router);
1962 let notif_store: Option<Arc<TaskStore>> =
1964 project_engines.first().map(|e| Arc::clone(&e.store));
1965 tokio::spawn(async move {
1966 loop {
1967 match notification_rx.recv().await {
1968 Ok(notification) => {
1969 let level = NotificationLevel::from_config();
1970 if !level.should_notify(¬ification.status) {
1971 tracing::debug!(
1972 task_id = %notification.task_id,
1973 status = %notification.status,
1974 "notification suppressed by level={:?}",
1975 level
1976 );
1977 continue;
1978 }
1979
1980 tracing::info!(
1981 task_id = %notification.task_id,
1982 status = %notification.status,
1983 repo = ?notification.repo,
1984 "routing notification to channels"
1985 );
1986
1987 let mut routed = false;
1989
1990 if let Some(ref target) = notification.notify_target {
1993 let telegram = channels.iter().find(|c| c.name() == "telegram");
1994 if let Some(channel) = telegram {
1995 let body = notification.format_telegram();
1996 let metadata = serde_json::json!({
1997 "preformatted_html": true,
1998 "chat_id_override": target
1999 });
2000 let msg = OutgoingMessage {
2001 thread_id: notification.task_id.clone(),
2002 body,
2003 reply_to: None,
2004 metadata,
2005 topic_id: None,
2006 };
2007 match channel.send(&msg).await {
2008 Err(e) => {
2009 tracing::warn!(
2010 task_id = %notification.task_id,
2011 target,
2012 ?e,
2013 "failed to send to notify_target"
2014 );
2015 }
2016 _ => {
2017 routed = true;
2018 }
2019 }
2020 }
2021 }
2022
2023 if routed {
2024 } else if let Some(repo) = notification.repo.as_deref() {
2026 let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2032
2033 for channel in channels.iter() {
2035 let ch_name = channel.name();
2036 if let Some(topic_id) =
2037 notif_router.target_for_project(repo, ch_name)
2038 {
2039 let body = match ch_name {
2040 "telegram" => notification.format_telegram(),
2041 "discord" => notification.format_discord(),
2042 "slack" => notification.format_slack(),
2043 _ => continue,
2044 };
2045 let metadata = if ch_name == "telegram" {
2046 serde_json::json!({"preformatted_html": true})
2047 } else {
2048 serde_json::json!({})
2049 };
2050 let resolved_topic = Some(topic_id.to_string());
2051 let msg = OutgoingMessage {
2052 thread_id: notification.task_id.clone(),
2053 body,
2054 reply_to: None,
2055 metadata,
2056 topic_id: resolved_topic.clone(),
2057 };
2058 if let Err(e) = channel.send(&msg).await {
2059 tracing::warn!(
2060 channel = ch_name,
2061 task_id = %notification.task_id,
2062 ?e,
2063 "failed to send to dedicated channel"
2064 );
2065 } else {
2066 sent.insert((ch_name.to_string(), resolved_topic));
2067 routed = true;
2068 }
2069 }
2070 }
2071
2072 if let Some(store) = ¬if_store {
2074 match store.list_subscribers_for_repo(repo).await {
2075 Ok(subscribers) => {
2076 for (ch_name, thread_id, topic_id) in subscribers {
2077 let channel =
2078 channels.iter().find(|c| c.name() == ch_name);
2079 let Some(channel) = channel else {
2080 continue;
2081 };
2082 let resolved_topic = if topic_id.is_empty() {
2083 None
2084 } else {
2085 Some(topic_id.clone())
2086 };
2087 if sent.contains(&(
2090 ch_name.clone(),
2091 resolved_topic.clone(),
2092 )) {
2093 tracing::debug!(
2094 channel = %ch_name,
2095 task_id = %notification.task_id,
2096 topic_id = ?resolved_topic,
2097 "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2098 );
2099 continue;
2100 }
2101 let body = notification.format_with_project(&ch_name);
2102 let metadata = if ch_name == "telegram" {
2103 serde_json::json!({"preformatted_html": true})
2104 } else {
2105 serde_json::json!({})
2106 };
2107 let msg = OutgoingMessage {
2108 thread_id: thread_id.clone(),
2109 body,
2110 reply_to: None,
2111 metadata,
2112 topic_id: resolved_topic.clone(),
2113 };
2114 if let Err(e) = channel.send(&msg).await {
2115 tracing::warn!(
2116 channel = %ch_name,
2117 task_id = %notification.task_id,
2118 ?e,
2119 "failed to send to subscribed channel"
2120 );
2121 } else {
2122 sent.insert((ch_name.clone(), resolved_topic));
2123 routed = true;
2124 }
2125 }
2126 }
2127 Err(e) => {
2128 tracing::warn!(
2129 repo,
2130 ?e,
2131 "failed to list subscribers for repo"
2132 );
2133 }
2134 }
2135 }
2136 }
2137
2138 if !routed {
2140 for channel in channels.iter() {
2141 let (body, should_send, is_telegram) = match channel.name() {
2142 "telegram" => (notification.format_telegram(), true, true),
2143 "discord" => (notification.format_discord(), true, false),
2144 "slack" => (notification.format_slack(), true, false),
2145 _ => (String::new(), false, false),
2148 };
2149
2150 if !should_send {
2151 continue;
2152 }
2153
2154 let metadata = if is_telegram {
2155 serde_json::json!({"preformatted_html": true})
2156 } else {
2157 serde_json::json!({})
2158 };
2159 let msg = OutgoingMessage {
2160 thread_id: notification.task_id.clone(),
2161 body,
2162 reply_to: None,
2163 metadata,
2164 topic_id: None,
2165 };
2166
2167 if let Err(e) = channel.send(&msg).await {
2168 tracing::warn!(
2169 channel = channel.name(),
2170 task_id = %notification.task_id,
2171 ?e,
2172 "failed to send notification"
2173 );
2174 }
2175 }
2176 }
2177 }
2178 Err(RecvError::Lagged(n)) => {
2179 tracing::warn!(missed = n, "notification receiver lagged");
2180 }
2181 Err(RecvError::Closed) => {
2182 tracing::debug!("notification channel closed");
2183 break;
2184 }
2185 }
2186 }
2187 });
2188 tracing::info!("notification dispatcher started");
2189 }
2190
2191 let webhook_notify = Arc::new(Notify::new());
2193
2194 let webhook_port: Option<u16>;
2197 let mut webhook_healthy: bool;
2198 let mut last_webhook_health_check = Instant::now();
2199 let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2201 let webhook_secret = if webhook_enabled {
2203 secret_ref::config_secret("webhook.secret")
2204 .await
2205 .ok()
2206 .filter(|s| !s.trim().is_empty())
2207 } else {
2208 None
2209 };
2210 if webhook_enabled && webhook_secret.is_none() {
2211 tracing::error!(
2212 "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2213 );
2214 }
2215 let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2216 let webhook_secret = webhook_secret.unwrap_or_default();
2217
2218 let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2220
2221 if webhook_enabled {
2222 const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2226 let port: u16 = config::get("webhook.port")
2227 .ok()
2228 .and_then(|v| v.parse().ok())
2229 .unwrap_or(8080);
2230 webhook_port = Some(port);
2231 let secret = webhook_secret.clone();
2232 let webhook_repo = project_engines
2233 .first()
2234 .map(|e| e.repo.clone())
2235 .unwrap_or_default();
2236
2237 let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2238
2239 let to_save = {
2241 let mut s = webhook_status.lock().await;
2242 s.configured = true;
2243 s.port = Some(port);
2244 s.healthy = false;
2245 s.fallback_mode = false;
2246 s.clone()
2247 };
2248 to_save.save().await;
2249
2250 let status_for_spawn = Arc::clone(&webhook_status);
2251 tokio::spawn(async move {
2252 let mut attempt = 0u32;
2253 loop {
2254 attempt += 1;
2255 tracing::info!(attempt, port, "attempting to start webhook server");
2256 match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2257 .await
2258 {
2259 Ok(()) => {
2260 tracing::info!(attempt, "webhook server exited cleanly");
2261 break;
2262 }
2263 Err(e) => {
2264 let reason = e.to_string();
2265 tracing::error!(attempt, %reason, "webhook server failed");
2266 if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2267 let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2268 "max attempts reached"
2269 } else {
2270 "non-transient error"
2271 };
2272 tracing::error!(
2273 attempt,
2274 %reason,
2275 kind,
2276 orch_webhook_in_fallback = true,
2277 "webhook server giving up, switching to polling fallback"
2278 );
2279 let to_save = {
2280 let mut s = status_for_spawn.lock().await;
2281 s.fallback_mode = true;
2282 s.healthy = false;
2283 s.last_failure_reason = Some(reason);
2284 s.startup_attempts = attempt;
2285 s.clone()
2286 };
2287 to_save.save().await;
2288 break;
2289 }
2290 let delay = webhook_backoff_delay(attempt);
2291 tracing::warn!(
2292 attempt,
2293 delay_ms = delay.as_millis(),
2294 %reason,
2295 "webhook bind failed (transient), retrying with backoff"
2296 );
2297 {
2298 let to_save = {
2299 let mut s = status_for_spawn.lock().await;
2300 s.startup_attempts = attempt;
2301 s.last_failure_reason = Some(reason);
2302 s.clone()
2303 };
2304 to_save.save().await;
2305 }
2306 time::sleep(delay).await;
2307 }
2308 }
2309 }
2310 });
2311
2312 let notify = Arc::clone(&webhook_notify);
2314 let transport_for_webhook = Arc::clone(&transport);
2315 tokio::spawn(async move {
2316 while let Some(msg) = rx.recv().await {
2317 tracing::info!(
2318 channel = %msg.channel,
2319 thread = %msg.thread_id,
2320 event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2321 action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2322 "webhook event received, triggering immediate tick"
2323 );
2324 let route = transport_for_webhook.route(&msg).await;
2325 tracing::info!(
2326 channel = %msg.channel,
2327 route = ?route,
2328 "webhook event routed"
2329 );
2330 notify.notify_one();
2332 }
2333 });
2334
2335 webhook_healthy = true;
2336 tracing::info!(port, orch_webhook_up = true, "webhook server started");
2337 } else {
2338 webhook_port = None;
2339 webhook_healthy = false;
2340 {
2341 let to_save = {
2342 let mut s = webhook_status.lock().await;
2343 s.configured = false;
2344 s.fallback_mode = true;
2345 s.clone()
2346 };
2347 to_save.save().await;
2348 }
2349 tracing::info!(
2350 orch_webhook_in_fallback = true,
2351 "webhook server disabled, using polling fallback mode"
2352 );
2353 }
2354
2355 start_relay_client(&project_engines, &webhook_notify);
2356 app::start_loader();
2357
2358 let router = Arc::new(RwLock::new(
2362 match task::spawn_blocking(Router::from_config).await {
2363 Ok(r) => r,
2364 Err(e) => {
2365 tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2366 task::spawn_blocking(|| Router::new(RouterConfig::default()))
2367 .await
2368 .unwrap_or_else(|e2| {
2369 panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2370 })
2371 }
2372 },
2373 ));
2374 {
2375 let r = router.read().await;
2376 tracing::info!(
2377 mode = %r.config.mode,
2378 agents = ?r.available_agents,
2379 fallback = %r.config.fallback_executor,
2380 "router initialized"
2381 );
2382 }
2383
2384 let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2388 let mut parallel_in_force = config.max_parallel;
2389 let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2392 let mut review_in_force = config.max_parallel;
2393
2394 let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2401
2402 let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2406
2407 let mut config_rx = config::subscribe();
2409
2410 let mut last_sync = Instant::now();
2412
2413 let mut last_upgrade_check: Option<Instant> = None;
2415
2416 let mut rate_limit_reported: Option<i64> = None;
2419
2420 let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2422
2423 for engine in project_engines.iter().filter(|_| leads) {
2425 let reset =
2426 reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2427
2428 if reset > 0 {
2429 tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2430 }
2431 }
2432
2433 for engine in &project_engines {
2435 subscribers::dispatch::spawn(
2436 event_bus.subscribe(),
2437 Arc::clone(&engine.backend),
2438 Arc::clone(&tmux),
2439 Arc::clone(&engine.runner),
2440 Arc::clone(&capture_for_tick),
2441 Arc::clone(&semaphore),
2442 Arc::clone(&engine.task_manager),
2443 weight_tx.clone(),
2444 Arc::clone(&router),
2445 Arc::clone(&dispatching),
2446 Arc::clone(&engine.store),
2447 engine.repo.clone(),
2448 );
2449 }
2450 tracing::info!("dispatch subscriber started");
2451
2452 for engine in &project_engines {
2454 subscribers::review::spawn(
2455 event_bus.subscribe(),
2456 Arc::clone(&engine.backend),
2457 Arc::clone(&tmux),
2458 Arc::clone(&review_semaphore),
2459 Arc::clone(&engine.task_manager),
2460 Arc::clone(&router),
2461 Arc::clone(&dispatching),
2462 Arc::clone(&engine.store),
2463 engine.repo.clone(),
2464 );
2465 }
2466 tracing::info!("review subscriber started");
2467
2468 for engine in &project_engines {
2470 subscribers::unblock::spawn(
2471 event_bus.subscribe(),
2472 Arc::clone(&engine.backend),
2473 Arc::clone(&engine.task_manager),
2474 Arc::clone(&engine.store),
2475 engine.repo.clone(),
2476 );
2477 }
2478 tracing::info!("unblock subscriber started");
2479
2480 subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2483 tracing::info!("notify subscriber started");
2484
2485 tracing::info!(
2487 tick = ?config.tick_interval,
2488 sync = ?config.sync_interval,
2489 parallel = config.max_parallel,
2490 "entering main loop"
2491 );
2492 let mut restart_only = restart_only_snapshot();
2495
2496 let mut interval = time::interval(config.tick_interval);
2497
2498 let mut sigterm = unix::signal(SignalKind::terminate())?;
2500 let mut sighup = unix::signal(SignalKind::hangup())?;
2501
2502 let sync_in_progress = Arc::new(AtomicBool::new(false));
2504
2505 let last_tick_epoch = Arc::new(AtomicU64::new(
2508 SystemTime::now()
2509 .duration_since(std_time::UNIX_EPOCH)
2510 .unwrap_or_default()
2511 .as_secs(),
2512 ));
2513 {
2514 let last_tick_epoch = Arc::clone(&last_tick_epoch);
2515 let tick_interval_secs = config.tick_interval.as_secs();
2516 tokio::spawn(async move {
2517 let mut watchdog_interval = time::interval(Duration::from_secs(30));
2519 let mut suspend_detector = SuspendDetector::new();
2524 loop {
2525 watchdog_interval.tick().await;
2526 let own_gap = suspend_detector.check();
2527 let last = last_tick_epoch.load(Ordering::Relaxed);
2528 let now = SystemTime::now()
2529 .duration_since(std_time::UNIX_EPOCH)
2530 .unwrap_or_default()
2531 .as_secs();
2532 let stale_secs = now.saturating_sub(last);
2533 let threshold = (tick_interval_secs * 6).max(60);
2535 if stale_secs > threshold {
2536 let shared_gap =
2543 engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2544 let gap = match (own_gap, shared_gap) {
2545 (Some(a), Some(b)) => Some(a.max(b)),
2546 (Some(a), None) => Some(a),
2547 (None, Some(b)) => Some(b),
2548 (None, None) => None,
2549 };
2550 if let Some(gap) = gap {
2551 let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2552 if gap_secs >= stale_secs {
2553 tracing::info!(
2554 stale_secs,
2555 threshold,
2556 suspend_gap_secs = gap_secs,
2557 "tick loop stale window explained by host suspend/resume, not a stall"
2558 );
2559 } else {
2560 tracing::error!(
2561 stale_secs,
2562 threshold,
2563 suspend_gap_secs = gap_secs,
2564 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2565 stale_secs,
2566 threshold,
2567 gap_secs,
2568 );
2569 }
2570 } else {
2571 tracing::error!(
2572 stale_secs,
2573 threshold,
2574 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2575 stale_secs,
2576 threshold,
2577 );
2578 }
2579 }
2580 }
2581 });
2582 }
2583
2584 loop {
2585 tokio::select! {
2586 _ = interval.tick() => {
2587 let tick_start = Instant::now();
2588
2589 if let Some(gap) = engine_suspend::checkpoint() {
2593 tracing::info!(
2594 gap_secs = gap.num_seconds(),
2595 "host suspend/resume gap detected, discounting from stuck-task age checks"
2596 );
2597 }
2598
2599 if review_in_force != config.max_parallel {
2603 review_in_force =
2604 resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2605 }
2606 if parallel_in_force != config.max_parallel {
2607 let resized =
2608 resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2609 if resized != parallel_in_force {
2610 parallel_in_force = resized;
2611 tracing::info!(
2612 in_force = parallel_in_force,
2613 target = config.max_parallel,
2614 "dispatch pool resized"
2615 );
2616 }
2617 }
2618
2619 {
2621 let mut rw = router.write().await;
2622 rw.tick_weight_recovery();
2623 }
2624
2625 if let Some(remaining) = GhHttp::is_rate_limited() {
2629 let reset_at = chrono::Utc::now()
2630 .timestamp()
2631 .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2632 if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2634 rate_limit_reported = Some(reset_at);
2635 report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2636 }
2637 } else if let Some(prev) = rate_limit_reported.take() {
2638 tracing::info!(
2639 limited_until = prev,
2640 "GitHub API rate limit cleared"
2641 );
2642 if let Some(engine) = project_engines.first() {
2643 let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2644 }
2645 }
2646
2647 tick::tick_all(
2649 &project_engines,
2650 &tmux,
2651 &capture_for_tick,
2652 &semaphore,
2653 &config,
2654 &router,
2655 &weight_tx,
2656 &dispatching,
2657 Some(&transport),
2658 &active_repos,
2659 &auto_merge_in_flight,
2660 ).await;
2661
2662 while let Ok(signal) = weight_rx.try_recv() {
2665 let mut rw = router.write().await;
2666 match signal {
2667 WeightSignal::RateLimited { ref agent } => {
2668 rw.record_rate_limit(agent);
2669 }
2670 WeightSignal::Success { ref agent } => {
2671 rw.record_success(agent);
2672 }
2673 WeightSignal::Failed { .. }
2674 | WeightSignal::Rerouted
2675 | WeightSignal::Blocked
2676 | WeightSignal::None => {}
2677 }
2678 }
2679
2680 if last_sync.elapsed() >= config.sync_interval {
2685 if sync_in_progress.compare_exchange(
2686 false,
2687 true,
2688 Ordering::AcqRel,
2689 Ordering::Relaxed,
2690 ).is_ok() {
2691 spawn_sync_pass(
2692 &project_engines,
2693 &tmux,
2694 &config,
2695 &router,
2696 &dispatching,
2697 &auto_merge_in_flight,
2698 &sync_in_progress,
2699 "periodic",
2700 );
2701 } else {
2702 tracing::debug!("sync tick still in progress, skipping");
2703 }
2704 last_sync = Instant::now();
2708 }
2709
2710 if webhook_enabled {
2712 let health_check_interval = config.webhook_health_check_interval
2713 .unwrap_or(Duration::from_secs(60));
2714
2715 if last_webhook_health_check.elapsed() >= health_check_interval {
2716 if let Some(port) = webhook_port {
2717 let (health, failure_reason) =
2718 github::check_webhook_health(port).await;
2719 if health != webhook_healthy {
2720 webhook_healthy = health;
2721 if webhook_healthy {
2722 tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2723 } else {
2724 tracing::warn!(
2725 port,
2726 orch_webhook_up = false,
2727 reason = failure_reason.as_deref().unwrap_or("unknown"),
2728 "webhook health check failed"
2729 );
2730 }
2731 }
2732 {
2734 let to_save = {
2735 let mut s = webhook_status.lock().await;
2736 s.healthy = health;
2737 s.last_check_utc = Some(chrono::Utc::now());
2738 if health {
2739 s.last_failure_reason = None;
2740 } else if failure_reason.is_some() {
2741 s.last_failure_reason = failure_reason;
2742 }
2743 s.clone()
2744 };
2745 to_save.save().await;
2746 }
2747 }
2748 last_webhook_health_check = Instant::now();
2749 }
2750 }
2751
2752 if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2754 if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2755 let channels = Arc::clone(&channel_registry);
2756 let current = env!("ORCH_VERSION").to_string();
2757 let check_interval =
2758 Duration::from_secs(config.upgrade_check_interval);
2759 tokio::spawn(async move {
2760 check_and_notify_upgrade(&store, &channels, ¤t, check_interval)
2761 .await;
2762 });
2763 }
2764 last_upgrade_check = Some(Instant::now());
2765 }
2766
2767 let tick_elapsed = tick_start.elapsed();
2769 if tick_elapsed.as_secs() > 30 {
2770 tracing::warn!(
2771 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2772 "slow tick"
2773 );
2774 } else {
2775 tracing::debug!(
2776 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2777 "tick complete"
2778 );
2779 }
2780 last_tick_epoch.store(
2781 SystemTime::now()
2782 .duration_since(std_time::UNIX_EPOCH)
2783 .unwrap_or_default()
2784 .as_secs(),
2785 Ordering::Relaxed,
2786 );
2787 }
2788 () = webhook_notify.notified() => {
2790 if let Some(remaining) = GhHttp::is_rate_limited() {
2791 tracing::warn!(
2792 remaining_secs = remaining.as_secs(),
2793 "GitHub API rate-limited, skipping webhook-triggered tick"
2794 );
2795 } else {
2796 tracing::info!("webhook event triggered immediate tick");
2797
2798 tick::tick_all(
2799 &project_engines,
2800 &tmux,
2801 &capture_for_tick,
2802 &semaphore,
2803 &config,
2804 &router,
2805 &weight_tx,
2806 &dispatching,
2807 Some(&transport),
2808 &active_repos,
2809 &auto_merge_in_flight,
2810 ).await;
2811
2812 while let Ok(signal) = weight_rx.try_recv() {
2814 let mut rw = router.write().await;
2815 match signal {
2816 WeightSignal::RateLimited { ref agent } => {
2817 rw.record_rate_limit(agent);
2818 }
2819 WeightSignal::Success { ref agent } => {
2820 rw.record_success(agent);
2821 }
2822 WeightSignal::Failed { .. }
2823 | WeightSignal::Rerouted
2824 | WeightSignal::Blocked
2825 | WeightSignal::None => {}
2826 }
2827 }
2828
2829 if sync_in_progress.compare_exchange(
2835 false,
2836 true,
2837 Ordering::AcqRel,
2838 Ordering::Relaxed,
2839 ).is_ok() {
2840 spawn_sync_pass(
2841 &project_engines,
2842 &tmux,
2843 &config,
2844 &router,
2845 &dispatching,
2846 &auto_merge_in_flight,
2847 &sync_in_progress,
2848 "webhook",
2849 );
2850 last_sync = Instant::now();
2852 } else {
2853 tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2854 }
2855 }
2856
2857 interval.reset();
2859 }
2860 result = config_rx.recv() => {
2861 let reload = match result {
2864 Ok(path) => {
2865 tracing::info!(path = %path.display(), "config file changed, reloading");
2866 true
2867 }
2868 Err(RecvError::Lagged(n)) => {
2869 tracing::warn!(missed = n, "config change receiver lagged, reloading");
2870 true
2871 }
2872 Err(RecvError::Closed) => {
2873 tracing::warn!("config change channel closed");
2874 false
2875 }
2876 };
2877
2878 if reload {
2879 let new_config = EngineConfig::from_config();
2881 log_engine_config_changes(&config, &new_config);
2882
2883 let new_restart_only = restart_only_snapshot();
2884 log_restart_only_changes(&restart_only, &new_restart_only);
2885 restart_only = new_restart_only;
2886
2887 let tick_changed = new_config.tick_interval != config.tick_interval;
2888 config = new_config;
2889
2890 if tick_changed {
2892 interval = time::interval(config.tick_interval);
2893 tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2894 }
2895
2896 if review_in_force != config.max_parallel {
2899 review_in_force = resize_dispatch_pool(
2900 &review_semaphore,
2901 review_in_force,
2902 config.max_parallel,
2903 );
2904 }
2905 if parallel_in_force != config.max_parallel {
2906 parallel_in_force = resize_dispatch_pool(
2907 &semaphore,
2908 parallel_in_force,
2909 config.max_parallel,
2910 );
2911 tracing::info!(
2912 in_force = parallel_in_force,
2913 target = config.max_parallel,
2914 "dispatch pool resized"
2915 );
2916 }
2917
2918 {
2920 let mut router_guard = router.write().await;
2921 router_guard.reload_async().await;
2922 }
2923
2924 sync_project_engines(
2926 &mut project_engines,
2927 &mut active_repos,
2928 &event_bus,
2929 &tmux,
2930 &capture_for_tick,
2931 &semaphore,
2932 &review_semaphore,
2933 &weight_tx,
2934 &router,
2935 &dispatching,
2936 ).await;
2937
2938 tracing::info!(
2939 tick = ?config.tick_interval,
2940 sync = ?config.sync_interval,
2941 parallel = config.max_parallel,
2942 projects = project_engines.len(),
2943 "config reloaded"
2944 );
2945 }
2946 }
2947 signal_name = async {
2948 tokio::select! {
2949 _ = signal::ctrl_c() => "SIGINT",
2950 _ = sigterm.recv() => "SIGTERM",
2951 _ = sighup.recv() => "SIGHUP",
2952 }
2953 } => {
2954 tracing::info!(signal = signal_name, "beginning graceful shutdown");
2955
2956 let shutdown_result = time::timeout(
2959 config.graceful_shutdown_timeout,
2960 async {
2961 let mut reset_count = 0u32;
2965 let mut review_reset_count = 0u32;
2966 for engine in &project_engines {
2967 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
2968 for task in &tasks {
2969 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
2970 tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
2971 } else {
2972 if let Ok(Some(store_id)) =
2973 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
2974 && let Err(e) = engine
2975 .store
2976 .finalize_incomplete_runs(
2977 store_id,
2978 "aborted",
2979 "graceful shutdown: reset in_progress task to routed",
2980 )
2981 .await
2982 {
2983 tracing::warn!(
2984 task_id = task.id.0,
2985 ?e,
2986 "failed to finalize incomplete runs on shutdown"
2987 );
2988 }
2989 reset_count += 1;
2990 }
2991 }
2992 }
2993 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
2995 for task in &tasks {
2996 let task_id = format!("internal:{}", task.id);
2997 if let Err(e) = engine.task_manager.update_task_status(
2998 &ExternalId(task_id.clone()),
2999 Status::Routed,
3000 ).await {
3001 tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3002 } else {
3003 if let Err(e) = engine
3004 .store
3005 .finalize_incomplete_runs(
3006 task.id,
3007 "aborted",
3008 "graceful shutdown: reset in_progress task to routed",
3009 )
3010 .await
3011 {
3012 tracing::warn!(
3013 task_id,
3014 ?e,
3015 "failed to finalize incomplete internal runs on shutdown"
3016 );
3017 }
3018 reset_count += 1;
3019 }
3020 }
3021 }
3022 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3024 for task in &tasks {
3025 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3026 tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3027 } else {
3028 if let Ok(Some(store_id)) =
3029 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3030 && let Err(e) = engine
3031 .store
3032 .finalize_incomplete_runs(
3033 store_id,
3034 "aborted",
3035 "graceful shutdown: reset in_review task to needs_review",
3036 )
3037 .await
3038 {
3039 tracing::warn!(
3040 task_id = task.id.0,
3041 ?e,
3042 "failed to finalize incomplete review runs on shutdown"
3043 );
3044 }
3045 review_reset_count += 1;
3046 }
3047 }
3048 }
3049 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3051 for task in &tasks {
3052 let task_id = format!("internal:{}", task.id);
3053 if let Err(e) = engine.task_manager.update_task_status(
3054 &ExternalId(task_id.clone()),
3055 Status::NeedsReview,
3056 ).await {
3057 tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3058 } else {
3059 if let Err(e) = engine
3060 .store
3061 .finalize_incomplete_runs(
3062 task.id,
3063 "aborted",
3064 "graceful shutdown: reset in_review task to needs_review",
3065 )
3066 .await
3067 {
3068 tracing::warn!(
3069 task_id,
3070 ?e,
3071 "failed to finalize incomplete internal review runs on shutdown"
3072 );
3073 }
3074 review_reset_count += 1;
3075 }
3076 }
3077 }
3078 }
3079 if reset_count > 0 {
3080 tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3081 }
3082 if review_reset_count > 0 {
3083 tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3084 }
3085
3086 let mut killed_sessions = Vec::new();
3089 if let Ok(sessions) = tmux.list_sessions().await {
3090 for session in &sessions {
3091 if session.name.starts_with("orch-") {
3092 match tmux.kill_session(&session.name).await { Err(e) => {
3093 tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3094 } _ => {
3095 killed_sessions.push(session.name.clone());
3096 }}
3097 }
3098 }
3099 if !killed_sessions.is_empty() {
3100 tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3101 }
3102 }
3103
3104 if !killed_sessions.is_empty() {
3107 let still_alive = tmux
3108 .wait_for_sessions_dead(
3109 &killed_sessions,
3110 Duration::from_millis(100),
3111 Duration::from_secs(5),
3112 )
3113 .await;
3114 if still_alive > 0 {
3115 tracing::warn!(
3116 still_alive,
3117 "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3118 );
3119 }
3120 tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3121 }
3122 },
3123 ).await;
3124
3125 if shutdown_result.is_err() {
3126 tracing::error!(
3127 timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3128 "graceful shutdown timed out, forcing exit"
3129 );
3130 }
3131
3132 break;
3133 }
3134 }
3135 }
3136
3137 events::cleanup_port_file().await;
3139 local::cleanup().await;
3140 events::cleanup_version_file();
3141
3142 let _ = transport;
3144 tracing::info!("orch engine stopped");
3145 Ok(())
3146}
3147
3148#[cfg(test)]
3149mod tests {
3150 use std::sync::Arc;
3151
3152 use super::*;
3153 use crate::backends::test_helpers::NoopBackend;
3154 use crate::channels::capture::CaptureService;
3155 use crate::channels::transport::Transport;
3156 use crate::tmux::SessionGuard;
3157 use crate::tmux::TmuxManager;
3158
3159 async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3160 let id = store
3161 .upsert_external(&crate::store::UpsertExternal {
3162 repo: "owner/repo",
3163 ext_id: number,
3164 title: "Under review",
3165 body: "",
3166 author: "",
3167 url: "",
3168 labels: &[],
3169 origin: "github",
3170 })
3171 .await
3172 .unwrap();
3173 store.update_status(id, TaskStatus::InReview).await.unwrap();
3174 id
3175 }
3176
3177 #[tokio::test]
3178 async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3179 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3180 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3181 let task_manager =
3182 TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3183 let tmux = TmuxManager::isolated();
3184 let orphaned = in_review_task(&store, "11").await;
3185 let live = in_review_task(&store, "12").await;
3186 let session = tmux
3187 .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3188 .await
3189 .unwrap();
3190 let _guard = SessionGuard(session);
3191
3192 let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3193
3194 assert_eq!(reset, 1);
3195 assert_eq!(
3196 store.get(orphaned).await.unwrap().status,
3197 TaskStatus::NeedsReview
3198 );
3199 assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3200 }
3201
3202 #[test]
3203 fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3204 let semaphore = Arc::new(Semaphore::new(4));
3205 let held: Vec<_> = (0..4)
3206 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3207 .collect();
3208 assert!(
3209 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3210 "pool is full"
3211 );
3212
3213 let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3214 assert_eq!(in_force, 8);
3215
3216 let grown: Vec<_> = (0..4)
3217 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3218 .collect();
3219 assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3220 assert!(
3221 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3222 "pool is full"
3223 );
3224 }
3225
3226 #[test]
3227 fn shrinking_max_parallel_never_cancels_a_running_task() {
3228 let semaphore = Arc::new(Semaphore::new(8));
3229 let mut held: Vec<_> = (0..8)
3230 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3231 .collect();
3232
3233 let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3235 assert_eq!(in_force, 8);
3236 assert_eq!(held.len(), 8);
3237
3238 held.truncate(5);
3240 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3241 assert_eq!(in_force, 5);
3242 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3243
3244 held.truncate(4);
3246 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3247 assert_eq!(in_force, 4);
3248 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3249
3250 held.truncate(3);
3252 assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3253 }
3254
3255 #[test]
3256 fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3257 let configured = vec![
3258 ("owner/one".to_string(), PathBuf::from("/srv/one")),
3259 ("owner/two".to_string(), PathBuf::from("/srv/two")),
3260 ];
3261 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3262
3263 let (added, removed) = diff_configured_projects(&configured, &active);
3264 assert_eq!(
3265 added,
3266 vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3267 "only the new project is started, so owner/one keeps its single set of subscribers"
3268 );
3269 assert!(removed.is_empty(), "{removed:?}");
3270 }
3271
3272 #[test]
3273 fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3274 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3275 let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3276 .into_iter()
3277 .collect();
3278
3279 let (added, removed) = diff_configured_projects(&configured, &active);
3280 assert!(added.is_empty(), "nothing new to start");
3281 assert_eq!(removed, vec!["owner/gone".to_string()]);
3282 }
3283
3284 #[test]
3285 fn an_unchanged_project_list_starts_nothing() {
3286 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3287 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3288
3289 assert_eq!(
3290 diff_configured_projects(&configured, &active),
3291 (vec![], vec![])
3292 );
3293 }
3294
3295 #[test]
3296 fn restart_only_keys_are_the_settings_read_once_at_startup() {
3297 assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3300 }
3301
3302 #[test]
3303 fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3304 let before = vec![Some("bot_token: a\n".to_string()), None];
3305
3306 assert!(
3307 log_restart_only_changes(&before, &before.clone()).is_empty(),
3308 "a reload that did not touch these keys must stay quiet"
3309 );
3310
3311 let after = vec![
3313 Some("bot_token: b\n".to_string()),
3314 Some("port: 8080\n".to_string()),
3315 ];
3316 assert_eq!(
3317 log_restart_only_changes(&before, &after),
3318 ["channels", "webhook"]
3319 );
3320
3321 let token_only = vec![Some("bot_token: b\n".to_string()), None];
3323 assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3324 }
3325
3326 #[test]
3327 fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3328 let semaphore = Arc::new(Semaphore::new(4));
3329 assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3330 assert_eq!(semaphore.available_permits(), 4);
3331 }
3332
3333 #[test]
3334 fn live_session_stuck_limit_follows_the_run_timeout() {
3335 let config = EngineConfig {
3336 stuck_timeout: 1800,
3337 run_timeouts: RunTimeouts {
3338 base: 1800,
3339 medium: 2700,
3340 complex: 3600,
3341 },
3342 ..EngineConfig::default()
3343 };
3344 assert_eq!(
3345 config.live_session_stuck_limit(Some("simple")),
3346 1800 + RUN_TIMEOUT_GRACE_SECS
3347 );
3348 assert_eq!(
3349 config.live_session_stuck_limit(Some("medium")),
3350 2700 + RUN_TIMEOUT_GRACE_SECS
3351 );
3352 assert_eq!(
3353 config.live_session_stuck_limit(Some("complex")),
3354 3600 + RUN_TIMEOUT_GRACE_SECS
3355 );
3356
3357 let long_stuck = EngineConfig {
3358 stuck_timeout: 7200,
3359 ..config
3360 };
3361 assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3362 }
3363
3364 #[test]
3365 fn engine_config_defaults() {
3366 let config = EngineConfig::default();
3367 assert_eq!(config.tick_interval, Duration::from_secs(10));
3368 assert_eq!(config.sync_interval, Duration::from_secs(45));
3369 assert_eq!(config.max_parallel, 4);
3370 assert_eq!(config.stuck_timeout, 1800);
3371 assert_eq!(config.no_session_stuck_timeout, 600);
3372 assert!(
3373 config.no_session_stuck_timeout < config.stuck_timeout,
3374 "no_session_stuck_timeout must be shorter than stuck_timeout"
3375 );
3376 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3377 assert_eq!(config.upgrade_check_interval, 3600);
3378 }
3379
3380 #[test]
3381 fn engine_config_from_config_uses_defaults_when_no_config() {
3382 let config = EngineConfig::from_config();
3383 assert_eq!(config.tick_interval, Duration::from_secs(10));
3384 assert_eq!(config.sync_interval, Duration::from_secs(45));
3385 assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3387 assert_eq!(config.stuck_timeout, 1800);
3388 assert_eq!(config.no_session_stuck_timeout, 600);
3389 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3390 assert_eq!(config.upgrade_check_interval, 3600);
3391 }
3392
3393 #[test]
3394 fn engine_config_no_session_timeout_is_under_15_minutes() {
3395 let config = EngineConfig::default();
3396 assert!(
3397 config.no_session_stuck_timeout <= 900,
3398 "no_session_stuck_timeout ({}) exceeds 15 minutes",
3399 config.no_session_stuck_timeout
3400 );
3401 }
3402
3403 #[test]
3404 fn upgrade_check_runs_immediately_when_never_checked() {
3405 assert!(should_run_upgrade_check(None, 3600));
3406 }
3407
3408 #[test]
3409 fn upgrade_check_waits_for_interval_after_first_run() {
3410 assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3411 }
3412
3413 #[test]
3414 fn upgrade_check_can_be_disabled() {
3415 assert!(!should_run_upgrade_check(None, 0));
3416 }
3417
3418 #[test]
3419 fn configured_agents_parses_yaml_block_array() {
3420 let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3422 let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3423 assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3424 }
3425
3426 #[test]
3427 fn configured_agents_parses_json_inline_array() {
3428 let json_str = r#"["claude","codex","olm"]"#;
3429 let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3430 assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3431 }
3432
3433 #[test]
3434 fn configured_agents_fallback_returns_default_agents() {
3435 let defaults = configured_agents();
3437 assert!(defaults.contains(&"claude".to_string()));
3438 assert!(defaults.contains(&"codex".to_string()));
3439 assert!(defaults.contains(&"opencode".to_string()));
3440 }
3441
3442 #[tokio::test]
3451 #[ignore = "capture-pane timing is unreliable on CI runners"]
3452 async fn integration_channel_to_tmux_to_capture() {
3453 let transport = Arc::new(Transport::new());
3455 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3456 let tmux = TmuxManager::new();
3457
3458 let task_id = "testtask1".to_string();
3459 let session_name = format!("orch-test-{task_id}");
3460
3461 let _ = tmux
3463 .command()
3464 .args(["new-session", "-d", "-s", &session_name])
3465 .output()
3466 .await
3467 .expect("failed to start tmux session");
3468
3469 let _guard = SessionGuard(session_name.clone());
3471
3472 capture
3474 .register_session("owner/repo", &task_id, &session_name)
3475 .await;
3476 transport
3477 .bind(
3478 "owner/repo",
3479 &task_id,
3480 &session_name,
3481 "telegram",
3482 "12345",
3483 None,
3484 )
3485 .await;
3486
3487 let capture_clone = Arc::clone(&capture);
3489 let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3490
3491 let mut rx = transport
3493 .subscribe("owner/repo", &task_id)
3494 .await
3495 .expect("no subscription");
3496
3497 let send = tmux
3499 .command()
3500 .args([
3501 "send-keys",
3502 "-t",
3503 &session_name,
3504 "echo from-channel",
3505 "Enter",
3506 ])
3507 .output()
3508 .await
3509 .expect("failed to send keys");
3510 assert!(send.status.success());
3511
3512 let mut got = false;
3514 for _ in 0..20 {
3515 if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3516 && c.content.contains("from-channel")
3517 {
3518 got = true;
3519 break;
3520 }
3521 }
3522
3523 capture.unregister_session("owner/repo", &task_id).await;
3525
3526 let _ = capture_handle.await;
3528
3529 assert!(got, "did not observe tmux output via capture/transport");
3530 }
3531
3532 #[test]
3535 fn read_project_channel_config_returns_defaults_when_no_file() {
3536 let dir = tempfile::tempdir().expect("tempdir");
3537 let cfg = read_project_channel_config(dir.path());
3538 assert!(cfg.telegram_topic_id.is_none());
3539 assert!(cfg.discord_channel_id.is_none());
3540 }
3541
3542 #[test]
3543 fn read_project_channel_config_parses_telegram_and_discord() {
3544 let dir = tempfile::tempdir().expect("tempdir");
3545 let orch_yml = dir.path().join(".orch.yml");
3546 fs::write(
3547 &orch_yml,
3548 "channels:\n telegram:\n topic_id: \"42\"\n discord:\n channel_id: \"9999\"\n",
3549 )
3550 .expect("write .orch.yml");
3551 let cfg = read_project_channel_config(dir.path());
3552 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3553 assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3554 }
3555
3556 #[test]
3557 fn read_project_channel_config_handles_partial_channels() {
3558 let dir = tempfile::tempdir().expect("tempdir");
3559 let orch_yml = dir.path().join(".orch.yml");
3560 fs::write(&orch_yml, "channels:\n telegram:\n topic_id: \"7\"\n")
3561 .expect("write .orch.yml");
3562 let cfg = read_project_channel_config(dir.path());
3563 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3564 assert!(cfg.discord_channel_id.is_none());
3565 }
3566
3567 #[test]
3568 fn read_project_channel_config_handles_malformed_yaml() {
3569 let dir = tempfile::tempdir().expect("tempdir");
3570 let orch_yml = dir.path().join(".orch.yml");
3571 fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3572 let cfg = read_project_channel_config(dir.path());
3573 assert!(cfg.telegram_topic_id.is_none());
3575 assert!(cfg.discord_channel_id.is_none());
3576 }
3577
3578 #[test]
3579 #[cfg(unix)]
3580 fn read_project_channel_config_handles_unreadable_file() {
3581 let dir = tempfile::tempdir().expect("tempdir");
3582 fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3584 let cfg = read_project_channel_config(dir.path());
3585 assert!(cfg.telegram_topic_id.is_none());
3587 assert!(cfg.discord_channel_id.is_none());
3588 }
3589}