1pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{
58 local_branch_exists, remove_worktree_and_branch, remove_worktree_and_branch_for,
59};
60use crate::engine::router::{Router, RouterConfig};
61use crate::engine::runner::worktree::{
62 abort_worktree_rebase, list_project_worktrees, task_id_from_worktree_name,
63 validate_worktree_gitdir,
64};
65use crate::engine::tasks::TaskManager;
66use crate::github::http::{GhHttp, rate_limit_metrics};
67use crate::repo_context::REPO_CONTEXT;
68use crate::store::{TaskStatus, TaskStore};
69use crate::tmux::TmuxManager;
70use runner::WeightSignal;
71use crate::backends::Status;
73use crate::backends::github::GitHubBackend;
74use crate::channels::github;
75use crate::channels::github::is_transient_bind_error;
76use crate::channels::github::webhook_backoff_delay;
77use crate::channels::notification;
78use crate::channels::notification::TaskNotification;
79use crate::channels::transport;
80use crate::cluster;
81use crate::cluster::local;
82use crate::engine::cooldown as engine_cooldown;
83use crate::engine::review as engine_review;
84use crate::engine::runner::git_ops;
85use crate::engine::runner::worktree;
86use crate::engine::suspend as engine_suspend;
87use crate::engine::suspend::SuspendDetector;
88use crate::engine::tasks as engine_tasks;
89use crate::github::app;
90use crate::home;
91use crate::secret_ref;
92use crate::store;
93use crate::webhook_status::WebhookStatus;
94use futures::future;
95use std::collections::HashMap;
96use std::collections::HashSet;
97use std::fmt;
98use std::fmt::Debug;
99use std::fmt::Formatter;
100use std::fs;
101use std::path::Path;
102use std::path::PathBuf;
103use std::sync::Arc;
104use std::sync::atomic::AtomicBool;
105use std::sync::atomic::AtomicU64;
106use std::sync::atomic::Ordering;
107use std::time as std_time;
108use std::time::Duration;
109use std::time::Instant;
110use std::time::SystemTime;
111use tokio::process::Command;
112use tokio::signal;
113use tokio::signal::unix;
114use tokio::signal::unix::SignalKind;
115use tokio::sync::Mutex;
116use tokio::sync::broadcast::Sender;
117use tokio::sync::broadcast::error::RecvError;
118use tokio::sync::mpsc::Receiver;
119use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
120use tokio::task;
121use tokio::time;
122
123pub fn orch_footer() -> &'static str {
124 "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
125}
126
127pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
131 match model {
132 Some(m) => format!(
133 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
134 ),
135 None => format!(
136 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
137 ),
138 }
139}
140
141type EngineRef = (
143 String,
144 Arc<dyn ExternalBackend>,
145 Arc<TaskManager>,
146 Option<Arc<TaskStore>>,
147);
148
149#[derive(Clone)]
154pub struct ProjectEngine {
155 pub repo: String,
156 pub project_dir: PathBuf,
157 pub backend: Arc<dyn ExternalBackend>,
158 pub task_manager: Arc<TaskManager>,
159 pub runner: Arc<runner::TaskRunner>,
160 pub store: Arc<TaskStore>,
161}
162
163impl Debug for ProjectEngine {
164 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
165 f.debug_struct("ProjectEngine")
166 .field("repo", &self.repo)
167 .field("project_dir", &self.project_dir)
168 .finish_non_exhaustive()
169 }
170}
171
172pub(crate) async fn reset_orphaned_reviews(
175 task_manager: &TaskManager,
176 store: &Arc<TaskStore>,
177 repo: &str,
178 tmux: &TmuxManager,
179) -> usize {
180 let mut ids: Vec<ExternalId> = store
182 .list_by_status(repo, TaskStatus::InReview)
183 .await
184 .unwrap_or_default()
185 .iter()
186 .map(|t| engine_tasks::store_task_to_external(t).id)
187 .collect();
188 if let Ok(legacy) = task_manager
189 .list_internal_by_status(TaskStatus::InReview)
190 .await
191 {
192 ids.extend(legacy.into_iter().map(|t| t.id));
193 }
194 ids.sort_by(|a, b| a.0.cmp(&b.0));
195 ids.dedup_by(|a, b| a.0 == b.0);
196
197 let mut reset = 0;
198 for id in &ids {
199 let session = tmux.session_name(repo, &format!("{}-review", id.0));
200
201 if tmux.session_exists(&session).await {
202 continue;
203 }
204
205 if let Err(e) = task_manager
206 .update_task_status(id, Status::NeedsReview)
207 .await
208 {
209 tracing::warn!(
210 task_id = id.0,
211 ?e,
212 "failed to reset an orphaned review at start"
213 );
214 continue;
215 }
216
217 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
218 let reason = "startup: review session gone, reset to needs_review";
219
220 if let Err(e) = store
221 .finalize_incomplete_runs(store_id, "aborted", reason)
222 .await
223 {
224 tracing::warn!(
225 task_id = id.0,
226 ?e,
227 "failed to finalize the orphaned review's runs"
228 );
229 }
230 }
231
232 reset += 1;
233 }
234
235 reset
236}
237
238fn start_relay_client(project_engines: &[ProjectEngine], notify: &Arc<Notify>) {
241 let url = config::get_global("relay.url").unwrap_or_default();
243 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
244 return;
245 };
246
247 if url.is_empty() {
248 return;
249 }
250
251 let token_ref = config::get_global("relay.token").unwrap_or_default();
252 let notify = Arc::clone(notify);
253 tokio::spawn(async move {
254 let mut warned = false;
255 let token = loop {
256 let reference = token_ref.clone();
257 let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
258 .await
259 .map_err(anyhow::Error::from)
260 .and_then(|r| r);
261
262 match read {
263 Ok(token) if !token.is_empty() => break token,
264 Ok(_) => {
265 tracing::warn!("relay.token is empty, the relay client stays off");
266 return;
267 }
268 Err(e) if !warned => {
269 tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
270 warned = true;
271 }
272 Err(_) => {}
273 }
274
275 time::sleep(Duration::from_secs(300)).await;
276 };
277
278 relay::run(url, token, store, notify).await;
279 });
280}
281
282#[expect(
288 clippy::too_many_arguments,
289 reason = "the spawned task takes its own handle to each piece of engine state"
290)]
291fn spawn_sync_pass(
292 project_engines: &[ProjectEngine],
293 tmux: &Arc<TmuxManager>,
294 engine_config: &EngineConfig,
295 router: &Arc<RwLock<Router>>,
296 dispatching: &Arc<dashmap::DashMap<String, String>>,
297 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
298 sync_in_progress: &Arc<AtomicBool>,
299 trigger: &'static str,
300) {
301 let engines = project_engines.to_vec();
302 let tmux = Arc::clone(tmux);
303 let engine_config = engine_config.clone();
304 let router = Arc::clone(router);
305 let dispatching = Arc::clone(dispatching);
306 let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
307 let guard = Arc::clone(sync_in_progress);
308
309 tokio::spawn(async move {
310 let started = Instant::now();
311 let mut passes = Vec::with_capacity(engines.len());
312 for engine in &engines {
313 passes.push(REPO_CONTEXT.scope(
314 engine.repo.clone(),
315 sync_one_project(
316 engine,
317 &tmux,
318 &engine_config,
319 &router,
320 &dispatching,
321 &auto_merge_in_flight,
322 ),
323 ));
324 }
325 future::join_all(passes).await;
326
327 sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
328 tracing::info!(
329 trigger,
330 elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
331 "sync tick complete"
332 );
333 guard.store(false, Ordering::Release);
334 });
335}
336
337async fn sync_one_project(
338 engine: &ProjectEngine,
339 tmux: &Arc<TmuxManager>,
340 engine_config: &EngineConfig,
341 router: &Arc<RwLock<Router>>,
342 dispatching: &Arc<dashmap::DashMap<String, String>>,
343 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
344) {
345 if let Err(e) = Box::pin(sync::sync_tick(
346 &engine.backend,
347 tmux,
348 &engine.repo,
349 engine_config,
350 router,
351 &engine.task_manager,
352 &engine.store,
353 dispatching,
354 auto_merge_in_flight,
355 ))
356 .await
357 {
358 tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
359 }
360}
361
362const SLOW_PHASE_GROUP_MS: u128 = 1_000;
364
365pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
371 let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
372 if total_ms >= SLOW_PHASE_GROUP_MS {
373 tracing::info!(group, repo, total_ms, ?phases, "phase timings");
374 } else {
375 tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
376 }
377}
378
379pub fn configured_agents() -> Vec<String> {
391 if let Ok(agents_str) = config::get("agents")
392 && !agents_str.is_empty()
393 && agents_str != "[]"
394 {
395 if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
398 && !agents_arr.is_empty()
399 {
400 return agents_arr;
401 }
402 if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
404 && !agents_arr.is_empty()
405 {
406 return agents_arr;
407 }
408 }
409 router::config::DEFAULT_AGENTS
410 .iter()
411 .map(ToString::to_string)
412 .collect()
413}
414
415pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
418
419#[derive(Clone, Copy, Debug, PartialEq, Eq)]
421pub struct RunTimeouts {
422 pub base: u64,
423 pub medium: u64,
424 pub complex: u64,
425}
426
427impl Default for RunTimeouts {
428 fn default() -> Self {
429 Self {
430 base: 1800,
431 medium: 1800,
432 complex: 1800,
433 }
434 }
435}
436
437impl RunTimeouts {
438 pub fn from_config() -> Self {
441 let read =
442 |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
443 let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
444 Self {
445 base,
446 medium: read("workflow.timeout_by_complexity.medium")
447 .unwrap_or(base)
448 .max(base),
449 complex: read("workflow.timeout_by_complexity.complex")
450 .unwrap_or(base)
451 .max(base),
452 }
453 }
454
455 pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
456 match complexity {
457 Some("complex") => self.complex,
458 Some("medium") => self.medium,
459 _ => self.base,
460 }
461 }
462}
463
464#[derive(Clone, Debug)]
466pub struct EngineConfig {
467 pub tick_interval: Duration,
469 pub sync_interval: Duration,
471 pub webhook_health_check_interval: Option<Duration>,
473 pub max_parallel: usize,
475 pub stuck_timeout: u64,
477 pub no_session_stuck_timeout: u64,
480 pub in_review_no_session_stuck_timeout: u64,
487 pub run_timeouts: RunTimeouts,
489 pub auto_create_followup_on_changes: bool,
491 pub graceful_shutdown_timeout: Duration,
493 pub silence_grace_period: u64,
496 pub silence_cooldown: u64,
498 pub upgrade_check_interval: u64,
501 pub startup_failure_escalation_secs: u64,
506}
507
508impl Default for EngineConfig {
509 fn default() -> Self {
510 Self {
511 tick_interval: Duration::from_secs(10),
512 sync_interval: Duration::from_secs(45),
513 webhook_health_check_interval: Some(Duration::from_secs(60)),
514 max_parallel: 4,
515 stuck_timeout: 1800,
516 no_session_stuck_timeout: 600,
517 in_review_no_session_stuck_timeout: 1920,
518 run_timeouts: RunTimeouts::default(),
519 auto_create_followup_on_changes: true,
520 graceful_shutdown_timeout: Duration::from_secs(600),
521 silence_grace_period: 300,
522 silence_cooldown: 3600,
523 upgrade_check_interval: 3600,
524 startup_failure_escalation_secs: 3600,
525 }
526 }
527}
528
529impl EngineConfig {
530 pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
533 self.stuck_timeout.max(
534 self.run_timeouts
535 .for_complexity(complexity)
536 .saturating_add(RUN_TIMEOUT_GRACE_SECS),
537 )
538 }
539
540 pub fn from_config() -> Self {
542 let mut config = Self::default();
543
544 if let Ok(val) = config::get("engine.tick_interval") {
545 if let Ok(secs) = val.parse::<u64>() {
546 config.tick_interval = Duration::from_secs(secs);
547 } else {
548 tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
549 }
550 }
551
552 if let Ok(val) = config::get("engine.sync_interval") {
553 if let Ok(secs) = val.parse::<u64>() {
554 config.sync_interval = Duration::from_secs(secs);
555 } else {
556 tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
557 }
558 }
559
560 if let Ok(val) = config::get("engine.max_parallel") {
561 if let Ok(n) = val.parse::<usize>() {
562 config.max_parallel = n;
563 } else {
564 tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
565 }
566 }
567
568 if let Ok(val) = config::get("engine.stuck_timeout") {
569 if let Ok(secs) = val.parse::<u64>() {
570 config.stuck_timeout = secs;
571 } else {
572 tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
573 }
574 }
575
576 if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
577 if let Ok(secs) = val.parse::<u64>() {
578 config.no_session_stuck_timeout = secs;
579 } else {
580 tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
581 }
582 }
583
584 let mut in_review_from_config = false;
585 if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
586 if let Ok(secs) = val.parse::<u64>() {
587 config.in_review_no_session_stuck_timeout = secs;
588 in_review_from_config = true;
589 } else {
590 tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
591 }
592 }
593
594 let review_timeout = engine_review::review_timeout_seconds();
598 let floored = engine_review::no_session_review_floor(
599 config.in_review_no_session_stuck_timeout,
600 review_timeout,
601 );
602 if floored != config.in_review_no_session_stuck_timeout {
603 if in_review_from_config {
604 tracing::warn!(
605 configured_secs = config.in_review_no_session_stuck_timeout,
606 review_timeout_secs = review_timeout,
607 raised_to_secs = floored,
608 "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
609 );
610 } else {
611 tracing::debug!(
612 review_timeout_secs = review_timeout,
613 raised_to_secs = floored,
614 "raised the default in_review no-session limit over a long review timeout"
615 );
616 }
617 config.in_review_no_session_stuck_timeout = floored;
618 }
619
620 config.run_timeouts = RunTimeouts::from_config();
621
622 if let Ok(val) = config::get("engine.webhook_health_check_interval") {
623 if let Ok(secs) = val.parse::<u64>() {
624 if secs > 0 {
625 config.webhook_health_check_interval = Some(Duration::from_secs(secs));
626 } else {
627 config.webhook_health_check_interval = None;
628 }
629 } else {
630 tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
631 }
632 }
633
634 if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
635 config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
636 }
637
638 if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
639 if let Ok(secs) = val.parse::<u64>() {
640 config.graceful_shutdown_timeout = Duration::from_secs(secs);
641 } else {
642 tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
643 }
644 }
645
646 if let Ok(val) = config::get("engine.silence_grace_period") {
647 if let Ok(secs) = val.parse::<u64>() {
648 config.silence_grace_period = secs;
649 } else {
650 tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
651 }
652 }
653
654 if let Ok(val) = config::get("engine.silence_cooldown") {
655 if let Ok(secs) = val.parse::<u64>() {
656 config.silence_cooldown = secs;
657 } else {
658 tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
659 }
660 }
661
662 if let Ok(val) = config::get("engine.upgrade_check_interval") {
663 if let Ok(secs) = val.parse::<u64>() {
664 config.upgrade_check_interval = secs;
665 } else {
666 tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
667 }
668 }
669
670 if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
671 if let Ok(secs) = val.parse::<u64>() {
672 config.startup_failure_escalation_secs = secs;
673 } else {
674 tracing::warn!(
675 key = "engine.startup_failure_escalation_secs",
676 value = %val,
677 default_secs = config.startup_failure_escalation_secs,
678 "invalid value for engine.startup_failure_escalation_secs, using default"
679 );
680 }
681 }
682
683 config
684 }
685}
686
687pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
690
691pub fn format_rate_limited_until(reset_at: i64) -> String {
693 chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
694 || reset_at.to_string(),
695 |t| {
696 t.with_timezone(&chrono::Local)
697 .format("%H:%M:%S %Z")
698 .to_string()
699 },
700 )
701}
702
703async fn report_rate_limited(
706 reset_at: i64,
707 store: Option<&Arc<TaskStore>>,
708 transport: Option<&Arc<Transport>>,
709) {
710 let m = rate_limit_metrics();
711 let until = format_rate_limited_until(reset_at);
712 tracing::warn!(
713 reset_at,
714 rate_limit_hits = m.hits,
715 skipped_calls = m.skipped_calls,
716 five_xx_hits = m.five_xx_hits,
717 rest_remaining = ?m.rest_remaining,
718 graphql_remaining = ?m.graphql_remaining,
719 "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
720 );
721
722 if let Some(store) = store
723 && let Err(e) = store
724 .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
725 .await
726 {
727 tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
728 }
729
730 if let Some(transport) = transport {
731 transport.push_notification(TaskNotification {
732 task_id: "github".to_string(),
733 title: "GitHub rate limit".to_string(),
734 status: "blocked".to_string(),
735 agent: "orch".to_string(),
736 duration_seconds: 0.0,
737 summary: format!(
738 "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
739 ),
740 repo: None,
741 notify_target: None,
742 pr_number: None,
743 });
744 }
745}
746
747fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
753 if target > in_force {
754 semaphore.add_permits(target - in_force);
755 return target;
756 }
757
758 if target < in_force {
759 return in_force - semaphore.forget_permits(in_force - target);
760 }
761
762 in_force
763}
764
765const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
768
769fn restart_only_snapshot() -> Vec<Option<String>> {
771 RESTART_ONLY_KEYS
772 .iter()
773 .map(|key| config::get(key).ok())
774 .collect()
775}
776
777fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
780 let changed: Vec<&'static str> = RESTART_ONLY_KEYS
781 .iter()
782 .zip(old)
783 .zip(new)
784 .filter(|((_, old), new)| old != new)
785 .map(|((key, _), _)| *key)
786 .collect();
787
788 for key in &changed {
789 tracing::info!(key = key, "config changed, takes effect after restart");
790 }
791
792 changed
793}
794
795fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
798 macro_rules! log_changed {
799 ($($key:literal => $field:ident),+ $(,)?) => {
800 $(if old.$field != new.$field {
801 tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
802 })+
803 };
804 }
805
806 log_changed!(
807 "engine.tick_interval" => tick_interval,
808 "engine.sync_interval" => sync_interval,
809 "engine.webhook_health_check_interval" => webhook_health_check_interval,
810 "engine.max_parallel" => max_parallel,
811 "engine.stuck_timeout" => stuck_timeout,
812 "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
813 "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
814 "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
815 "engine.silence_grace_period" => silence_grace_period,
816 "engine.silence_cooldown" => silence_cooldown,
817 "engine.upgrade_check_interval" => upgrade_check_interval,
818 "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
819 "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
820 );
821}
822
823fn should_run_upgrade_check(
824 last_upgrade_check: Option<Instant>,
825 upgrade_check_interval: u64,
826) -> bool {
827 if upgrade_check_interval == 0 {
828 return false;
829 }
830
831 last_upgrade_check
832 .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
833}
834
835async fn connect_project_engine(
842 repo: String,
843 project_dir: PathBuf,
844 store: Arc<TaskStore>,
845 event_tx: Option<Sender<events::TaskEvent>>,
846) -> anyhow::Result<ProjectEngine> {
847 let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
848
849 match backend.health_check().await {
853 Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
854 Err(e) => tracing::warn!(
855 repo = %repo,
856 error = %e,
857 "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
858 ),
859 }
860
861 let task_manager = Arc::new(match event_tx {
862 Some(tx) => {
863 TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
864 }
865 None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
866 });
867 let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
868
869 Ok(ProjectEngine {
870 repo,
871 project_dir,
872 backend,
873 task_manager,
874 runner,
875 store,
876 })
877}
878
879fn diff_configured_projects(
886 configured: &[(String, PathBuf)],
887 active_repos: &HashSet<String>,
888) -> (Vec<(String, PathBuf)>, Vec<String>) {
889 let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
890
891 let added = configured
892 .iter()
893 .filter(|(repo, _)| !active_repos.contains(repo))
894 .cloned()
895 .collect();
896 let removed = active_repos
897 .iter()
898 .filter(|repo| !configured_repos.contains(repo.as_str()))
899 .cloned()
900 .collect();
901
902 (added, removed)
903}
904
905#[expect(
911 clippy::too_many_arguments,
912 reason = "a new project engine needs every shared engine handle"
913)]
914async fn sync_project_engines(
915 project_engines: &mut Vec<ProjectEngine>,
916 active_repos: &mut HashSet<String>,
917 event_bus: &events::EventBus,
918 tmux: &Arc<TmuxManager>,
919 capture: &Arc<CaptureService>,
920 semaphore: &Arc<Semaphore>,
921 review_semaphore: &Arc<Semaphore>,
922 weight_tx: &mpsc::Sender<WeightSignal>,
923 router: &Arc<RwLock<Router>>,
924 dispatching: &Arc<dashmap::DashMap<String, String>>,
925) {
926 let configured = match config::get_projects_with_paths() {
927 Ok(p) if !p.is_empty() => p,
928 Ok(_) => return,
929 Err(e) => {
930 tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
931 return;
932 }
933 };
934 let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) else {
935 return;
936 };
937
938 let (added, removed) = diff_configured_projects(&configured, active_repos);
939 for repo in removed {
940 tracing::info!(
941 repo = %repo,
942 "project removed from config, takes effect after restart"
943 );
944 }
945
946 for (repo, project_dir) in added {
947 config::watch(&project_dir.join(".orch.yml"));
948 let engine = match connect_project_engine(
949 repo.clone(),
950 project_dir,
951 Arc::clone(&store),
952 Some(event_bus.sender()),
953 )
954 .await
955 {
956 Ok(engine) => engine,
957 Err(e) => {
958 tracing::warn!(
961 repo = %repo,
962 err = %e,
963 "project added to config but its backend could not be built, retrying on the next config change"
964 );
965 continue;
966 }
967 };
968
969 subscribers::dispatch::spawn(
970 event_bus.subscribe(),
971 Arc::clone(&engine.backend),
972 Arc::clone(tmux),
973 Arc::clone(&engine.runner),
974 Arc::clone(capture),
975 Arc::clone(semaphore),
976 Arc::clone(&engine.task_manager),
977 weight_tx.clone(),
978 Arc::clone(router),
979 Arc::clone(dispatching),
980 Arc::clone(&engine.store),
981 engine.repo.clone(),
982 );
983 subscribers::review::spawn(
984 event_bus.subscribe(),
985 Arc::clone(&engine.backend),
986 Arc::clone(tmux),
987 Arc::clone(review_semaphore),
988 Arc::clone(&engine.task_manager),
989 Arc::clone(router),
990 Arc::clone(dispatching),
991 Arc::clone(&engine.store),
992 engine.repo.clone(),
993 );
994 subscribers::unblock::spawn(
995 event_bus.subscribe(),
996 Arc::clone(&engine.backend),
997 Arc::clone(&engine.task_manager),
998 Arc::clone(&engine.store),
999 engine.repo.clone(),
1000 );
1001
1002 active_repos.insert(repo.clone());
1003 project_engines.push(engine);
1004 tracing::info!(repo = %repo, "project added from config, now syncing");
1005 }
1006}
1007
1008async fn init_project_engines() -> anyhow::Result<Vec<ProjectEngine>> {
1012 let projects = config::get_projects_with_paths()?;
1013 let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1014 tracing::info!(repos = ?repos, "loading projects from config");
1015
1016 if projects.is_empty() {
1017 let config_path = home::config_path().map_or_else(
1018 |_| "~/.orch/config.yml".to_string(),
1019 |p| p.display().to_string(),
1020 );
1021 anyhow::bail!("no projects configured — add repos under `projects:` in {config_path}");
1022 }
1023
1024 let mut engines = Vec::new();
1025
1026 let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1029 if cluster::enabled() {
1030 cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1031 local::serve((*store).clone()).await?;
1032 }
1033
1034 engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1037
1038 for (repo, project_dir) in projects {
1039 tracing::info!(repo = %repo, "initializing project engine");
1040 config::watch(&project_dir.join(".orch.yml"));
1041
1042 engines.push(connect_project_engine(repo, project_dir, Arc::clone(&store), None).await?);
1043 }
1044
1045 Ok(engines)
1046}
1047
1048async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1049 for engine in project_engines {
1050 let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1051 let repo_root_path = PathBuf::from(&repo_root);
1052 let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1053 let worktrees = list_project_worktrees(&repo_root_path).await?;
1054
1055 if let Err(e) = git_ops::git()
1058 .await
1059 .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1060 .output_with_context()
1061 .await
1062 {
1063 tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1064 }
1065
1066 let mut orphans_removed: u32 = 0;
1067 let mut invalid_removed: u32 = 0;
1068 let mut cleaned: u32 = 0;
1069 let mut valid_kept: u32 = 0;
1070
1071 for worktree_dir in worktrees {
1072 let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1073 continue;
1074 };
1075
1076 if !validate_worktree_gitdir(&worktree_dir).await {
1079 tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1080 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1081 .await;
1082 invalid_removed += 1;
1083 continue;
1084 }
1085
1086 let Some(task_id) = task_id_from_worktree_name(name) else {
1087 tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1088 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1089 .await;
1090 orphans_removed += 1;
1091 continue;
1092 };
1093
1094 let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1095 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1096 remove_worktree_and_branch(
1097 &task_id,
1098 &worktree_dir,
1099 Some(name),
1100 &repo_root_path,
1101 false,
1102 )
1103 .await;
1104 orphans_removed += 1;
1105 continue;
1106 };
1107
1108 let task = engine.store.get(store_id).await?;
1109 let branch_name = if task.branch.is_empty() {
1110 name
1111 } else {
1112 task.branch.as_str()
1113 };
1114
1115 match task.status {
1116 TaskStatus::New
1117 | TaskStatus::Routed
1118 | TaskStatus::InProgress
1119 | TaskStatus::NeedsReview
1120 | TaskStatus::InReview => {
1121 let keep_conflict = worktree::keep_conflicted_rebase(
1122 task.merge_conflict_retries,
1123 task.pr_number,
1124 );
1125 if keep_conflict && git_ops::rebase_in_progress(&worktree_dir).await {
1126 tracing::info!(
1127 repo = %engine.repo,
1128 task_id = %task_id,
1129 worktree = %worktree_dir.display(),
1130 "keeping the conflicted rebase auto_merge left for the agent"
1131 );
1132 valid_kept += 1;
1133 continue;
1134 }
1135 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1136 let origin_branch = format!("origin/{default_branch}");
1137 let outcome = git_ops::stash_rebase_restore(
1138 &worktree_dir,
1139 &origin_branch,
1140 git_ops::StashRebaseOpts {
1141 abort_on_failure: false,
1142 },
1143 )
1144 .await;
1145 let conflict_left = git_ops::rebase_in_progress(&worktree_dir).await;
1146 let failed_err = match outcome {
1147 Ok(
1148 git_ops::RebaseOutcome::Succeeded | git_ops::RebaseOutcome::Skipped(_),
1149 ) => None,
1150 Ok(git_ops::RebaseOutcome::Failed(e)) if keep_conflict && conflict_left => {
1151 tracing::info!(
1152 repo = %engine.repo,
1153 task_id = %task_id,
1154 err = %e,
1155 "startup rebase hit a content conflict, leaving it for the agent"
1156 );
1157 None
1158 }
1159 Ok(git_ops::RebaseOutcome::Failed(e)) => Some(e),
1160 Err(e) => Some(e.to_string()),
1161 };
1162 if let Some(e) = failed_err {
1163 reset_startup_task_after_failed_rebase(
1164 engine,
1165 &task_id,
1166 &worktree_dir,
1167 branch_name,
1168 &repo_root_path,
1169 task.pr_number.is_some(),
1170 &e,
1171 )
1172 .await;
1173 cleaned += 1;
1174 } else {
1175 valid_kept += 1;
1176 }
1177 }
1178 TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1181 valid_kept += 1;
1182 }
1183 TaskStatus::Done => {
1184 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1185 remove_worktree_and_branch_for(
1186 &task_id,
1187 &worktree_dir,
1188 Some(branch_name),
1189 &repo_root_path,
1190 false,
1191 task.pr_number.is_some(),
1192 )
1193 .await;
1194 cleaned += 1;
1195 }
1196 }
1197 }
1198
1199 tracing::info!(
1200 repo = %engine.repo,
1201 orphans_removed,
1202 invalid_removed,
1203 cleaned,
1204 valid_kept,
1205 "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1206 );
1207
1208 let prune = Command::new("git")
1211 .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1212 .output_with_context()
1213 .await;
1214 match prune {
1215 Ok(output) if output.status.success() => {
1216 tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1217 }
1218 Ok(output) => {
1219 let stderr = String::from_utf8_lossy(&output.stderr);
1220 tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1221 }
1222 Err(e) => {
1223 tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1224 }
1225 }
1226 }
1227
1228 Ok(())
1229}
1230
1231async fn reset_startup_task_after_failed_rebase(
1232 engine: &ProjectEngine,
1233 task_id: &str,
1234 worktree_dir: &Path,
1235 branch_name: &str,
1236 repo_root: &Path,
1237 keep_remote: bool,
1238 err: &str,
1239) {
1240 tracing::warn!(repo = %engine.repo, task_id, err, "startup rebase failed, resetting task");
1241 abort_worktree_rebase(worktree_dir).await;
1242 remove_worktree_and_branch(
1243 task_id,
1244 worktree_dir,
1245 Some(branch_name),
1246 repo_root,
1247 keep_remote,
1248 )
1249 .await;
1250 let keep_branch = local_branch_exists(repo_root, branch_name).await;
1251 match engine.store.resolve_task_id(&engine.repo, task_id).await {
1252 Ok(Some(reset_id)) => {
1253 let result = if keep_branch {
1254 engine.store.reset_to_new_keep_branch(reset_id).await
1255 } else {
1256 engine.store.reset_to_new(reset_id).await
1257 };
1258 if let Err(e) = result {
1259 tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1260 }
1261 }
1262 Ok(None) => {
1263 tracing::warn!(task_id, "task not found in store during reconciliation");
1264 }
1265 Err(e) => {
1266 tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1267 }
1268 }
1269 if let Err(e) = engine
1270 .task_manager
1271 .update_task_status(&ExternalId(task_id.to_string()), Status::New)
1272 .await
1273 {
1274 tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1275 }
1276}
1277
1278async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1286 for engine in project_engines {
1287 let tasks = match engine.store.list_external_tasks_with_estimates().await {
1288 Ok(t) => t,
1289 Err(e) => {
1290 tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1291 continue;
1292 }
1293 };
1294
1295 if tasks.is_empty() {
1296 continue;
1297 }
1298
1299 let mut synced: u32 = 0;
1300 let mut failed: u32 = 0;
1301 for (external_id, estimate) in &tasks {
1302 match engine
1303 .backend
1304 .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1305 .await
1306 {
1307 Ok(()) => synced += 1,
1308 Err(e) => {
1309 tracing::debug!(
1310 repo = %engine.repo,
1311 task_id = %external_id,
1312 estimate,
1313 err = %e,
1314 "startup estimate reconciliation: sync failed"
1315 );
1316 failed += 1;
1317 }
1318 }
1319 }
1320
1321 if synced > 0 || failed > 0 {
1322 tracing::info!(
1323 repo = %engine.repo,
1324 synced,
1325 failed,
1326 "startup estimate reconciliation complete"
1327 );
1328 }
1329 }
1330}
1331
1332fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1334 let config_path = project_dir.join(".orch.yml");
1335 if !config_path.exists() {
1336 return ProjectChannelConfig::default();
1337 }
1338 let content = match fs::read_to_string(&config_path) {
1339 Ok(c) => c,
1340 Err(e) => {
1341 tracing::warn!(
1342 path = %config_path.display(),
1343 error = %e,
1344 "failed to read .orch.yml; channel config will use defaults"
1345 );
1346 return ProjectChannelConfig::default();
1347 }
1348 };
1349 let val: serde_norway::Value = match serde_norway::from_str(&content) {
1350 Ok(v) => v,
1351 Err(e) => {
1352 tracing::warn!(
1353 path = %config_path.display(),
1354 error = %e,
1355 "failed to parse .orch.yml; channel config will use defaults"
1356 );
1357 return ProjectChannelConfig::default();
1358 }
1359 };
1360 let Some(channels) = val.get("channels") else {
1361 return ProjectChannelConfig::default();
1362 };
1363 let get_str = |section: &str, key: &str| -> Option<String> {
1364 channels
1365 .get(section)
1366 .and_then(|s| s.get(key))
1367 .and_then(|v| v.as_str())
1368 .map(String::from)
1369 };
1370 ProjectChannelConfig {
1371 telegram_topic_id: get_str("telegram", "topic_id"),
1372 discord_channel_id: get_str("discord", "channel_id"),
1373 }
1374}
1375
1376async fn fetch_latest_release_version() -> Option<String> {
1381 let output = Command::new("gh")
1382 .args([
1383 "api",
1384 "repos/gabrielkoerich/orch/releases/latest",
1385 "--jq",
1386 ".tag_name",
1387 ])
1388 .output()
1389 .await
1390 .ok()?;
1391 if !output.status.success() {
1392 return None;
1393 }
1394 let tag = String::from_utf8(output.stdout).ok()?;
1395 let version = tag.trim().trim_start_matches('v').to_string();
1396 if version.is_empty() {
1397 None
1398 } else {
1399 Some(version)
1400 }
1401}
1402
1403async fn check_and_notify_upgrade(
1412 store: &Arc<TaskStore>,
1413 channels: &Arc<ChannelRegistry>,
1414 current_version: &str,
1415 check_interval: Duration,
1416) {
1417 let last_check_key = "upgrade:last_check_at";
1418 let last_notified_key = "upgrade:last_notified_version";
1419
1420 let now = chrono::Utc::now().timestamp();
1422 if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1423 && let Ok(last_check) = last_check_str.parse::<i64>()
1424 && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1425 {
1426 return;
1427 }
1428
1429 let _ = store.kv_set(last_check_key, &now.to_string()).await;
1431
1432 let Some(latest) = fetch_latest_release_version().await else {
1433 tracing::debug!("upgrade check: failed to fetch latest release version");
1434 return;
1435 };
1436
1437 if latest == current_version {
1438 tracing::debug!(current = %current_version, "orch is up to date");
1439 return;
1440 }
1441
1442 tracing::warn!(
1443 current_version = %current_version,
1444 latest_version = %latest,
1445 "orch upgrade available"
1446 );
1447
1448 if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1453 && last_notified == latest
1454 {
1455 tracing::debug!(latest = %latest, "upgrade already notified");
1456 return;
1457 }
1458
1459 let msg_telegram = format!(
1460 "⚠️ <b>Orch Upgrade Available</b>\n\n\
1461 Service: <code>{current_version}</code>\n\
1462 Latest: <code>{latest}</code>\n\n\
1463 Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1464 );
1465
1466 let msg_discord = format!(
1467 "⚠️ **Orch Upgrade Available**\n\n\
1468 Service: `{current_version}`\n\
1469 Latest: `{latest}`\n\n\
1470 Run: `brew update && brew upgrade orch && brew services restart orch`"
1471 );
1472
1473 let msg_slack = format!(
1474 "⚠️ *Orch Upgrade Available*\n\n\
1475 Service: `{current_version}`\n\
1476 Latest: `{latest}`\n\n\
1477 Run: `brew update && brew upgrade orch && brew services restart orch`"
1478 );
1479
1480 for channel in channels.iter() {
1481 let (body, metadata, topic_id) = match channel.name() {
1482 "telegram" => (
1483 msg_telegram.clone(),
1484 serde_json::json!({"preformatted_html": true}),
1485 None,
1486 ),
1487 "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1488 "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1489 _ => continue,
1490 };
1491
1492 let msg = OutgoingMessage {
1493 thread_id: "upgrade".to_string(),
1494 body,
1495 reply_to: None,
1496 metadata,
1497 topic_id,
1498 };
1499
1500 if let Err(e) = channel.send(&msg).await {
1501 tracing::warn!(
1502 channel = channel.name(),
1503 ?e,
1504 "failed to send upgrade notification"
1505 );
1506 }
1507 }
1508
1509 let _ = store.kv_set(last_notified_key, &latest).await;
1511}
1512
1513async fn send_startup_escalation_notification(elapsed_secs: u64) {
1521 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1522 let duration = notification::format_duration(elapsed_secs as f64);
1523
1524 let msg_telegram = format!(
1525 "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1526 The engine has been retrying backend connections for {duration}\n\
1527 without success. All task processing, routing, and scheduled\n\
1528 jobs are paused. Check service logs with `orch log -f`."
1529 );
1530
1531 let msg_discord = format!(
1532 "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1533 The engine has been retrying backend connections for {duration}\n\
1534 without success. All task processing, routing, and scheduled\n\
1535 jobs are paused. Check service logs with `orch log -f`."
1536 );
1537
1538 let msg_slack = format!(
1539 "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1540 The engine has been retrying backend connections for {duration}\n\
1541 without success. All task processing, routing, and scheduled\n\
1542 jobs are paused. Check service logs with `orch log -f`."
1543 );
1544
1545 let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1547 let tg_chat_id = config::get("channels.telegram.chat_id");
1548 if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1549 && !token.is_empty()
1550 && !chat_id.is_empty()
1551 && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1552 {
1553 let msg = OutgoingMessage {
1554 thread_id: "startup_escalation".to_string(),
1555 body: msg_telegram.clone(),
1556 reply_to: None,
1557 metadata: serde_json::json!({ "preformatted_html": true }),
1558 topic_id: config::get("channels.telegram.general_topic_id").ok(),
1559 };
1560 match channel.send(&msg).await {
1561 Err(e) => {
1562 tracing::warn!(?e, "failed to send startup escalation via telegram");
1563 }
1564 _ => {
1565 tracing::info!("startup escalation notification sent to telegram");
1566 }
1567 }
1568 }
1569
1570 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1572 let channel_id = config::get("channels.discord.channel_id")
1573 .or_else(|_| config::get("channels.discord.general_channel_id"));
1574 if !token.is_empty()
1575 && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1576 {
1577 let shard_id = config::get("channels.discord.shard_id")
1578 .ok()
1579 .and_then(|v| v.parse::<u64>().ok())
1580 .unwrap_or(0);
1581 let shard_count = config::get("channels.discord.shard_count")
1582 .ok()
1583 .and_then(|v| v.parse::<u64>().ok())
1584 .unwrap_or(1);
1585 if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1586 let msg = OutgoingMessage {
1587 thread_id: "startup_escalation".to_string(),
1588 body: msg_discord.clone(),
1589 reply_to: None,
1590 metadata: serde_json::Value::Null,
1591 topic_id: None,
1592 };
1593 match gateway.send(&msg).await {
1594 Err(e) => {
1595 tracing::warn!(?e, "failed to send startup escalation via discord");
1596 }
1597 _ => {
1598 tracing::info!("startup escalation notification sent to discord");
1599 }
1600 }
1601 }
1602 }
1603 }
1604
1605 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1607 let channel_id = config::get("channels.slack.channel_id");
1608 if !token.is_empty()
1609 && let Ok(cid) = channel_id
1610 && let Ok(channel) = SlackChannel::new(token, Some(cid))
1611 {
1612 let msg = OutgoingMessage {
1613 thread_id: "startup_escalation".to_string(),
1614 body: msg_slack.clone(),
1615 reply_to: None,
1616 metadata: serde_json::Value::Null,
1617 topic_id: None,
1618 };
1619 match channel.send(&msg).await {
1620 Err(e) => {
1621 tracing::warn!(?e, "failed to send startup escalation via slack");
1622 }
1623 _ => {
1624 tracing::info!("startup escalation notification sent to slack");
1625 }
1626 }
1627 }
1628 }
1629}
1630
1631pub async fn serve() -> anyhow::Result<()> {
1635 let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1636 if killed > 0 {
1637 tracing::info!(count = killed, "terminated stale orch serve processes");
1638 }
1639
1640 tracing::info!("orch engine starting");
1641
1642 if let Err(e) = home::init_dirs().await {
1646 tracing::warn!("failed to initialize orch directories: {e}");
1647 }
1648
1649 if let Err(e) = config::warm_cache().await {
1652 tracing::warn!("failed to warm config cache: {e}");
1653 }
1654
1655 let mut config = EngineConfig::from_config();
1656
1657 tracing::info!("internal database ready");
1658
1659 let startup_escalation_secs = config.startup_failure_escalation_secs;
1668 let mut project_engines = {
1669 let mut delay_secs = 5u64;
1670 let mut attempt = 0u32;
1671 let mut escalated = false;
1672 let start = Instant::now();
1673 loop {
1674 attempt += 1;
1675 match init_project_engines().await {
1676 Ok(engines) => break engines,
1677 Err(e) => {
1678 let elapsed = start.elapsed().as_secs();
1679
1680 if startup_escalation_secs > 0
1686 && !escalated
1687 && elapsed >= startup_escalation_secs
1688 {
1689 escalated = true;
1690 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1691 let outage = notification::format_duration(elapsed as f64);
1692 tracing::error!(
1693 delay_secs,
1694 attempt,
1695 elapsed_secs = elapsed,
1696 threshold_secs = startup_escalation_secs,
1697 "project backends unavailable for {outage} — escalating startup failure: {e}",
1698 );
1699 send_startup_escalation_notification(elapsed).await;
1703 } else {
1704 tracing::warn!(
1705 delay_secs,
1706 attempt,
1707 "project backends unavailable, retrying: {e}"
1708 );
1709 }
1710
1711 time::sleep(Duration::from_secs(delay_secs)).await;
1712 delay_secs = (delay_secs * 2).min(120);
1713 }
1714 }
1715 }
1716 };
1717
1718 tracing::info!(
1719 project_count = project_engines.len(),
1720 "initialized project engines"
1721 );
1722
1723 let mut active_repos: HashSet<String> =
1727 project_engines.iter().map(|e| e.repo.clone()).collect();
1728
1729 let event_bus = events::EventBus::new(256);
1731 if let Err(e) = event_bus.start_ws_server().await {
1732 tracing::warn!(
1733 ?e,
1734 "failed to start event websocket server, continuing without it"
1735 );
1736 }
1737
1738 {
1742 let current = env!("ORCH_VERSION").to_string();
1743 tokio::spawn(async move {
1744 if let Some(latest) = fetch_latest_release_version().await
1745 && latest != current
1746 {
1747 tracing::warn!(
1748 current_version = %current,
1749 latest_version = %latest,
1750 "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1751 );
1752 }
1753 });
1754 }
1755
1756 for engine in &mut project_engines {
1758 engine.task_manager = Arc::new(TaskManager::with_events(
1759 Arc::clone(&engine.backend),
1760 Arc::clone(&engine.store),
1761 engine.repo.clone(),
1762 event_bus.sender(),
1763 ));
1764 }
1765
1766 let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1768 if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1769 tracing::warn!(err = %e, "startup worktree reconciliation failed");
1770 }
1771
1772 if leads {
1773 let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1777 tokio::spawn(async move {
1778 reconcile_startup_estimates(&engines_snapshot).await;
1779 });
1780 }
1781
1782 let global_channel_config = GlobalChannelConfig {
1784 telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1785 discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1786 control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1787 control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1788 };
1789 let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1790 .iter()
1791 .map(|e| {
1792 let cfg = read_project_channel_config(&e.project_dir);
1793 (e.repo.clone(), cfg)
1794 })
1795 .collect();
1796 let channel_router = Arc::new(ChannelRouter::new(
1797 &global_channel_config,
1798 &project_channel_configs,
1799 ));
1800 tracing::info!(
1801 projects = ?channel_router.projects(),
1802 "channel router initialized"
1803 );
1804
1805 let tmux = Arc::new(TmuxManager::new());
1807 tick::startup_cleanup(&tmux).await;
1810
1811 let transport = Arc::new(Transport::new());
1813
1814 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1816 let capture_for_tick = Arc::clone(&capture);
1817 tokio::spawn(async move {
1818 capture.start().await;
1819 });
1820
1821 let mut channel_registry = ChannelRegistry::new();
1823
1824 if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1826 && !token.is_empty()
1827 {
1828 let chat_id = config::get("channels.telegram.chat_id").ok();
1829 match TelegramChannel::new(token, chat_id) {
1830 Ok(telegram) => {
1831 if let Err(e) = telegram.health_check().await {
1832 tracing::warn!(?e, "telegram channel health check failed, skipping");
1833 } else {
1834 channel_registry.register(Box::new(telegram));
1835 tracing::info!("telegram channel registered");
1836 }
1837 }
1838 Err(e) => {
1839 tracing::warn!(?e, "failed to create telegram channel, skipping");
1840 }
1841 }
1842 }
1843
1844 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1846 && !token.is_empty()
1847 {
1848 let channel_id = config::get("channels.discord.channel_id").ok();
1849 let shard_id = config::get("channels.discord.shard_id")
1850 .ok()
1851 .and_then(|v| v.parse::<u64>().ok())
1852 .unwrap_or(0);
1853 let shard_count = config::get("channels.discord.shard_count")
1854 .ok()
1855 .and_then(|v| v.parse::<u64>().ok())
1856 .unwrap_or(1);
1857 match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1858 Ok(discord) => {
1859 if let Err(e) = discord.health_check().await {
1860 tracing::warn!(?e, "discord gateway health check failed, skipping");
1861 } else {
1862 channel_registry.register(Box::new(discord));
1863 tracing::info!(shard_id, shard_count, "discord gateway registered");
1864 }
1865 }
1866 Err(e) => {
1867 tracing::warn!(?e, "failed to create discord gateway, skipping");
1868 }
1869 }
1870 }
1871
1872 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1874 && !token.is_empty()
1875 {
1876 let channel_id = config::get("channels.slack.channel_id").ok();
1877 match SlackChannel::new(token, channel_id) {
1878 Ok(slack) => {
1879 if let Err(e) = slack.health_check().await {
1880 tracing::warn!(?e, "slack channel health check failed, skipping");
1881 } else {
1882 channel_registry.register(Box::new(slack));
1883 tracing::info!("slack channel registered");
1884 }
1885 }
1886 Err(e) => {
1887 tracing::warn!(?e, "failed to create slack channel, skipping");
1888 }
1889 }
1890 }
1891
1892 let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1894 channel_registry.register(Box::new(tmux_channel));
1895 tracing::info!("tmux channel registered");
1896
1897 let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1899 for channel in channel_registry.iter() {
1900 match channel.start().await {
1901 Ok(rx) => {
1902 tracing::info!(channel = channel.name(), "channel started");
1903 channel_receivers.push(rx);
1904 }
1905 Err(e) => {
1906 tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1907 }
1908 }
1909 }
1910
1911 let channel_registry = Arc::new(channel_registry);
1913
1914 let transport_for_messages = Arc::clone(&transport);
1916 let tmux_for_messages = Arc::clone(&tmux);
1917 let capture_for_messages = Arc::clone(&capture_for_tick);
1918 let channels_for_messages = Arc::clone(&channel_registry);
1919 let engine_refs: Vec<EngineRef> = project_engines
1921 .iter()
1922 .map(|e| {
1923 (
1924 e.repo.clone(),
1925 Arc::clone(&e.backend),
1926 Arc::clone(&e.task_manager),
1927 Some(Arc::clone(&e.store)),
1928 )
1929 })
1930 .collect();
1931 let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1933 let router_for_messages = Arc::clone(&channel_router);
1934 for mut rx in channel_receivers {
1935 let transport = Arc::clone(&transport_for_messages);
1936 let tmux = Arc::clone(&tmux_for_messages);
1937 let capture = Arc::clone(&capture_for_messages);
1938 let channels = Arc::clone(&channels_for_messages);
1939 let engine_refs = engine_refs.clone();
1940 let ch_router = Arc::clone(&router_for_messages);
1941 let picks = Arc::clone(&pending_picks);
1942 tokio::spawn(async move {
1943 while let Some(msg) = rx.recv().await {
1944 tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1945 channel_handler::handle_channel_message(
1946 msg,
1947 &transport,
1948 &tmux,
1949 &capture,
1950 &channels,
1951 &engine_refs,
1952 &ch_router,
1953 &picks,
1954 )
1955 .await;
1956 }
1957 });
1958 }
1959
1960 {
1964 let restart_channels = Arc::clone(&channel_registry);
1965 let restart_store = project_engines.first().map(|e| Arc::clone(&e.store));
1966 if let Some(store) = restart_store {
1967 tokio::spawn(async move {
1968 time::sleep(Duration::from_secs(5)).await;
1969 match store.kv_get("control:restart_pending").await {
1970 Ok(Some(conv_key)) => {
1971 if let Some((ch_name, thread_id, topic_id)) =
1972 transport::parse_conversation_key(&conv_key)
1973 {
1974 for ch in restart_channels.iter() {
1975 if ch.name() == ch_name {
1976 let msg = OutgoingMessage {
1977 thread_id: thread_id.to_string(),
1978 body: "orch restarted successfully.".to_string(),
1979 reply_to: None,
1980 metadata: serde_json::json!({}),
1981 topic_id: topic_id.map(ToString::to_string),
1982 };
1983 if let Err(e) = ch.send(&msg).await {
1984 tracing::warn!(
1985 channel = ch_name,
1986 error = %e,
1987 "failed to send restart confirmation"
1988 );
1989 }
1990 break;
1991 }
1992 }
1993 }
1994 if let Err(e) = store.kv_delete("control:restart_pending").await {
1995 tracing::warn!(error = %e, "failed to clear control:restart_pending");
1996 }
1997 }
1998 Ok(None) => {} Err(e) => {
2000 tracing::warn!(error = %e, "failed to read control:restart_pending");
2001 }
2002 }
2003 });
2004 }
2005 }
2006
2007 {
2018 let mut notification_rx = transport.subscribe_notifications();
2019 let channels = Arc::clone(&channel_registry);
2020 let notif_router = Arc::clone(&channel_router);
2021 let notif_store: Option<Arc<TaskStore>> =
2023 project_engines.first().map(|e| Arc::clone(&e.store));
2024 tokio::spawn(async move {
2025 loop {
2026 match notification_rx.recv().await {
2027 Ok(notification) => {
2028 let level = NotificationLevel::from_config();
2029 if !level.should_notify(¬ification.status) {
2030 tracing::debug!(
2031 task_id = %notification.task_id,
2032 status = %notification.status,
2033 "notification suppressed by level={:?}",
2034 level
2035 );
2036 continue;
2037 }
2038
2039 tracing::info!(
2040 task_id = %notification.task_id,
2041 status = %notification.status,
2042 repo = ?notification.repo,
2043 "routing notification to channels"
2044 );
2045
2046 let mut routed = false;
2048
2049 if let Some(ref target) = notification.notify_target {
2052 let telegram = channels.iter().find(|c| c.name() == "telegram");
2053 if let Some(channel) = telegram {
2054 let body = notification.format_telegram();
2055 let metadata = serde_json::json!({
2056 "preformatted_html": true,
2057 "chat_id_override": target
2058 });
2059 let msg = OutgoingMessage {
2060 thread_id: notification.task_id.clone(),
2061 body,
2062 reply_to: None,
2063 metadata,
2064 topic_id: None,
2065 };
2066 match channel.send(&msg).await {
2067 Err(e) => {
2068 tracing::warn!(
2069 task_id = %notification.task_id,
2070 target,
2071 ?e,
2072 "failed to send to notify_target"
2073 );
2074 }
2075 _ => {
2076 routed = true;
2077 }
2078 }
2079 }
2080 }
2081
2082 if routed {
2083 } else if let Some(repo) = notification.repo.as_deref() {
2085 let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2091
2092 for channel in channels.iter() {
2094 let ch_name = channel.name();
2095 if let Some(topic_id) =
2096 notif_router.target_for_project(repo, ch_name)
2097 {
2098 let body = match ch_name {
2099 "telegram" => notification.format_telegram(),
2100 "discord" => notification.format_discord(),
2101 "slack" => notification.format_slack(),
2102 _ => continue,
2103 };
2104 let metadata = if ch_name == "telegram" {
2105 serde_json::json!({"preformatted_html": true})
2106 } else {
2107 serde_json::json!({})
2108 };
2109 let resolved_topic = Some(topic_id.to_string());
2110 let msg = OutgoingMessage {
2111 thread_id: notification.task_id.clone(),
2112 body,
2113 reply_to: None,
2114 metadata,
2115 topic_id: resolved_topic.clone(),
2116 };
2117 if let Err(e) = channel.send(&msg).await {
2118 tracing::warn!(
2119 channel = ch_name,
2120 task_id = %notification.task_id,
2121 ?e,
2122 "failed to send to dedicated channel"
2123 );
2124 } else {
2125 sent.insert((ch_name.to_string(), resolved_topic));
2126 routed = true;
2127 }
2128 }
2129 }
2130
2131 if let Some(store) = ¬if_store {
2133 match store.list_subscribers_for_repo(repo).await {
2134 Ok(subscribers) => {
2135 for (ch_name, thread_id, topic_id) in subscribers {
2136 let channel =
2137 channels.iter().find(|c| c.name() == ch_name);
2138 let Some(channel) = channel else {
2139 continue;
2140 };
2141 let resolved_topic = if topic_id.is_empty() {
2142 None
2143 } else {
2144 Some(topic_id.clone())
2145 };
2146 if sent.contains(&(
2149 ch_name.clone(),
2150 resolved_topic.clone(),
2151 )) {
2152 tracing::debug!(
2153 channel = %ch_name,
2154 task_id = %notification.task_id,
2155 topic_id = ?resolved_topic,
2156 "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2157 );
2158 continue;
2159 }
2160 let body = notification.format_with_project(&ch_name);
2161 let metadata = if ch_name == "telegram" {
2162 serde_json::json!({"preformatted_html": true})
2163 } else {
2164 serde_json::json!({})
2165 };
2166 let msg = OutgoingMessage {
2167 thread_id: thread_id.clone(),
2168 body,
2169 reply_to: None,
2170 metadata,
2171 topic_id: resolved_topic.clone(),
2172 };
2173 if let Err(e) = channel.send(&msg).await {
2174 tracing::warn!(
2175 channel = %ch_name,
2176 task_id = %notification.task_id,
2177 ?e,
2178 "failed to send to subscribed channel"
2179 );
2180 } else {
2181 sent.insert((ch_name.clone(), resolved_topic));
2182 routed = true;
2183 }
2184 }
2185 }
2186 Err(e) => {
2187 tracing::warn!(
2188 repo,
2189 ?e,
2190 "failed to list subscribers for repo"
2191 );
2192 }
2193 }
2194 }
2195 }
2196
2197 if !routed {
2199 for channel in channels.iter() {
2200 let (body, should_send, is_telegram) = match channel.name() {
2201 "telegram" => (notification.format_telegram(), true, true),
2202 "discord" => (notification.format_discord(), true, false),
2203 "slack" => (notification.format_slack(), true, false),
2204 _ => (String::new(), false, false),
2207 };
2208
2209 if !should_send {
2210 continue;
2211 }
2212
2213 let metadata = if is_telegram {
2214 serde_json::json!({"preformatted_html": true})
2215 } else {
2216 serde_json::json!({})
2217 };
2218 let msg = OutgoingMessage {
2219 thread_id: notification.task_id.clone(),
2220 body,
2221 reply_to: None,
2222 metadata,
2223 topic_id: None,
2224 };
2225
2226 if let Err(e) = channel.send(&msg).await {
2227 tracing::warn!(
2228 channel = channel.name(),
2229 task_id = %notification.task_id,
2230 ?e,
2231 "failed to send notification"
2232 );
2233 }
2234 }
2235 }
2236 }
2237 Err(RecvError::Lagged(n)) => {
2238 tracing::warn!(missed = n, "notification receiver lagged");
2239 }
2240 Err(RecvError::Closed) => {
2241 tracing::debug!("notification channel closed");
2242 break;
2243 }
2244 }
2245 }
2246 });
2247 tracing::info!("notification dispatcher started");
2248 }
2249
2250 let webhook_notify = Arc::new(Notify::new());
2252
2253 let webhook_port: Option<u16>;
2256 let mut webhook_healthy: bool;
2257 let mut last_webhook_health_check = Instant::now();
2258 let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2260 let webhook_secret = if webhook_enabled {
2262 secret_ref::config_secret("webhook.secret")
2263 .await
2264 .ok()
2265 .filter(|s| !s.trim().is_empty())
2266 } else {
2267 None
2268 };
2269 if webhook_enabled && webhook_secret.is_none() {
2270 tracing::error!(
2271 "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2272 );
2273 }
2274 let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2275 let webhook_secret = webhook_secret.unwrap_or_default();
2276
2277 let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2279
2280 if webhook_enabled {
2281 const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2285 let port: u16 = config::get("webhook.port")
2286 .ok()
2287 .and_then(|v| v.parse().ok())
2288 .unwrap_or(8080);
2289 webhook_port = Some(port);
2290 let secret = webhook_secret.clone();
2291 let webhook_repo = project_engines
2292 .first()
2293 .map(|e| e.repo.clone())
2294 .unwrap_or_default();
2295
2296 let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2297
2298 let to_save = {
2300 let mut s = webhook_status.lock().await;
2301 s.configured = true;
2302 s.port = Some(port);
2303 s.healthy = false;
2304 s.fallback_mode = false;
2305 s.clone()
2306 };
2307 to_save.save().await;
2308
2309 let status_for_spawn = Arc::clone(&webhook_status);
2310 tokio::spawn(async move {
2311 let mut attempt = 0u32;
2312 loop {
2313 attempt += 1;
2314 tracing::info!(attempt, port, "attempting to start webhook server");
2315 match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2316 .await
2317 {
2318 Ok(()) => {
2319 tracing::info!(attempt, "webhook server exited cleanly");
2320 break;
2321 }
2322 Err(e) => {
2323 let reason = e.to_string();
2324 tracing::error!(attempt, %reason, "webhook server failed");
2325 if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2326 let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2327 "max attempts reached"
2328 } else {
2329 "non-transient error"
2330 };
2331 tracing::error!(
2332 attempt,
2333 %reason,
2334 kind,
2335 orch_webhook_in_fallback = true,
2336 "webhook server giving up, switching to polling fallback"
2337 );
2338 let to_save = {
2339 let mut s = status_for_spawn.lock().await;
2340 s.fallback_mode = true;
2341 s.healthy = false;
2342 s.last_failure_reason = Some(reason);
2343 s.startup_attempts = attempt;
2344 s.clone()
2345 };
2346 to_save.save().await;
2347 break;
2348 }
2349 let delay = webhook_backoff_delay(attempt);
2350 tracing::warn!(
2351 attempt,
2352 delay_ms = delay.as_millis(),
2353 %reason,
2354 "webhook bind failed (transient), retrying with backoff"
2355 );
2356 {
2357 let to_save = {
2358 let mut s = status_for_spawn.lock().await;
2359 s.startup_attempts = attempt;
2360 s.last_failure_reason = Some(reason);
2361 s.clone()
2362 };
2363 to_save.save().await;
2364 }
2365 time::sleep(delay).await;
2366 }
2367 }
2368 }
2369 });
2370
2371 let notify = Arc::clone(&webhook_notify);
2373 let transport_for_webhook = Arc::clone(&transport);
2374 tokio::spawn(async move {
2375 while let Some(msg) = rx.recv().await {
2376 tracing::info!(
2377 channel = %msg.channel,
2378 thread = %msg.thread_id,
2379 event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2380 action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2381 "webhook event received, triggering immediate tick"
2382 );
2383 let route = transport_for_webhook.route(&msg).await;
2384 tracing::info!(
2385 channel = %msg.channel,
2386 route = ?route,
2387 "webhook event routed"
2388 );
2389 notify.notify_one();
2391 }
2392 });
2393
2394 webhook_healthy = true;
2395 tracing::info!(port, orch_webhook_up = true, "webhook server started");
2396 } else {
2397 webhook_port = None;
2398 webhook_healthy = false;
2399 {
2400 let to_save = {
2401 let mut s = webhook_status.lock().await;
2402 s.configured = false;
2403 s.fallback_mode = true;
2404 s.clone()
2405 };
2406 to_save.save().await;
2407 }
2408 tracing::info!(
2409 orch_webhook_in_fallback = true,
2410 "webhook server disabled, using polling fallback mode"
2411 );
2412 }
2413
2414 start_relay_client(&project_engines, &webhook_notify);
2415 app::start_loader();
2416
2417 let router = Arc::new(RwLock::new(
2421 match task::spawn_blocking(Router::from_config).await {
2422 Ok(r) => r,
2423 Err(e) => {
2424 tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2425 task::spawn_blocking(|| Router::new(RouterConfig::default()))
2426 .await
2427 .unwrap_or_else(|e2| {
2428 panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2429 })
2430 }
2431 },
2432 ));
2433 {
2434 let r = router.read().await;
2435 tracing::info!(
2436 mode = %r.config.mode,
2437 agents = ?r.available_agents,
2438 fallback = %r.config.fallback_executor,
2439 "router initialized"
2440 );
2441 }
2442
2443 let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2447 let mut parallel_in_force = config.max_parallel;
2448 let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2451 let mut review_in_force = config.max_parallel;
2452
2453 let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2460
2461 let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2465
2466 let mut config_rx = config::subscribe();
2468
2469 let mut last_sync = Instant::now();
2471
2472 let mut last_upgrade_check: Option<Instant> = None;
2474
2475 let mut rate_limit_reported: Option<i64> = None;
2478
2479 let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2481
2482 for engine in project_engines.iter().filter(|_| leads) {
2484 let reset =
2485 reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2486
2487 if reset > 0 {
2488 tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2489 }
2490 }
2491
2492 for engine in &project_engines {
2494 subscribers::dispatch::spawn(
2495 event_bus.subscribe(),
2496 Arc::clone(&engine.backend),
2497 Arc::clone(&tmux),
2498 Arc::clone(&engine.runner),
2499 Arc::clone(&capture_for_tick),
2500 Arc::clone(&semaphore),
2501 Arc::clone(&engine.task_manager),
2502 weight_tx.clone(),
2503 Arc::clone(&router),
2504 Arc::clone(&dispatching),
2505 Arc::clone(&engine.store),
2506 engine.repo.clone(),
2507 );
2508 }
2509 tracing::info!("dispatch subscriber started");
2510
2511 for engine in &project_engines {
2513 subscribers::review::spawn(
2514 event_bus.subscribe(),
2515 Arc::clone(&engine.backend),
2516 Arc::clone(&tmux),
2517 Arc::clone(&review_semaphore),
2518 Arc::clone(&engine.task_manager),
2519 Arc::clone(&router),
2520 Arc::clone(&dispatching),
2521 Arc::clone(&engine.store),
2522 engine.repo.clone(),
2523 );
2524 }
2525 tracing::info!("review subscriber started");
2526
2527 for engine in &project_engines {
2529 subscribers::unblock::spawn(
2530 event_bus.subscribe(),
2531 Arc::clone(&engine.backend),
2532 Arc::clone(&engine.task_manager),
2533 Arc::clone(&engine.store),
2534 engine.repo.clone(),
2535 );
2536 }
2537 tracing::info!("unblock subscriber started");
2538
2539 subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2542 tracing::info!("notify subscriber started");
2543
2544 tracing::info!(
2546 tick = ?config.tick_interval,
2547 sync = ?config.sync_interval,
2548 parallel = config.max_parallel,
2549 "entering main loop"
2550 );
2551 let mut restart_only = restart_only_snapshot();
2554
2555 let mut interval = time::interval(config.tick_interval);
2556
2557 let mut sigterm = unix::signal(SignalKind::terminate())?;
2559 let mut sighup = unix::signal(SignalKind::hangup())?;
2560
2561 let sync_in_progress = Arc::new(AtomicBool::new(false));
2563
2564 let last_tick_epoch = Arc::new(AtomicU64::new(
2567 SystemTime::now()
2568 .duration_since(std_time::UNIX_EPOCH)
2569 .unwrap_or_default()
2570 .as_secs(),
2571 ));
2572 {
2573 let last_tick_epoch = Arc::clone(&last_tick_epoch);
2574 let tick_interval_secs = config.tick_interval.as_secs();
2575 tokio::spawn(async move {
2576 let mut watchdog_interval = time::interval(Duration::from_secs(30));
2578 let mut suspend_detector = SuspendDetector::new();
2583 loop {
2584 watchdog_interval.tick().await;
2585 let own_gap = suspend_detector.check();
2586 let last = last_tick_epoch.load(Ordering::Relaxed);
2587 let now = SystemTime::now()
2588 .duration_since(std_time::UNIX_EPOCH)
2589 .unwrap_or_default()
2590 .as_secs();
2591 let stale_secs = now.saturating_sub(last);
2592 let threshold = (tick_interval_secs * 6).max(60);
2594 if stale_secs > threshold {
2595 let shared_gap =
2602 engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2603 let gap = match (own_gap, shared_gap) {
2604 (Some(a), Some(b)) => Some(a.max(b)),
2605 (Some(a), None) => Some(a),
2606 (None, Some(b)) => Some(b),
2607 (None, None) => None,
2608 };
2609 if let Some(gap) = gap {
2610 let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2611 if gap_secs >= stale_secs {
2612 tracing::info!(
2613 stale_secs,
2614 threshold,
2615 suspend_gap_secs = gap_secs,
2616 "tick loop stale window explained by host suspend/resume, not a stall"
2617 );
2618 } else {
2619 tracing::error!(
2620 stale_secs,
2621 threshold,
2622 suspend_gap_secs = gap_secs,
2623 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2624 stale_secs,
2625 threshold,
2626 gap_secs,
2627 );
2628 }
2629 } else {
2630 tracing::error!(
2631 stale_secs,
2632 threshold,
2633 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2634 stale_secs,
2635 threshold,
2636 );
2637 }
2638 }
2639 }
2640 });
2641 }
2642
2643 loop {
2644 tokio::select! {
2645 _ = interval.tick() => {
2646 let tick_start = Instant::now();
2647
2648 if let Some(gap) = engine_suspend::checkpoint() {
2652 tracing::info!(
2653 gap_secs = gap.num_seconds(),
2654 "host suspend/resume gap detected, discounting from stuck-task age checks"
2655 );
2656 }
2657
2658 if review_in_force != config.max_parallel {
2662 review_in_force =
2663 resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2664 }
2665 if parallel_in_force != config.max_parallel {
2666 let resized =
2667 resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2668 if resized != parallel_in_force {
2669 parallel_in_force = resized;
2670 tracing::info!(
2671 in_force = parallel_in_force,
2672 target = config.max_parallel,
2673 "dispatch pool resized"
2674 );
2675 }
2676 }
2677
2678 {
2680 let mut rw = router.write().await;
2681 rw.tick_weight_recovery();
2682 }
2683
2684 if let Some(remaining) = GhHttp::is_rate_limited() {
2688 let reset_at = chrono::Utc::now()
2689 .timestamp()
2690 .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2691 if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2693 rate_limit_reported = Some(reset_at);
2694 report_rate_limited(reset_at, project_engines.first().map(|e| &e.store), Some(&transport)).await;
2695 }
2696 } else if let Some(prev) = rate_limit_reported.take() {
2697 tracing::info!(
2698 limited_until = prev,
2699 "GitHub API rate limit cleared"
2700 );
2701 if let Some(engine) = project_engines.first() {
2702 let _ = engine.store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2703 }
2704 }
2705
2706 tick::tick_all(
2708 &project_engines,
2709 &tmux,
2710 &capture_for_tick,
2711 &semaphore,
2712 &config,
2713 &router,
2714 &weight_tx,
2715 &dispatching,
2716 Some(&transport),
2717 &active_repos,
2718 &auto_merge_in_flight,
2719 ).await;
2720
2721 while let Ok(signal) = weight_rx.try_recv() {
2724 let mut rw = router.write().await;
2725 match signal {
2726 WeightSignal::RateLimited { ref agent } => {
2727 rw.record_rate_limit(agent);
2728 }
2729 WeightSignal::Success { ref agent } => {
2730 rw.record_success(agent);
2731 }
2732 WeightSignal::Failed { .. }
2733 | WeightSignal::Rerouted
2734 | WeightSignal::Blocked
2735 | WeightSignal::None => {}
2736 }
2737 }
2738
2739 if last_sync.elapsed() >= config.sync_interval {
2744 if sync_in_progress.compare_exchange(
2745 false,
2746 true,
2747 Ordering::AcqRel,
2748 Ordering::Relaxed,
2749 ).is_ok() {
2750 spawn_sync_pass(
2751 &project_engines,
2752 &tmux,
2753 &config,
2754 &router,
2755 &dispatching,
2756 &auto_merge_in_flight,
2757 &sync_in_progress,
2758 "periodic",
2759 );
2760 } else {
2761 tracing::debug!("sync tick still in progress, skipping");
2762 }
2763 last_sync = Instant::now();
2767 }
2768
2769 if webhook_enabled {
2771 let health_check_interval = config.webhook_health_check_interval
2772 .unwrap_or(Duration::from_secs(60));
2773
2774 if last_webhook_health_check.elapsed() >= health_check_interval {
2775 if let Some(port) = webhook_port {
2776 let (health, failure_reason) =
2777 github::check_webhook_health(port).await;
2778 if health != webhook_healthy {
2779 webhook_healthy = health;
2780 if webhook_healthy {
2781 tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2782 } else {
2783 tracing::warn!(
2784 port,
2785 orch_webhook_up = false,
2786 reason = failure_reason.as_deref().unwrap_or("unknown"),
2787 "webhook health check failed"
2788 );
2789 }
2790 }
2791 {
2793 let to_save = {
2794 let mut s = webhook_status.lock().await;
2795 s.healthy = health;
2796 s.last_check_utc = Some(chrono::Utc::now());
2797 if health {
2798 s.last_failure_reason = None;
2799 } else if failure_reason.is_some() {
2800 s.last_failure_reason = failure_reason;
2801 }
2802 s.clone()
2803 };
2804 to_save.save().await;
2805 }
2806 }
2807 last_webhook_health_check = Instant::now();
2808 }
2809 }
2810
2811 if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2813 if let Some(store) = project_engines.first().map(|e| Arc::clone(&e.store)) {
2814 let channels = Arc::clone(&channel_registry);
2815 let current = env!("ORCH_VERSION").to_string();
2816 let check_interval =
2817 Duration::from_secs(config.upgrade_check_interval);
2818 tokio::spawn(async move {
2819 check_and_notify_upgrade(&store, &channels, ¤t, check_interval)
2820 .await;
2821 });
2822 }
2823 last_upgrade_check = Some(Instant::now());
2824 }
2825
2826 let tick_elapsed = tick_start.elapsed();
2828 if tick_elapsed.as_secs() > 30 {
2829 tracing::warn!(
2830 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2831 "slow tick"
2832 );
2833 } else {
2834 tracing::debug!(
2835 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2836 "tick complete"
2837 );
2838 }
2839 last_tick_epoch.store(
2840 SystemTime::now()
2841 .duration_since(std_time::UNIX_EPOCH)
2842 .unwrap_or_default()
2843 .as_secs(),
2844 Ordering::Relaxed,
2845 );
2846 }
2847 () = webhook_notify.notified() => {
2849 if let Some(remaining) = GhHttp::is_rate_limited() {
2850 tracing::warn!(
2851 remaining_secs = remaining.as_secs(),
2852 "GitHub API rate-limited, skipping webhook-triggered tick"
2853 );
2854 } else {
2855 tracing::info!("webhook event triggered immediate tick");
2856
2857 tick::tick_all(
2858 &project_engines,
2859 &tmux,
2860 &capture_for_tick,
2861 &semaphore,
2862 &config,
2863 &router,
2864 &weight_tx,
2865 &dispatching,
2866 Some(&transport),
2867 &active_repos,
2868 &auto_merge_in_flight,
2869 ).await;
2870
2871 while let Ok(signal) = weight_rx.try_recv() {
2873 let mut rw = router.write().await;
2874 match signal {
2875 WeightSignal::RateLimited { ref agent } => {
2876 rw.record_rate_limit(agent);
2877 }
2878 WeightSignal::Success { ref agent } => {
2879 rw.record_success(agent);
2880 }
2881 WeightSignal::Failed { .. }
2882 | WeightSignal::Rerouted
2883 | WeightSignal::Blocked
2884 | WeightSignal::None => {}
2885 }
2886 }
2887
2888 if sync_in_progress.compare_exchange(
2894 false,
2895 true,
2896 Ordering::AcqRel,
2897 Ordering::Relaxed,
2898 ).is_ok() {
2899 spawn_sync_pass(
2900 &project_engines,
2901 &tmux,
2902 &config,
2903 &router,
2904 &dispatching,
2905 &auto_merge_in_flight,
2906 &sync_in_progress,
2907 "webhook",
2908 );
2909 last_sync = Instant::now();
2911 } else {
2912 tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2913 }
2914 }
2915
2916 interval.reset();
2918 }
2919 result = config_rx.recv() => {
2920 let reload = match result {
2923 Ok(path) => {
2924 tracing::info!(path = %path.display(), "config file changed, reloading");
2925 true
2926 }
2927 Err(RecvError::Lagged(n)) => {
2928 tracing::warn!(missed = n, "config change receiver lagged, reloading");
2929 true
2930 }
2931 Err(RecvError::Closed) => {
2932 tracing::warn!("config change channel closed");
2933 false
2934 }
2935 };
2936
2937 if reload {
2938 let new_config = EngineConfig::from_config();
2940 log_engine_config_changes(&config, &new_config);
2941
2942 let new_restart_only = restart_only_snapshot();
2943 log_restart_only_changes(&restart_only, &new_restart_only);
2944 restart_only = new_restart_only;
2945
2946 let tick_changed = new_config.tick_interval != config.tick_interval;
2947 config = new_config;
2948
2949 if tick_changed {
2951 interval = time::interval(config.tick_interval);
2952 tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2953 }
2954
2955 if review_in_force != config.max_parallel {
2958 review_in_force = resize_dispatch_pool(
2959 &review_semaphore,
2960 review_in_force,
2961 config.max_parallel,
2962 );
2963 }
2964 if parallel_in_force != config.max_parallel {
2965 parallel_in_force = resize_dispatch_pool(
2966 &semaphore,
2967 parallel_in_force,
2968 config.max_parallel,
2969 );
2970 tracing::info!(
2971 in_force = parallel_in_force,
2972 target = config.max_parallel,
2973 "dispatch pool resized"
2974 );
2975 }
2976
2977 {
2979 let mut router_guard = router.write().await;
2980 router_guard.reload_async().await;
2981 }
2982
2983 sync_project_engines(
2985 &mut project_engines,
2986 &mut active_repos,
2987 &event_bus,
2988 &tmux,
2989 &capture_for_tick,
2990 &semaphore,
2991 &review_semaphore,
2992 &weight_tx,
2993 &router,
2994 &dispatching,
2995 ).await;
2996
2997 tracing::info!(
2998 tick = ?config.tick_interval,
2999 sync = ?config.sync_interval,
3000 parallel = config.max_parallel,
3001 projects = project_engines.len(),
3002 "config reloaded"
3003 );
3004 }
3005 }
3006 signal_name = async {
3007 tokio::select! {
3008 _ = signal::ctrl_c() => "SIGINT",
3009 _ = sigterm.recv() => "SIGTERM",
3010 _ = sighup.recv() => "SIGHUP",
3011 }
3012 } => {
3013 tracing::info!(signal = signal_name, "beginning graceful shutdown");
3014
3015 let shutdown_result = time::timeout(
3018 config.graceful_shutdown_timeout,
3019 async {
3020 let mut reset_count = 0u32;
3024 let mut review_reset_count = 0u32;
3025 for engine in &project_engines {
3026 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
3027 for task in &tasks {
3028 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
3029 tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
3030 } else {
3031 if let Ok(Some(store_id)) =
3032 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3033 && let Err(e) = engine
3034 .store
3035 .finalize_incomplete_runs(
3036 store_id,
3037 "aborted",
3038 "graceful shutdown: reset in_progress task to routed",
3039 )
3040 .await
3041 {
3042 tracing::warn!(
3043 task_id = task.id.0,
3044 ?e,
3045 "failed to finalize incomplete runs on shutdown"
3046 );
3047 }
3048 reset_count += 1;
3049 }
3050 }
3051 }
3052 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
3054 for task in &tasks {
3055 let task_id = format!("internal:{}", task.id);
3056 if let Err(e) = engine.task_manager.update_task_status(
3057 &ExternalId(task_id.clone()),
3058 Status::Routed,
3059 ).await {
3060 tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3061 } else {
3062 if let Err(e) = engine
3063 .store
3064 .finalize_incomplete_runs(
3065 task.id,
3066 "aborted",
3067 "graceful shutdown: reset in_progress task to routed",
3068 )
3069 .await
3070 {
3071 tracing::warn!(
3072 task_id,
3073 ?e,
3074 "failed to finalize incomplete internal runs on shutdown"
3075 );
3076 }
3077 reset_count += 1;
3078 }
3079 }
3080 }
3081 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3083 for task in &tasks {
3084 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3085 tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3086 } else {
3087 if let Ok(Some(store_id)) =
3088 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3089 && let Err(e) = engine
3090 .store
3091 .finalize_incomplete_runs(
3092 store_id,
3093 "aborted",
3094 "graceful shutdown: reset in_review task to needs_review",
3095 )
3096 .await
3097 {
3098 tracing::warn!(
3099 task_id = task.id.0,
3100 ?e,
3101 "failed to finalize incomplete review runs on shutdown"
3102 );
3103 }
3104 review_reset_count += 1;
3105 }
3106 }
3107 }
3108 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3110 for task in &tasks {
3111 let task_id = format!("internal:{}", task.id);
3112 if let Err(e) = engine.task_manager.update_task_status(
3113 &ExternalId(task_id.clone()),
3114 Status::NeedsReview,
3115 ).await {
3116 tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3117 } else {
3118 if let Err(e) = engine
3119 .store
3120 .finalize_incomplete_runs(
3121 task.id,
3122 "aborted",
3123 "graceful shutdown: reset in_review task to needs_review",
3124 )
3125 .await
3126 {
3127 tracing::warn!(
3128 task_id,
3129 ?e,
3130 "failed to finalize incomplete internal review runs on shutdown"
3131 );
3132 }
3133 review_reset_count += 1;
3134 }
3135 }
3136 }
3137 }
3138 if reset_count > 0 {
3139 tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3140 }
3141 if review_reset_count > 0 {
3142 tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3143 }
3144
3145 let mut killed_sessions = Vec::new();
3148 if let Ok(sessions) = tmux.list_sessions().await {
3149 for session in &sessions {
3150 if session.name.starts_with("orch-") {
3151 match tmux.kill_session(&session.name).await { Err(e) => {
3152 tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3153 } _ => {
3154 killed_sessions.push(session.name.clone());
3155 }}
3156 }
3157 }
3158 if !killed_sessions.is_empty() {
3159 tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3160 }
3161 }
3162
3163 if !killed_sessions.is_empty() {
3166 let still_alive = tmux
3167 .wait_for_sessions_dead(
3168 &killed_sessions,
3169 Duration::from_millis(100),
3170 Duration::from_secs(5),
3171 )
3172 .await;
3173 if still_alive > 0 {
3174 tracing::warn!(
3175 still_alive,
3176 "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3177 );
3178 }
3179 tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3180 }
3181 },
3182 ).await;
3183
3184 if shutdown_result.is_err() {
3185 tracing::error!(
3186 timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3187 "graceful shutdown timed out, forcing exit"
3188 );
3189 }
3190
3191 break;
3192 }
3193 }
3194 }
3195
3196 events::cleanup_port_file().await;
3198 local::cleanup().await;
3199 events::cleanup_version_file();
3200
3201 let _ = transport;
3203 tracing::info!("orch engine stopped");
3204 Ok(())
3205}
3206
3207#[cfg(test)]
3208mod tests {
3209 use std::env;
3210 use std::process;
3211 use std::slice;
3212 use std::sync::Arc;
3213
3214 use super::*;
3215 use crate::backends::ExternalBackend;
3216 use crate::backends::test_helpers::NoopBackend;
3217 use crate::channels::capture::CaptureService;
3218 use crate::channels::transport::Transport;
3219 use crate::engine::runner::git_ops;
3220 use crate::store::UpsertExternal;
3221 use crate::tmux::SessionGuard;
3222 use crate::tmux::TmuxManager;
3223
3224 async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3225 let id = store
3226 .upsert_external(&UpsertExternal {
3227 repo: "owner/repo",
3228 ext_id: number,
3229 title: "Under review",
3230 body: "",
3231 author: "",
3232 url: "",
3233 labels: &[],
3234 origin: "github",
3235 })
3236 .await
3237 .unwrap();
3238 store.update_status(id, TaskStatus::InReview).await.unwrap();
3239 id
3240 }
3241
3242 #[tokio::test]
3243 async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3244 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3245 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3246 let task_manager =
3247 TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3248 let tmux = TmuxManager::isolated();
3249 let orphaned = in_review_task(&store, "11").await;
3250 let live = in_review_task(&store, "12").await;
3251 let session = tmux
3252 .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3253 .await
3254 .unwrap();
3255 let _guard = SessionGuard(session);
3256
3257 let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3258
3259 assert_eq!(reset, 1);
3260 assert_eq!(
3261 store.get(orphaned).await.unwrap().status,
3262 TaskStatus::NeedsReview
3263 );
3264 assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3265 }
3266
3267 #[test]
3268 fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3269 let semaphore = Arc::new(Semaphore::new(4));
3270 let held: Vec<_> = (0..4)
3271 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3272 .collect();
3273 assert!(
3274 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3275 "pool is full"
3276 );
3277
3278 let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3279 assert_eq!(in_force, 8);
3280
3281 let grown: Vec<_> = (0..4)
3282 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3283 .collect();
3284 assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3285 assert!(
3286 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3287 "pool is full"
3288 );
3289 }
3290
3291 #[test]
3292 fn shrinking_max_parallel_never_cancels_a_running_task() {
3293 let semaphore = Arc::new(Semaphore::new(8));
3294 let mut held: Vec<_> = (0..8)
3295 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3296 .collect();
3297
3298 let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3300 assert_eq!(in_force, 8);
3301 assert_eq!(held.len(), 8);
3302
3303 held.truncate(5);
3305 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3306 assert_eq!(in_force, 5);
3307 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3308
3309 held.truncate(4);
3311 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3312 assert_eq!(in_force, 4);
3313 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3314
3315 held.truncate(3);
3317 assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3318 }
3319
3320 #[test]
3321 fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3322 let configured = vec![
3323 ("owner/one".to_string(), PathBuf::from("/srv/one")),
3324 ("owner/two".to_string(), PathBuf::from("/srv/two")),
3325 ];
3326 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3327
3328 let (added, removed) = diff_configured_projects(&configured, &active);
3329 assert_eq!(
3330 added,
3331 vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3332 "only the new project is started, so owner/one keeps its single set of subscribers"
3333 );
3334 assert!(removed.is_empty(), "{removed:?}");
3335 }
3336
3337 #[test]
3338 fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3339 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3340 let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3341 .into_iter()
3342 .collect();
3343
3344 let (added, removed) = diff_configured_projects(&configured, &active);
3345 assert!(added.is_empty(), "nothing new to start");
3346 assert_eq!(removed, vec!["owner/gone".to_string()]);
3347 }
3348
3349 #[test]
3350 fn an_unchanged_project_list_starts_nothing() {
3351 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3352 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3353
3354 assert_eq!(
3355 diff_configured_projects(&configured, &active),
3356 (vec![], vec![])
3357 );
3358 }
3359
3360 #[test]
3361 fn restart_only_keys_are_the_settings_read_once_at_startup() {
3362 assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3365 }
3366
3367 #[test]
3368 fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3369 let before = vec![Some("bot_token: a\n".to_string()), None];
3370
3371 assert!(
3372 log_restart_only_changes(&before, &before.clone()).is_empty(),
3373 "a reload that did not touch these keys must stay quiet"
3374 );
3375
3376 let after = vec![
3378 Some("bot_token: b\n".to_string()),
3379 Some("port: 8080\n".to_string()),
3380 ];
3381 assert_eq!(
3382 log_restart_only_changes(&before, &after),
3383 ["channels", "webhook"]
3384 );
3385
3386 let token_only = vec![Some("bot_token: b\n".to_string()), None];
3388 assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3389 }
3390
3391 #[test]
3392 fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3393 let semaphore = Arc::new(Semaphore::new(4));
3394 assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3395 assert_eq!(semaphore.available_permits(), 4);
3396 }
3397
3398 #[test]
3399 fn live_session_stuck_limit_follows_the_run_timeout() {
3400 let config = EngineConfig {
3401 stuck_timeout: 1800,
3402 run_timeouts: RunTimeouts {
3403 base: 1800,
3404 medium: 2700,
3405 complex: 3600,
3406 },
3407 ..EngineConfig::default()
3408 };
3409 assert_eq!(
3410 config.live_session_stuck_limit(Some("simple")),
3411 1800 + RUN_TIMEOUT_GRACE_SECS
3412 );
3413 assert_eq!(
3414 config.live_session_stuck_limit(Some("medium")),
3415 2700 + RUN_TIMEOUT_GRACE_SECS
3416 );
3417 assert_eq!(
3418 config.live_session_stuck_limit(Some("complex")),
3419 3600 + RUN_TIMEOUT_GRACE_SECS
3420 );
3421
3422 let long_stuck = EngineConfig {
3423 stuck_timeout: 7200,
3424 ..config
3425 };
3426 assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3427 }
3428
3429 #[test]
3430 fn engine_config_defaults() {
3431 let config = EngineConfig::default();
3432 assert_eq!(config.tick_interval, Duration::from_secs(10));
3433 assert_eq!(config.sync_interval, Duration::from_secs(45));
3434 assert_eq!(config.max_parallel, 4);
3435 assert_eq!(config.stuck_timeout, 1800);
3436 assert_eq!(config.no_session_stuck_timeout, 600);
3437 assert!(
3438 config.no_session_stuck_timeout < config.stuck_timeout,
3439 "no_session_stuck_timeout must be shorter than stuck_timeout"
3440 );
3441 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3442 assert_eq!(config.upgrade_check_interval, 3600);
3443 }
3444
3445 #[test]
3446 fn engine_config_from_config_uses_defaults_when_no_config() {
3447 let config = EngineConfig::from_config();
3448 assert_eq!(config.tick_interval, Duration::from_secs(10));
3449 assert_eq!(config.sync_interval, Duration::from_secs(45));
3450 assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3452 assert_eq!(config.stuck_timeout, 1800);
3453 assert_eq!(config.no_session_stuck_timeout, 600);
3454 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3455 assert_eq!(config.upgrade_check_interval, 3600);
3456 }
3457
3458 #[test]
3459 fn engine_config_no_session_timeout_is_under_15_minutes() {
3460 let config = EngineConfig::default();
3461 assert!(
3462 config.no_session_stuck_timeout <= 900,
3463 "no_session_stuck_timeout ({}) exceeds 15 minutes",
3464 config.no_session_stuck_timeout
3465 );
3466 }
3467
3468 #[test]
3469 fn upgrade_check_runs_immediately_when_never_checked() {
3470 assert!(should_run_upgrade_check(None, 3600));
3471 }
3472
3473 #[test]
3474 fn upgrade_check_waits_for_interval_after_first_run() {
3475 assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3476 }
3477
3478 #[test]
3479 fn upgrade_check_can_be_disabled() {
3480 assert!(!should_run_upgrade_check(None, 0));
3481 }
3482
3483 #[test]
3484 fn configured_agents_parses_yaml_block_array() {
3485 let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3487 let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3488 assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3489 }
3490
3491 #[test]
3492 fn configured_agents_parses_json_inline_array() {
3493 let json_str = r#"["claude","codex","olm"]"#;
3494 let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3495 assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3496 }
3497
3498 #[test]
3499 fn configured_agents_fallback_returns_default_agents() {
3500 let defaults = configured_agents();
3502 assert!(defaults.contains(&"claude".to_string()));
3503 assert!(defaults.contains(&"codex".to_string()));
3504 assert!(defaults.contains(&"opencode".to_string()));
3505 }
3506
3507 #[tokio::test]
3516 #[ignore = "capture-pane timing is unreliable on CI runners"]
3517 async fn integration_channel_to_tmux_to_capture() {
3518 let transport = Arc::new(Transport::new());
3520 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3521 let tmux = TmuxManager::new();
3522
3523 let task_id = "testtask1".to_string();
3524 let session_name = format!("orch-test-{task_id}");
3525
3526 let _ = tmux
3528 .command()
3529 .args(["new-session", "-d", "-s", &session_name])
3530 .output()
3531 .await
3532 .expect("failed to start tmux session");
3533
3534 let _guard = SessionGuard(session_name.clone());
3536
3537 capture
3539 .register_session("owner/repo", &task_id, &session_name)
3540 .await;
3541 transport
3542 .bind(
3543 "owner/repo",
3544 &task_id,
3545 &session_name,
3546 "telegram",
3547 "12345",
3548 None,
3549 )
3550 .await;
3551
3552 let capture_clone = Arc::clone(&capture);
3554 let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3555
3556 let mut rx = transport
3558 .subscribe("owner/repo", &task_id)
3559 .await
3560 .expect("no subscription");
3561
3562 let send = tmux
3564 .command()
3565 .args([
3566 "send-keys",
3567 "-t",
3568 &session_name,
3569 "echo from-channel",
3570 "Enter",
3571 ])
3572 .output()
3573 .await
3574 .expect("failed to send keys");
3575 assert!(send.status.success());
3576
3577 let mut got = false;
3579 for _ in 0..20 {
3580 if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3581 && c.content.contains("from-channel")
3582 {
3583 got = true;
3584 break;
3585 }
3586 }
3587
3588 capture.unregister_session("owner/repo", &task_id).await;
3590
3591 let _ = capture_handle.await;
3593
3594 assert!(got, "did not observe tmux output via capture/transport");
3595 }
3596
3597 #[test]
3600 fn read_project_channel_config_returns_defaults_when_no_file() {
3601 let dir = tempfile::tempdir().expect("tempdir");
3602 let cfg = read_project_channel_config(dir.path());
3603 assert!(cfg.telegram_topic_id.is_none());
3604 assert!(cfg.discord_channel_id.is_none());
3605 }
3606
3607 #[test]
3608 fn read_project_channel_config_parses_telegram_and_discord() {
3609 let dir = tempfile::tempdir().expect("tempdir");
3610 let orch_yml = dir.path().join(".orch.yml");
3611 fs::write(
3612 &orch_yml,
3613 "channels:\n telegram:\n topic_id: \"42\"\n discord:\n channel_id: \"9999\"\n",
3614 )
3615 .expect("write .orch.yml");
3616 let cfg = read_project_channel_config(dir.path());
3617 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3618 assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3619 }
3620
3621 #[test]
3622 fn read_project_channel_config_handles_partial_channels() {
3623 let dir = tempfile::tempdir().expect("tempdir");
3624 let orch_yml = dir.path().join(".orch.yml");
3625 fs::write(&orch_yml, "channels:\n telegram:\n topic_id: \"7\"\n")
3626 .expect("write .orch.yml");
3627 let cfg = read_project_channel_config(dir.path());
3628 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3629 assert!(cfg.discord_channel_id.is_none());
3630 }
3631
3632 #[test]
3633 fn read_project_channel_config_handles_malformed_yaml() {
3634 let dir = tempfile::tempdir().expect("tempdir");
3635 let orch_yml = dir.path().join(".orch.yml");
3636 fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3637 let cfg = read_project_channel_config(dir.path());
3638 assert!(cfg.telegram_topic_id.is_none());
3640 assert!(cfg.discord_channel_id.is_none());
3641 }
3642
3643 #[test]
3644 #[cfg(unix)]
3645 fn read_project_channel_config_handles_unreadable_file() {
3646 let dir = tempfile::tempdir().expect("tempdir");
3647 fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3649 let cfg = read_project_channel_config(dir.path());
3650 assert!(cfg.telegram_topic_id.is_none());
3652 assert!(cfg.discord_channel_id.is_none());
3653 }
3654
3655 fn git(dir: &Path, args: &[&str]) {
3656 let mut cmd = process::Command::new("git");
3657 cmd.current_dir(dir);
3658 if args.first() == Some(&"commit") {
3659 cmd.args(["-c", "commit.gpgsign=false"]);
3660 }
3661 let out = cmd
3662 .args(args)
3663 .output()
3664 .unwrap_or_else(|e| panic!("git {args:?} failed to start: {e}"));
3665 assert!(
3666 out.status.success(),
3667 "git {:?} failed:\nstdout: {}\nstderr: {}",
3668 args,
3669 String::from_utf8_lossy(&out.stdout),
3670 String::from_utf8_lossy(&out.stderr)
3671 );
3672 }
3673
3674 fn restore_orch_home(prev: Option<String>) {
3675 match prev {
3676 Some(v) => unsafe { env::set_var("ORCH_HOME", v) },
3678 None => unsafe { env::remove_var("ORCH_HOME") },
3680 }
3681 }
3682
3683 struct StartupConflict {
3684 _tmp: tempfile::TempDir,
3685 _orch_home: tempfile::TempDir,
3686 prev_orch_home: Option<String>,
3687 project_dir: PathBuf,
3688 worktree_dir: PathBuf,
3689 branch: String,
3690 store: Arc<TaskStore>,
3691 store_id: i64,
3692 engine: ProjectEngine,
3693 }
3694
3695 impl Drop for StartupConflict {
3696 fn drop(&mut self) {
3697 restore_orch_home(self.prev_orch_home.take());
3698 }
3699 }
3700
3701 async fn startup_conflict_fixture(
3702 retries: i32,
3703 pr_number: Option<i32>,
3704 start_rebase: bool,
3705 ) -> StartupConflict {
3706 let orch_home = tempfile::tempdir().unwrap();
3707 let prev_orch_home = env::var("ORCH_HOME").ok();
3708 unsafe { env::set_var("ORCH_HOME", orch_home.path()) };
3710
3711 let tmp = tempfile::tempdir().unwrap();
3712 let origin = tmp.path().join("origin.git");
3713 fs::create_dir_all(&origin).unwrap();
3714 git(&origin, &["init", "--bare", "-b", "main"]);
3715 let project_dir = tmp.path().join("repo");
3716 git(
3717 tmp.path(),
3718 &[
3719 "clone",
3720 origin.to_str().unwrap(),
3721 project_dir.to_str().unwrap(),
3722 ],
3723 );
3724 git(&project_dir, &["config", "user.email", "t@t.com"]);
3725 git(&project_dir, &["config", "user.name", "T"]);
3726 fs::write(project_dir.join("f.txt"), "base\n").unwrap();
3727 git(&project_dir, &["add", "."]);
3728 git(&project_dir, &["commit", "-m", "base"]);
3729 git(&project_dir, &["push", "origin", "HEAD:main"]);
3730 git(&project_dir, &["remote", "set-head", "origin", "main"]);
3731
3732 let branch = "gh-issue-4087-conflict".to_string();
3733 git(&project_dir, &["checkout", "-b", &branch]);
3734 fs::write(project_dir.join("f.txt"), "agent\n").unwrap();
3735 git(&project_dir, &["commit", "-am", "agent"]);
3736 git(&project_dir, &["checkout", "main"]);
3737 fs::write(project_dir.join("f.txt"), "main\n").unwrap();
3738 git(&project_dir, &["commit", "-am", "main"]);
3739 git(&project_dir, &["push", "origin", "HEAD:main"]);
3740
3741 let wt_base = orch_home.path().join("worktrees").join("repo");
3742 fs::create_dir_all(&wt_base).unwrap();
3743 let worktree_dir = wt_base.join(&branch);
3744 git(
3745 &project_dir,
3746 &["worktree", "add", worktree_dir.to_str().unwrap(), &branch],
3747 );
3748 git(&worktree_dir, &["config", "user.email", "t@t.com"]);
3749 git(&worktree_dir, &["config", "user.name", "T"]);
3750 git(&worktree_dir, &["fetch", "origin"]);
3751 if start_rebase {
3752 let rebase = process::Command::new("git")
3753 .args(["rebase", "origin/main"])
3754 .current_dir(&worktree_dir)
3755 .output()
3756 .unwrap();
3757 assert!(!rebase.status.success(), "precondition: rebase conflicts");
3758 }
3759
3760 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3761 let store_id = store
3762 .upsert_external(&UpsertExternal {
3763 repo: "owner/repo",
3764 ext_id: "4087",
3765 title: "Conflict",
3766 body: "",
3767 author: "",
3768 url: "",
3769 labels: &[],
3770 origin: "github",
3771 })
3772 .await
3773 .unwrap();
3774 store
3775 .update_status(store_id, TaskStatus::Routed)
3776 .await
3777 .unwrap();
3778 let mut fields = vec![
3779 ("branch", serde_json::json!(branch)),
3780 (
3781 "worktree",
3782 serde_json::json!(worktree_dir.to_string_lossy().as_ref()),
3783 ),
3784 ("merge_conflict_retries", serde_json::json!(retries)),
3785 ];
3786 if let Some(pr) = pr_number {
3787 fields.push(("pr_number", serde_json::json!(pr)));
3788 }
3789 store.set_fields(store_id, &fields).await.unwrap();
3790
3791 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3792 let engine = ProjectEngine {
3793 repo: "owner/repo".to_string(),
3794 project_dir: project_dir.clone(),
3795 backend: Arc::clone(&backend),
3796 task_manager: Arc::new(TaskManager::with_store(
3797 backend,
3798 Arc::clone(&store),
3799 "owner/repo".to_string(),
3800 )),
3801 runner: Arc::new(
3802 runner::TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store)),
3803 ),
3804 store: Arc::clone(&store),
3805 };
3806
3807 StartupConflict {
3808 _tmp: tmp,
3809 _orch_home: orch_home,
3810 prev_orch_home,
3811 project_dir,
3812 worktree_dir,
3813 branch,
3814 store,
3815 store_id,
3816 engine,
3817 }
3818 }
3819
3820 #[tokio::test]
3821 #[serial_test::serial(orch_home)]
3822 async fn startup_keeps_a_content_conflict_when_the_task_retried_a_merge() {
3823 let fx = startup_conflict_fixture(2, Some(4043), false).await;
3824 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3825 .await
3826 .unwrap();
3827
3828 assert!(fx.worktree_dir.exists(), "worktree must stay");
3829 assert!(
3830 git_ops::rebase_in_progress(&fx.worktree_dir).await,
3831 "the content conflict must stay for the agent"
3832 );
3833 let task = fx.store.get(fx.store_id).await.unwrap();
3834 assert_eq!(task.status, TaskStatus::Routed);
3835 assert_eq!(task.branch, fx.branch);
3836 assert!(!task.worktree.is_empty());
3837 }
3838
3839 #[tokio::test]
3840 #[serial_test::serial(orch_home)]
3841 async fn startup_keeps_an_in_progress_rebase_when_the_task_has_a_pr() {
3842 let fx = startup_conflict_fixture(0, Some(12), true).await;
3843 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3844 .await
3845 .unwrap();
3846
3847 assert!(fx.worktree_dir.exists(), "worktree must stay");
3848 assert!(
3849 git_ops::rebase_in_progress(&fx.worktree_dir).await,
3850 "an in-progress rebase for a PR task must stay"
3851 );
3852 let task = fx.store.get(fx.store_id).await.unwrap();
3853 assert_eq!(task.status, TaskStatus::Routed);
3854 assert_eq!(task.branch, fx.branch);
3855 }
3856
3857 #[tokio::test]
3858 #[serial_test::serial(orch_home)]
3859 async fn startup_resets_a_conflict_without_retries_or_a_pr() {
3860 let fx = startup_conflict_fixture(0, None, false).await;
3861 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3862 .await
3863 .unwrap();
3864
3865 assert!(
3866 !fx.worktree_dir.exists(),
3867 "worktree is removed when there is nothing to keep"
3868 );
3869 let task = fx.store.get(fx.store_id).await.unwrap();
3870 assert_eq!(task.status, TaskStatus::New);
3871 }
3872
3873 #[tokio::test]
3874 #[serial_test::serial(orch_home)]
3875 async fn startup_reset_keeps_branch_when_rescue_cannot_push() {
3876 let fx = startup_conflict_fixture(0, None, false).await;
3877 git(
3878 &fx.project_dir,
3879 &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3880 );
3881 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3882 .await
3883 .unwrap();
3884
3885 let task = fx.store.get(fx.store_id).await.unwrap();
3886 assert_eq!(task.status, TaskStatus::New);
3887 assert_eq!(
3888 task.branch, fx.branch,
3889 "the task still names the local branch cleanup kept"
3890 );
3891 assert!(
3892 fx.worktree_dir.exists(),
3893 "rescue failure keeps the worktree on disk"
3894 );
3895 }
3896}