1pub mod auto_merge;
16pub mod channel_handler;
17pub mod ci_bootstrap;
18pub mod cleanup;
19pub mod commands;
20pub mod cooldown;
21pub mod dependencies;
22pub mod dispatch_guard;
23pub mod events;
24pub mod jobs;
25pub mod priority;
26pub mod push_retry;
27pub mod relay;
28pub mod review;
29pub mod review_gate;
30pub mod review_poll;
31pub mod review_shipcheck;
32pub mod router;
33pub mod runner;
34pub mod subscribers;
35pub mod suspend;
36pub mod sync;
37pub mod tasks;
38pub mod tick;
39
40use crate::backends::{ExternalBackend, ExternalId};
45use crate::channels::capture::CaptureService;
46use crate::channels::discord_ws::DiscordGateway;
47use crate::channels::github::start_webhook_server;
48use crate::channels::notification::NotificationLevel;
49use crate::channels::routing::{ChannelRouter, GlobalChannelConfig, ProjectChannelConfig};
50use crate::channels::slack::SlackChannel;
51use crate::channels::telegram::TelegramChannel;
52use crate::channels::tmux::TmuxChannel;
53use crate::channels::transport::Transport;
54use crate::channels::{Channel, ChannelRegistry, IncomingMessage, OutgoingMessage};
55use crate::cmd::CommandErrorContext;
56use crate::config;
57use crate::engine::cleanup::{
58 local_branch_exists, remove_worktree_and_branch, remove_worktree_and_branch_for,
59};
60use crate::engine::router::{Router, RouterConfig};
61use crate::engine::runner::worktree::{
62 abort_worktree_rebase, list_project_worktrees, task_id_from_worktree_name,
63 validate_worktree_gitdir,
64};
65use crate::engine::tasks::TaskManager;
66use crate::github::http::{GhHttp, rate_limit_metrics};
67use crate::repo_context::REPO_CONTEXT;
68use crate::store::{TaskStatus, TaskStore};
69use crate::tmux::TmuxManager;
70use runner::WeightSignal;
71use crate::backends::Status;
73use crate::backends::github::GitHubBackend;
74use crate::channels::github;
75use crate::channels::github::is_transient_bind_error;
76use crate::channels::github::webhook_backoff_delay;
77use crate::channels::notification;
78use crate::channels::notification::TaskNotification;
79use crate::channels::transport;
80use crate::cluster;
81use crate::cluster::local;
82use crate::engine::cooldown as engine_cooldown;
83use crate::engine::review as engine_review;
84use crate::engine::runner::git_ops;
85use crate::engine::runner::worktree;
86use crate::engine::suspend as engine_suspend;
87use crate::engine::suspend::SuspendDetector;
88use crate::engine::tasks as engine_tasks;
89use crate::github::app;
90use crate::home;
91use crate::secret_ref;
92use crate::store;
93use crate::ui;
94use crate::webhook_status::WebhookStatus;
95use futures::future;
96use std::collections::HashMap;
97use std::collections::HashSet;
98use std::fmt;
99use std::fmt::Debug;
100use std::fmt::Formatter;
101use std::fs;
102use std::path::Path;
103use std::path::PathBuf;
104use std::sync::Arc;
105use std::sync::atomic::AtomicBool;
106use std::sync::atomic::AtomicU64;
107use std::sync::atomic::Ordering;
108use std::time as std_time;
109use std::time::Duration;
110use std::time::Instant;
111use std::time::SystemTime;
112use tokio::process::Command;
113use tokio::signal;
114use tokio::signal::unix;
115use tokio::signal::unix::SignalKind;
116use tokio::sync::Mutex;
117use tokio::sync::broadcast::Sender;
118use tokio::sync::broadcast::error::RecvError;
119use tokio::sync::mpsc::Receiver;
120use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
121use tokio::task;
122use tokio::time;
123
124pub fn orch_footer() -> &'static str {
125 "\n\n---\n*Posted by [Orch](https://github.com/gabrielkoerich/orch)*"
126}
127
128pub fn attribution_footer(verb: &str, agent: &str, model: Option<&str>) -> String {
132 match model {
133 Some(m) => format!(
134 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch) using `{m}`*"
135 ),
136 None => format!(
137 "\n\n---\n*{verb} by {agent}[bot] via [Orch](https://github.com/gabrielkoerich/orch)*"
138 ),
139 }
140}
141
142type EngineRef = (
144 String,
145 Arc<dyn ExternalBackend>,
146 Arc<TaskManager>,
147 Option<Arc<TaskStore>>,
148);
149
150#[derive(Clone)]
155pub struct ProjectEngine {
156 pub repo: String,
157 pub project_dir: PathBuf,
158 pub backend: Arc<dyn ExternalBackend>,
159 pub task_manager: Arc<TaskManager>,
160 pub runner: Arc<runner::TaskRunner>,
161 pub store: Arc<TaskStore>,
162}
163
164impl Debug for ProjectEngine {
165 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
166 f.debug_struct("ProjectEngine")
167 .field("repo", &self.repo)
168 .field("project_dir", &self.project_dir)
169 .finish_non_exhaustive()
170 }
171}
172
173pub(crate) async fn reset_orphaned_reviews(
176 task_manager: &TaskManager,
177 store: &Arc<TaskStore>,
178 repo: &str,
179 tmux: &TmuxManager,
180) -> usize {
181 let mut ids: Vec<ExternalId> = store
183 .list_by_status(repo, TaskStatus::InReview)
184 .await
185 .unwrap_or_default()
186 .iter()
187 .map(|t| engine_tasks::store_task_to_external(t).id)
188 .collect();
189 if let Ok(legacy) = task_manager
190 .list_internal_by_status(TaskStatus::InReview)
191 .await
192 {
193 ids.extend(legacy.into_iter().map(|t| t.id));
194 }
195 ids.sort_by(|a, b| a.0.cmp(&b.0));
196 ids.dedup_by(|a, b| a.0 == b.0);
197
198 let mut reset = 0;
199 for id in &ids {
200 let session = tmux.session_name(repo, &format!("{}-review", id.0));
201
202 if tmux.session_exists(&session).await {
203 continue;
204 }
205
206 if let Err(e) = task_manager
207 .update_task_status(id, Status::NeedsReview)
208 .await
209 {
210 tracing::warn!(
211 task_id = id.0,
212 ?e,
213 "failed to reset an orphaned review at start"
214 );
215 continue;
216 }
217
218 if let Ok(Some(store_id)) = store.resolve_task_id(repo, &id.0).await {
219 let reason = "startup: review session gone, reset to needs_review";
220
221 if let Err(e) = store
222 .finalize_incomplete_runs(store_id, "aborted", reason)
223 .await
224 {
225 tracing::warn!(
226 task_id = id.0,
227 ?e,
228 "failed to finalize the orphaned review's runs"
229 );
230 }
231 }
232
233 reset += 1;
234 }
235
236 reset
237}
238
239fn start_relay_client(store: &Arc<TaskStore>, notify: &Arc<Notify>) {
242 let url = config::get_global("relay.url").unwrap_or_default();
244 let store = Arc::clone(store);
245
246 if url.is_empty() {
247 return;
248 }
249
250 let token_ref = config::get_global("relay.token").unwrap_or_default();
251 let notify = Arc::clone(notify);
252 tokio::spawn(async move {
253 let mut warned = false;
254 let token = loop {
255 let reference = token_ref.clone();
256 let read = task::spawn_blocking(move || secret_ref::resolve(&reference))
257 .await
258 .map_err(anyhow::Error::from)
259 .and_then(|r| r);
260
261 match read {
262 Ok(token) if !token.is_empty() => break token,
263 Ok(_) => {
264 tracing::warn!("relay.token is empty, the relay client stays off");
265 return;
266 }
267 Err(e) if !warned => {
268 tracing::warn!(err = %e, "could not read relay.token, retrying every 5 minutes");
269 warned = true;
270 }
271 Err(_) => {}
272 }
273
274 time::sleep(Duration::from_secs(300)).await;
275 };
276
277 relay::run(url, token, store, notify).await;
278 });
279}
280
281#[expect(
287 clippy::too_many_arguments,
288 reason = "the spawned task takes its own handle to each piece of engine state"
289)]
290fn spawn_sync_pass(
291 project_engines: &[ProjectEngine],
292 tmux: &Arc<TmuxManager>,
293 engine_config: &EngineConfig,
294 router: &Arc<RwLock<Router>>,
295 dispatching: &Arc<dashmap::DashMap<String, String>>,
296 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
297 sync_in_progress: &Arc<AtomicBool>,
298 trigger: &'static str,
299) {
300 let engines = project_engines.to_vec();
301 let tmux = Arc::clone(tmux);
302 let engine_config = engine_config.clone();
303 let router = Arc::clone(router);
304 let dispatching = Arc::clone(dispatching);
305 let auto_merge_in_flight = Arc::clone(auto_merge_in_flight);
306 let guard = Arc::clone(sync_in_progress);
307
308 tokio::spawn(async move {
309 let started = Instant::now();
310 let mut passes = Vec::with_capacity(engines.len());
311 for engine in &engines {
312 passes.push(REPO_CONTEXT.scope(
313 engine.repo.clone(),
314 sync_one_project(
315 engine,
316 &tmux,
317 &engine_config,
318 &router,
319 &dispatching,
320 &auto_merge_in_flight,
321 ),
322 ));
323 }
324 future::join_all(passes).await;
325
326 sync::sync_shared_tail(&router, engines.first().map(|e| &e.store)).await;
327 tracing::info!(
328 trigger,
329 elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
330 "sync tick complete"
331 );
332 guard.store(false, Ordering::Release);
333 });
334}
335
336async fn sync_one_project(
337 engine: &ProjectEngine,
338 tmux: &Arc<TmuxManager>,
339 engine_config: &EngineConfig,
340 router: &Arc<RwLock<Router>>,
341 dispatching: &Arc<dashmap::DashMap<String, String>>,
342 auto_merge_in_flight: &Arc<dashmap::DashSet<String>>,
343) {
344 if let Err(e) = Box::pin(sync::sync_tick(
345 &engine.backend,
346 tmux,
347 &engine.repo,
348 engine_config,
349 router,
350 &engine.task_manager,
351 &engine.store,
352 dispatching,
353 auto_merge_in_flight,
354 ))
355 .await
356 {
357 tracing::error!(repo = %engine.repo, ?e, "sync tick failed for project");
358 }
359}
360
361const SLOW_PHASE_GROUP_MS: u128 = 1_000;
363
364pub(crate) fn log_phase_timings(group: &str, repo: &str, phases: &[(&str, u128)]) {
370 let total_ms: u128 = phases.iter().map(|(_, ms)| ms).sum();
371 if total_ms >= SLOW_PHASE_GROUP_MS {
372 tracing::info!(group, repo, total_ms, ?phases, "phase timings");
373 } else {
374 tracing::debug!(group, repo, total_ms, ?phases, "phase timings");
375 }
376}
377
378pub fn configured_agents() -> Vec<String> {
390 if let Ok(agents_str) = config::get("agents")
391 && !agents_str.is_empty()
392 && agents_str != "[]"
393 {
394 if let Ok(agents_arr) = serde_norway::from_str::<Vec<String>>(&agents_str)
397 && !agents_arr.is_empty()
398 {
399 return agents_arr;
400 }
401 if let Ok(agents_arr) = serde_json::from_str::<Vec<String>>(&agents_str)
403 && !agents_arr.is_empty()
404 {
405 return agents_arr;
406 }
407 }
408 router::config::DEFAULT_AGENTS
409 .iter()
410 .map(ToString::to_string)
411 .collect()
412}
413
414pub const RUN_TIMEOUT_GRACE_SECS: u64 = 300;
417
418#[derive(Clone, Copy, Debug, PartialEq, Eq)]
420pub struct RunTimeouts {
421 pub base: u64,
422 pub medium: u64,
423 pub complex: u64,
424}
425
426impl Default for RunTimeouts {
427 fn default() -> Self {
428 Self {
429 base: 1800,
430 medium: 1800,
431 complex: 1800,
432 }
433 }
434}
435
436impl RunTimeouts {
437 pub fn from_config() -> Self {
440 let read =
441 |key: &str| -> Option<u64> { config::get(key).ok().and_then(|s| s.parse().ok()) };
442 let base = read("workflow.timeout_seconds").unwrap_or(1800).max(1800);
443 Self {
444 base,
445 medium: read("workflow.timeout_by_complexity.medium")
446 .unwrap_or(base)
447 .max(base),
448 complex: read("workflow.timeout_by_complexity.complex")
449 .unwrap_or(base)
450 .max(base),
451 }
452 }
453
454 pub fn for_complexity(&self, complexity: Option<&str>) -> u64 {
455 match complexity {
456 Some("complex") => self.complex,
457 Some("medium") => self.medium,
458 _ => self.base,
459 }
460 }
461}
462
463#[derive(Clone, Debug)]
465pub struct EngineConfig {
466 pub tick_interval: Duration,
468 pub sync_interval: Duration,
470 pub webhook_health_check_interval: Option<Duration>,
472 pub max_parallel: usize,
474 pub stuck_timeout: u64,
476 pub no_session_stuck_timeout: u64,
479 pub in_review_no_session_stuck_timeout: u64,
486 pub run_timeouts: RunTimeouts,
488 pub auto_create_followup_on_changes: bool,
490 pub graceful_shutdown_timeout: Duration,
492 pub silence_grace_period: u64,
495 pub silence_cooldown: u64,
497 pub upgrade_check_interval: u64,
500 pub startup_failure_escalation_secs: u64,
505}
506
507impl Default for EngineConfig {
508 fn default() -> Self {
509 Self {
510 tick_interval: Duration::from_secs(10),
511 sync_interval: Duration::from_secs(45),
512 webhook_health_check_interval: Some(Duration::from_secs(60)),
513 max_parallel: 4,
514 stuck_timeout: 1800,
515 no_session_stuck_timeout: 600,
516 in_review_no_session_stuck_timeout: 1920,
517 run_timeouts: RunTimeouts::default(),
518 auto_create_followup_on_changes: true,
519 graceful_shutdown_timeout: Duration::from_secs(600),
520 silence_grace_period: 300,
521 silence_cooldown: 3600,
522 upgrade_check_interval: 3600,
523 startup_failure_escalation_secs: 3600,
524 }
525 }
526}
527
528impl EngineConfig {
529 pub fn live_session_stuck_limit(&self, complexity: Option<&str>) -> u64 {
532 self.stuck_timeout.max(
533 self.run_timeouts
534 .for_complexity(complexity)
535 .saturating_add(RUN_TIMEOUT_GRACE_SECS),
536 )
537 }
538
539 pub fn from_config() -> Self {
541 let mut config = Self::default();
542
543 if let Ok(val) = config::get("engine.tick_interval") {
544 if let Ok(secs) = val.parse::<u64>() {
545 config.tick_interval = Duration::from_secs(secs);
546 } else {
547 tracing::warn!(key = "engine.tick_interval", value = %val, default_secs = config.tick_interval.as_secs(), "invalid value for engine.tick_interval, using default");
548 }
549 }
550
551 if let Ok(val) = config::get("engine.sync_interval") {
552 if let Ok(secs) = val.parse::<u64>() {
553 config.sync_interval = Duration::from_secs(secs);
554 } else {
555 tracing::warn!(key = "engine.sync_interval", value = %val, default_secs = config.sync_interval.as_secs(), "invalid value for engine.sync_interval, using default");
556 }
557 }
558
559 if let Ok(val) = config::get("engine.max_parallel") {
560 if let Ok(n) = val.parse::<usize>() {
561 config.max_parallel = n;
562 } else {
563 tracing::warn!(key = "engine.max_parallel", value = %val, default = config.max_parallel, "invalid value for engine.max_parallel, using default");
564 }
565 }
566
567 if let Ok(val) = config::get("engine.stuck_timeout") {
568 if let Ok(secs) = val.parse::<u64>() {
569 config.stuck_timeout = secs;
570 } else {
571 tracing::warn!(key = "engine.stuck_timeout", value = %val, default_secs = config.stuck_timeout, "invalid value for engine.stuck_timeout, using default");
572 }
573 }
574
575 if let Ok(val) = config::get("engine.no_session_stuck_timeout") {
576 if let Ok(secs) = val.parse::<u64>() {
577 config.no_session_stuck_timeout = secs;
578 } else {
579 tracing::warn!(key = "engine.no_session_stuck_timeout", value = %val, default_secs = config.no_session_stuck_timeout, "invalid value for engine.no_session_stuck_timeout, using default");
580 }
581 }
582
583 let mut in_review_from_config = false;
584 if let Ok(val) = config::get("engine.in_review_no_session_stuck_timeout") {
585 if let Ok(secs) = val.parse::<u64>() {
586 config.in_review_no_session_stuck_timeout = secs;
587 in_review_from_config = true;
588 } else {
589 tracing::warn!(key = "engine.in_review_no_session_stuck_timeout", value = %val, default_secs = config.in_review_no_session_stuck_timeout, "invalid value for engine.in_review_no_session_stuck_timeout, using default");
590 }
591 }
592
593 let review_timeout = engine_review::review_timeout_seconds();
597 let floored = engine_review::no_session_review_floor(
598 config.in_review_no_session_stuck_timeout,
599 review_timeout,
600 );
601 if floored != config.in_review_no_session_stuck_timeout {
602 if in_review_from_config {
603 tracing::warn!(
604 configured_secs = config.in_review_no_session_stuck_timeout,
605 review_timeout_secs = review_timeout,
606 raised_to_secs = floored,
607 "engine.in_review_no_session_stuck_timeout is below the review timeout plus result delivery time, raising it"
608 );
609 } else {
610 tracing::debug!(
611 review_timeout_secs = review_timeout,
612 raised_to_secs = floored,
613 "raised the default in_review no-session limit over a long review timeout"
614 );
615 }
616 config.in_review_no_session_stuck_timeout = floored;
617 }
618
619 config.run_timeouts = RunTimeouts::from_config();
620
621 if let Ok(val) = config::get("engine.webhook_health_check_interval") {
622 if let Ok(secs) = val.parse::<u64>() {
623 if secs > 0 {
624 config.webhook_health_check_interval = Some(Duration::from_secs(secs));
625 } else {
626 config.webhook_health_check_interval = None;
627 }
628 } else {
629 tracing::warn!(key = "engine.webhook_health_check_interval", value = %val, default = ?config.webhook_health_check_interval, "invalid value for engine.webhook_health_check_interval, using default");
630 }
631 }
632
633 if let Ok(val) = config::get("workflow.auto_create_followup_on_changes") {
634 config.auto_create_followup_on_changes = !val.eq_ignore_ascii_case("false");
635 }
636
637 if let Ok(val) = config::get("engine.graceful_shutdown_timeout") {
638 if let Ok(secs) = val.parse::<u64>() {
639 config.graceful_shutdown_timeout = Duration::from_secs(secs);
640 } else {
641 tracing::warn!(key = "engine.graceful_shutdown_timeout", value = %val, default_secs = config.graceful_shutdown_timeout.as_secs(), "invalid value for engine.graceful_shutdown_timeout, using default");
642 }
643 }
644
645 if let Ok(val) = config::get("engine.silence_grace_period") {
646 if let Ok(secs) = val.parse::<u64>() {
647 config.silence_grace_period = secs;
648 } else {
649 tracing::warn!(key = "engine.silence_grace_period", value = %val, default = config.silence_grace_period, "invalid value for engine.silence_grace_period, using default");
650 }
651 }
652
653 if let Ok(val) = config::get("engine.silence_cooldown") {
654 if let Ok(secs) = val.parse::<u64>() {
655 config.silence_cooldown = secs;
656 } else {
657 tracing::warn!(key = "engine.silence_cooldown", value = %val, default = config.silence_cooldown, "invalid value for engine.silence_cooldown, using default");
658 }
659 }
660
661 if let Ok(val) = config::get("engine.upgrade_check_interval") {
662 if let Ok(secs) = val.parse::<u64>() {
663 config.upgrade_check_interval = secs;
664 } else {
665 tracing::warn!(key = "engine.upgrade_check_interval", value = %val, default = config.upgrade_check_interval, "invalid value for engine.upgrade_check_interval, using default");
666 }
667 }
668
669 if let Ok(val) = config::get("engine.startup_failure_escalation_secs") {
670 if let Ok(secs) = val.parse::<u64>() {
671 config.startup_failure_escalation_secs = secs;
672 } else {
673 tracing::warn!(
674 key = "engine.startup_failure_escalation_secs",
675 value = %val,
676 default_secs = config.startup_failure_escalation_secs,
677 "invalid value for engine.startup_failure_escalation_secs, using default"
678 );
679 }
680 }
681
682 config
683 }
684}
685
686pub const RATE_LIMITED_UNTIL_KEY: &str = "github:rate_limited_until";
689
690pub fn format_rate_limited_until(reset_at: i64) -> String {
692 chrono::DateTime::from_timestamp(reset_at, 0).map_or_else(
693 || reset_at.to_string(),
694 |t| {
695 t.with_timezone(&chrono::Local)
696 .format("%H:%M:%S %Z")
697 .to_string()
698 },
699 )
700}
701
702async fn report_rate_limited(
705 reset_at: i64,
706 store: Option<&Arc<TaskStore>>,
707 transport: Option<&Arc<Transport>>,
708) {
709 let m = rate_limit_metrics();
710 let until = format_rate_limited_until(reset_at);
711 tracing::warn!(
712 reset_at,
713 rate_limit_hits = m.hits,
714 skipped_calls = m.skipped_calls,
715 five_xx_hits = m.five_xx_hits,
716 rest_remaining = ?m.rest_remaining,
717 graphql_remaining = ?m.graphql_remaining,
718 "GitHub API rate limited until {until} — skipping GitHub calls, local work continues"
719 );
720
721 if let Some(store) = store
722 && let Err(e) = store
723 .kv_set(RATE_LIMITED_UNTIL_KEY, &reset_at.to_string())
724 .await
725 {
726 tracing::warn!(err = %e, "failed to persist GitHub rate-limit flag");
727 }
728
729 if let Some(transport) = transport {
730 transport.push_notification(TaskNotification {
731 task_id: "github".to_string(),
732 title: "GitHub rate limit".to_string(),
733 status: "blocked".to_string(),
734 agent: "orch".to_string(),
735 duration_seconds: 0.0,
736 summary: format!(
737 "rate limited until {until}. GitHub calls are skipped until then; running agents and local work continue."
738 ),
739 repo: None,
740 notify_target: None,
741 pr_number: None,
742 });
743 }
744}
745
746fn resize_dispatch_pool(semaphore: &Semaphore, in_force: usize, target: usize) -> usize {
752 if target > in_force {
753 semaphore.add_permits(target - in_force);
754 return target;
755 }
756
757 if target < in_force {
758 return in_force - semaphore.forget_permits(in_force - target);
759 }
760
761 in_force
762}
763
764const RESTART_ONLY_KEYS: [&str; 3] = ["channels", "webhook", "relay"];
767
768fn restart_only_snapshot() -> Vec<Option<String>> {
770 RESTART_ONLY_KEYS
771 .iter()
772 .map(|key| config::get(key).ok())
773 .collect()
774}
775
776fn log_restart_only_changes(old: &[Option<String>], new: &[Option<String>]) -> Vec<&'static str> {
779 let changed: Vec<&'static str> = RESTART_ONLY_KEYS
780 .iter()
781 .zip(old)
782 .zip(new)
783 .filter(|((_, old), new)| old != new)
784 .map(|((key, _), _)| *key)
785 .collect();
786
787 for key in &changed {
788 tracing::info!(key = key, "config changed, takes effect after restart");
789 }
790
791 changed
792}
793
794fn log_engine_config_changes(old: &EngineConfig, new: &EngineConfig) {
797 macro_rules! log_changed {
798 ($($key:literal => $field:ident),+ $(,)?) => {
799 $(if old.$field != new.$field {
800 tracing::info!(key = $key, old = ?old.$field, new = ?new.$field, "config changed");
801 })+
802 };
803 }
804
805 log_changed!(
806 "engine.tick_interval" => tick_interval,
807 "engine.sync_interval" => sync_interval,
808 "engine.webhook_health_check_interval" => webhook_health_check_interval,
809 "engine.max_parallel" => max_parallel,
810 "engine.stuck_timeout" => stuck_timeout,
811 "engine.no_session_stuck_timeout" => no_session_stuck_timeout,
812 "engine.in_review_no_session_stuck_timeout" => in_review_no_session_stuck_timeout,
813 "engine.graceful_shutdown_timeout" => graceful_shutdown_timeout,
814 "engine.silence_grace_period" => silence_grace_period,
815 "engine.silence_cooldown" => silence_cooldown,
816 "engine.upgrade_check_interval" => upgrade_check_interval,
817 "engine.startup_failure_escalation_secs" => startup_failure_escalation_secs,
818 "workflow.auto_create_followup_on_changes" => auto_create_followup_on_changes,
819 );
820}
821
822fn should_run_upgrade_check(
823 last_upgrade_check: Option<Instant>,
824 upgrade_check_interval: u64,
825) -> bool {
826 if upgrade_check_interval == 0 {
827 return false;
828 }
829
830 last_upgrade_check
831 .is_none_or(|last| last.elapsed() >= Duration::from_secs(upgrade_check_interval))
832}
833
834async fn connect_project_engine(
841 repo: String,
842 project_dir: PathBuf,
843 store: Arc<TaskStore>,
844 event_tx: Option<Sender<events::TaskEvent>>,
845) -> anyhow::Result<ProjectEngine> {
846 let backend: Arc<dyn ExternalBackend> = Arc::new(GitHubBackend::new(repo.clone())?);
847
848 match backend.health_check().await {
852 Ok(()) => tracing::info!(repo = %repo, backend = backend.name(), "backend connected"),
853 Err(e) => tracing::warn!(
854 repo = %repo,
855 error = %e,
856 "GitHub health check failed for {repo}, starting engine anyway (rate limited, offline or bad credentials?)"
857 ),
858 }
859
860 let task_manager = Arc::new(match event_tx {
861 Some(tx) => {
862 TaskManager::with_events(Arc::clone(&backend), Arc::clone(&store), repo.clone(), tx)
863 }
864 None => TaskManager::with_store(Arc::clone(&backend), Arc::clone(&store), repo.clone()),
865 });
866 let runner = Arc::new(runner::TaskRunner::new(repo.clone()).with_store(Arc::clone(&store)));
867
868 Ok(ProjectEngine {
869 repo,
870 project_dir,
871 backend,
872 task_manager,
873 runner,
874 store,
875 })
876}
877
878fn diff_configured_projects(
885 configured: &[(String, PathBuf)],
886 active_repos: &HashSet<String>,
887) -> (Vec<(String, PathBuf)>, Vec<String>) {
888 let configured_repos: HashSet<&str> = configured.iter().map(|(r, _)| r.as_str()).collect();
889
890 let added = configured
891 .iter()
892 .filter(|(repo, _)| !active_repos.contains(repo))
893 .cloned()
894 .collect();
895 let removed = active_repos
896 .iter()
897 .filter(|repo| !configured_repos.contains(repo.as_str()))
898 .cloned()
899 .collect();
900
901 (added, removed)
902}
903
904#[expect(
910 clippy::too_many_arguments,
911 reason = "a new project engine needs every shared engine handle"
912)]
913async fn sync_project_engines(
914 store: &Arc<TaskStore>,
915 project_engines: &mut Vec<ProjectEngine>,
916 active_repos: &mut HashSet<String>,
917 event_bus: &events::EventBus,
918 tmux: &Arc<TmuxManager>,
919 capture: &Arc<CaptureService>,
920 semaphore: &Arc<Semaphore>,
921 review_semaphore: &Arc<Semaphore>,
922 weight_tx: &mpsc::Sender<WeightSignal>,
923 router: &Arc<RwLock<Router>>,
924 dispatching: &Arc<dashmap::DashMap<String, String>>,
925) {
926 let configured = match config::get_projects_with_paths() {
927 Ok(p) if !p.is_empty() => p,
928 Ok(_) => return,
929 Err(e) => {
930 tracing::warn!(err = %e, "could not re-read the project list, keeping the current one");
931 return;
932 }
933 };
934 let (added, removed) = diff_configured_projects(&configured, active_repos);
935 for repo in removed {
936 tracing::info!(
937 repo = %repo,
938 "project removed from config, takes effect after restart"
939 );
940 }
941
942 for (repo, project_dir) in added {
943 config::watch(&project_dir.join(".orch.yml"));
944 let engine = match connect_project_engine(
945 repo.clone(),
946 project_dir,
947 Arc::clone(store),
948 Some(event_bus.sender()),
949 )
950 .await
951 {
952 Ok(engine) => engine,
953 Err(e) => {
954 tracing::warn!(
957 repo = %repo,
958 err = %e,
959 "project added to config but its backend could not be built, retrying on the next config change"
960 );
961 continue;
962 }
963 };
964
965 subscribers::dispatch::spawn(
966 event_bus.subscribe(),
967 Arc::clone(&engine.backend),
968 Arc::clone(tmux),
969 Arc::clone(&engine.runner),
970 Arc::clone(capture),
971 Arc::clone(semaphore),
972 Arc::clone(&engine.task_manager),
973 weight_tx.clone(),
974 Arc::clone(router),
975 Arc::clone(dispatching),
976 Arc::clone(&engine.store),
977 engine.repo.clone(),
978 );
979 subscribers::review::spawn(
980 event_bus.subscribe(),
981 Arc::clone(&engine.backend),
982 Arc::clone(tmux),
983 Arc::clone(review_semaphore),
984 Arc::clone(&engine.task_manager),
985 Arc::clone(router),
986 Arc::clone(dispatching),
987 Arc::clone(&engine.store),
988 engine.repo.clone(),
989 );
990 subscribers::unblock::spawn(
991 event_bus.subscribe(),
992 Arc::clone(&engine.backend),
993 Arc::clone(&engine.task_manager),
994 Arc::clone(&engine.store),
995 engine.repo.clone(),
996 );
997
998 active_repos.insert(repo.clone());
999 project_engines.push(engine);
1000 tracing::info!(repo = %repo, "project added from config, now syncing");
1001 }
1002}
1003
1004async fn open_shared_store() -> anyhow::Result<Arc<TaskStore>> {
1009 let store = Arc::new(TaskStore::open(&store::default_db_path().await?).await?);
1010 if cluster::enabled() {
1011 cluster::start(&store, &home::orch_home()?.join("raft.db")).await?;
1012 local::serve((*store).clone()).await?;
1013 }
1014 Ok(store)
1015}
1016
1017async fn init_project_engines(store: &Arc<TaskStore>) -> anyhow::Result<Vec<ProjectEngine>> {
1020 let projects = config::get_projects_with_paths()?;
1021 let repos: Vec<&str> = projects.iter().map(|(r, _)| r.as_str()).collect();
1022 tracing::info!(repos = ?repos, "loading projects from config");
1023
1024 if projects.is_empty() {
1025 tracing::info!("no projects yet, add one with `orch ui` or `orch project add`");
1026 }
1027
1028 let mut engines = Vec::new();
1029 for (repo, project_dir) in projects {
1030 tracing::info!(repo = %repo, "initializing project engine");
1031 config::watch(&project_dir.join(".orch.yml"));
1032
1033 engines.push(connect_project_engine(repo, project_dir, Arc::clone(store), None).await?);
1034 }
1035
1036 Ok(engines)
1037}
1038
1039async fn reconcile_startup_worktrees(project_engines: &[ProjectEngine]) -> anyhow::Result<()> {
1040 for engine in project_engines {
1041 let repo_root = worktree::resolve_main_repo(&engine.project_dir).await;
1042 let repo_root_path = PathBuf::from(&repo_root);
1043 let default_branch = worktree::detect_default_branch(&repo_root_path).await;
1044 let worktrees = list_project_worktrees(&repo_root_path).await?;
1045
1046 if let Err(e) = git_ops::git()
1049 .await
1050 .args(["-C", &repo_root.to_string_lossy(), "fetch", "origin"])
1051 .output_with_context()
1052 .await
1053 {
1054 tracing::warn!(repo = %engine.repo, err = %e, "startup git fetch origin failed, rebases may use stale refs");
1055 }
1056
1057 let mut orphans_removed: u32 = 0;
1058 let mut invalid_removed: u32 = 0;
1059 let mut cleaned: u32 = 0;
1060 let mut valid_kept: u32 = 0;
1061
1062 for worktree_dir in worktrees {
1063 let Some(name) = worktree_dir.file_name().and_then(|n| n.to_str()) else {
1064 continue;
1065 };
1066
1067 if !validate_worktree_gitdir(&worktree_dir).await {
1070 tracing::warn!(repo = %engine.repo, worktree = %worktree_dir.display(), "worktree has invalid or missing gitdir, removing");
1071 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1072 .await;
1073 invalid_removed += 1;
1074 continue;
1075 }
1076
1077 let Some(task_id) = task_id_from_worktree_name(name) else {
1078 tracing::info!(repo = %engine.repo, worktree = %worktree_dir.display(), "orphan worktree without task id, removing");
1079 remove_worktree_and_branch(name, &worktree_dir, Some(name), &repo_root_path, false)
1080 .await;
1081 orphans_removed += 1;
1082 continue;
1083 };
1084
1085 let Some(store_id) = engine.store.resolve_task_id(&engine.repo, &task_id).await? else {
1086 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "worktree has no matching task, removing");
1087 remove_worktree_and_branch(
1088 &task_id,
1089 &worktree_dir,
1090 Some(name),
1091 &repo_root_path,
1092 false,
1093 )
1094 .await;
1095 orphans_removed += 1;
1096 continue;
1097 };
1098
1099 let task = engine.store.get(store_id).await?;
1100 let branch_name = if task.branch.is_empty() {
1101 name
1102 } else {
1103 task.branch.as_str()
1104 };
1105
1106 match task.status {
1107 TaskStatus::New
1108 | TaskStatus::Routed
1109 | TaskStatus::InProgress
1110 | TaskStatus::NeedsReview
1111 | TaskStatus::InReview => {
1112 let keep_conflict = worktree::keep_conflicted_rebase(
1113 task.merge_conflict_retries,
1114 task.pr_number,
1115 );
1116 if keep_conflict && git_ops::rebase_in_progress(&worktree_dir).await {
1117 tracing::info!(
1118 repo = %engine.repo,
1119 task_id = %task_id,
1120 worktree = %worktree_dir.display(),
1121 "keeping the conflicted rebase auto_merge left for the agent"
1122 );
1123 valid_kept += 1;
1124 continue;
1125 }
1126 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), "rebasing startup worktree");
1127 let origin_branch = format!("origin/{default_branch}");
1128 let outcome = git_ops::stash_rebase_restore(
1129 &worktree_dir,
1130 &origin_branch,
1131 git_ops::StashRebaseOpts {
1132 abort_on_failure: false,
1133 },
1134 )
1135 .await;
1136 let conflict_left = git_ops::rebase_in_progress(&worktree_dir).await;
1137 let failed_err = match outcome {
1138 Ok(
1139 git_ops::RebaseOutcome::Succeeded | git_ops::RebaseOutcome::Skipped(_),
1140 ) => None,
1141 Ok(git_ops::RebaseOutcome::Failed(e)) if keep_conflict && conflict_left => {
1142 tracing::info!(
1143 repo = %engine.repo,
1144 task_id = %task_id,
1145 err = %e,
1146 "startup rebase hit a content conflict, leaving it for the agent"
1147 );
1148 None
1149 }
1150 Ok(git_ops::RebaseOutcome::Failed(e)) => Some(e),
1151 Err(e) => Some(e.to_string()),
1152 };
1153 if let Some(e) = failed_err {
1154 reset_startup_task_after_failed_rebase(
1155 engine,
1156 &task_id,
1157 &worktree_dir,
1158 branch_name,
1159 &repo_root_path,
1160 task.pr_number.is_some(),
1161 &e,
1162 )
1163 .await;
1164 cleaned += 1;
1165 } else {
1166 valid_kept += 1;
1167 }
1168 }
1169 TaskStatus::Blocked | TaskStatus::Waiting | TaskStatus::Approved => {
1172 valid_kept += 1;
1173 }
1174 TaskStatus::Done => {
1175 tracing::info!(repo = %engine.repo, task_id = %task_id, worktree = %worktree_dir.display(), status = ?task.status, "terminal task worktree, removing");
1176 remove_worktree_and_branch_for(
1177 &task_id,
1178 &worktree_dir,
1179 Some(branch_name),
1180 &repo_root_path,
1181 false,
1182 task.pr_number.is_some(),
1183 )
1184 .await;
1185 cleaned += 1;
1186 }
1187 }
1188 }
1189
1190 tracing::info!(
1191 repo = %engine.repo,
1192 orphans_removed,
1193 invalid_removed,
1194 cleaned,
1195 valid_kept,
1196 "worktree reconciliation: {orphans_removed} orphans removed, {invalid_removed} invalid removed, {cleaned} cleaned, {valid_kept} valid kept"
1197 );
1198
1199 let prune = Command::new("git")
1202 .args(["-C", &repo_root.to_string_lossy(), "worktree", "prune"])
1203 .output_with_context()
1204 .await;
1205 match prune {
1206 Ok(output) if output.status.success() => {
1207 tracing::debug!(repo = %engine.repo, "startup worktree prune completed");
1208 }
1209 Ok(output) => {
1210 let stderr = String::from_utf8_lossy(&output.stderr);
1211 tracing::warn!(repo = %engine.repo, err = %stderr, "startup worktree prune failed");
1212 }
1213 Err(e) => {
1214 tracing::warn!(repo = %engine.repo, err = %e, "startup worktree prune failed");
1215 }
1216 }
1217 }
1218
1219 Ok(())
1220}
1221
1222async fn reset_startup_task_after_failed_rebase(
1223 engine: &ProjectEngine,
1224 task_id: &str,
1225 worktree_dir: &Path,
1226 branch_name: &str,
1227 repo_root: &Path,
1228 keep_remote: bool,
1229 err: &str,
1230) {
1231 tracing::warn!(repo = %engine.repo, task_id, err, "startup rebase failed, resetting task");
1232 abort_worktree_rebase(worktree_dir).await;
1233 remove_worktree_and_branch(
1234 task_id,
1235 worktree_dir,
1236 Some(branch_name),
1237 repo_root,
1238 keep_remote,
1239 )
1240 .await;
1241 let keep_branch = local_branch_exists(repo_root, branch_name).await;
1242 match engine.store.resolve_task_id(&engine.repo, task_id).await {
1243 Ok(Some(reset_id)) => {
1244 let result = if keep_branch {
1245 engine.store.reset_to_new_keep_branch(reset_id).await
1246 } else {
1247 engine.store.reset_to_new(reset_id).await
1248 };
1249 if let Err(e) = result {
1250 tracing::warn!(task_id, err = %e, "failed to reset task to new after rebase failure");
1251 }
1252 }
1253 Ok(None) => {
1254 tracing::warn!(task_id, "task not found in store during reconciliation");
1255 }
1256 Err(e) => {
1257 tracing::warn!(task_id, err = %e, "failed to resolve task ID during reconciliation");
1258 }
1259 }
1260 if let Err(e) = engine
1261 .task_manager
1262 .update_task_status(&ExternalId(task_id.to_string()), Status::New)
1263 .await
1264 {
1265 tracing::warn!(task_id, err = %e, "failed to update backend status during reconciliation");
1266 }
1267}
1268
1269async fn reconcile_startup_estimates(project_engines: &[ProjectEngine]) {
1277 for engine in project_engines {
1278 let tasks = match engine.store.list_external_tasks_with_estimates().await {
1279 Ok(t) => t,
1280 Err(e) => {
1281 tracing::warn!(repo = %engine.repo, err = %e, "startup estimate reconciliation: store query failed");
1282 continue;
1283 }
1284 };
1285
1286 if tasks.is_empty() {
1287 continue;
1288 }
1289
1290 let mut synced: u32 = 0;
1291 let mut failed: u32 = 0;
1292 for (external_id, estimate) in &tasks {
1293 match engine
1294 .backend
1295 .sync_estimate_to_project(&ExternalId(external_id.clone()), *estimate)
1296 .await
1297 {
1298 Ok(()) => synced += 1,
1299 Err(e) => {
1300 tracing::debug!(
1301 repo = %engine.repo,
1302 task_id = %external_id,
1303 estimate,
1304 err = %e,
1305 "startup estimate reconciliation: sync failed"
1306 );
1307 failed += 1;
1308 }
1309 }
1310 }
1311
1312 if synced > 0 || failed > 0 {
1313 tracing::info!(
1314 repo = %engine.repo,
1315 synced,
1316 failed,
1317 "startup estimate reconciliation complete"
1318 );
1319 }
1320 }
1321}
1322
1323fn read_project_channel_config(project_dir: &Path) -> ProjectChannelConfig {
1325 let config_path = project_dir.join(".orch.yml");
1326 if !config_path.exists() {
1327 return ProjectChannelConfig::default();
1328 }
1329 let content = match fs::read_to_string(&config_path) {
1330 Ok(c) => c,
1331 Err(e) => {
1332 tracing::warn!(
1333 path = %config_path.display(),
1334 error = %e,
1335 "failed to read .orch.yml; channel config will use defaults"
1336 );
1337 return ProjectChannelConfig::default();
1338 }
1339 };
1340 let val: serde_norway::Value = match serde_norway::from_str(&content) {
1341 Ok(v) => v,
1342 Err(e) => {
1343 tracing::warn!(
1344 path = %config_path.display(),
1345 error = %e,
1346 "failed to parse .orch.yml; channel config will use defaults"
1347 );
1348 return ProjectChannelConfig::default();
1349 }
1350 };
1351 let Some(channels) = val.get("channels") else {
1352 return ProjectChannelConfig::default();
1353 };
1354 let get_str = |section: &str, key: &str| -> Option<String> {
1355 channels
1356 .get(section)
1357 .and_then(|s| s.get(key))
1358 .and_then(|v| v.as_str())
1359 .map(String::from)
1360 };
1361 ProjectChannelConfig {
1362 telegram_topic_id: get_str("telegram", "topic_id"),
1363 discord_channel_id: get_str("discord", "channel_id"),
1364 }
1365}
1366
1367async fn fetch_latest_release_version() -> Option<String> {
1372 let output = Command::new("gh")
1373 .args([
1374 "api",
1375 "repos/gabrielkoerich/orch/releases/latest",
1376 "--jq",
1377 ".tag_name",
1378 ])
1379 .output()
1380 .await
1381 .ok()?;
1382 if !output.status.success() {
1383 return None;
1384 }
1385 let tag = String::from_utf8(output.stdout).ok()?;
1386 let version = tag.trim().trim_start_matches('v').to_string();
1387 if version.is_empty() {
1388 None
1389 } else {
1390 Some(version)
1391 }
1392}
1393
1394async fn check_and_notify_upgrade(
1403 store: &Arc<TaskStore>,
1404 channels: &Arc<ChannelRegistry>,
1405 current_version: &str,
1406 check_interval: Duration,
1407) {
1408 let last_check_key = "upgrade:last_check_at";
1409 let last_notified_key = "upgrade:last_notified_version";
1410
1411 let now = chrono::Utc::now().timestamp();
1413 if let Ok(Some(last_check_str)) = store.kv_get(last_check_key).await
1414 && let Ok(last_check) = last_check_str.parse::<i64>()
1415 && now - last_check < i64::try_from(check_interval.as_secs()).unwrap_or(i64::MAX)
1416 {
1417 return;
1418 }
1419
1420 let _ = store.kv_set(last_check_key, &now.to_string()).await;
1422
1423 let Some(latest) = fetch_latest_release_version().await else {
1424 tracing::debug!("upgrade check: failed to fetch latest release version");
1425 return;
1426 };
1427
1428 if latest == current_version {
1429 tracing::debug!(current = %current_version, "orch is up to date");
1430 return;
1431 }
1432
1433 tracing::warn!(
1434 current_version = %current_version,
1435 latest_version = %latest,
1436 "orch upgrade available"
1437 );
1438
1439 if let Ok(Some(last_notified)) = store.kv_get(last_notified_key).await
1444 && last_notified == latest
1445 {
1446 tracing::debug!(latest = %latest, "upgrade already notified");
1447 return;
1448 }
1449
1450 let msg_telegram = format!(
1451 "⚠️ <b>Orch Upgrade Available</b>\n\n\
1452 Service: <code>{current_version}</code>\n\
1453 Latest: <code>{latest}</code>\n\n\
1454 Run: <code>brew update && brew upgrade orch && brew services restart orch</code>"
1455 );
1456
1457 let msg_discord = format!(
1458 "⚠️ **Orch Upgrade Available**\n\n\
1459 Service: `{current_version}`\n\
1460 Latest: `{latest}`\n\n\
1461 Run: `brew update && brew upgrade orch && brew services restart orch`"
1462 );
1463
1464 let msg_slack = format!(
1465 "⚠️ *Orch Upgrade Available*\n\n\
1466 Service: `{current_version}`\n\
1467 Latest: `{latest}`\n\n\
1468 Run: `brew update && brew upgrade orch && brew services restart orch`"
1469 );
1470
1471 for channel in channels.iter() {
1472 let (body, metadata, topic_id) = match channel.name() {
1473 "telegram" => (
1474 msg_telegram.clone(),
1475 serde_json::json!({"preformatted_html": true}),
1476 None,
1477 ),
1478 "discord" => (msg_discord.clone(), serde_json::json!({}), None),
1479 "slack" => (msg_slack.clone(), serde_json::json!({}), None),
1480 _ => continue,
1481 };
1482
1483 let msg = OutgoingMessage {
1484 thread_id: "upgrade".to_string(),
1485 body,
1486 reply_to: None,
1487 metadata,
1488 topic_id,
1489 };
1490
1491 if let Err(e) = channel.send(&msg).await {
1492 tracing::warn!(
1493 channel = channel.name(),
1494 ?e,
1495 "failed to send upgrade notification"
1496 );
1497 }
1498 }
1499
1500 let _ = store.kv_set(last_notified_key, &latest).await;
1502}
1503
1504async fn send_startup_escalation_notification(elapsed_secs: u64) {
1512 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1513 let duration = notification::format_duration(elapsed_secs as f64);
1514
1515 let msg_telegram = format!(
1516 "🚨 <b>Orch startup blocked — GitHub unreachable</b>\n\n\
1517 The engine has been retrying backend connections for {duration}\n\
1518 without success. All task processing, routing, and scheduled\n\
1519 jobs are paused. Check service logs with `orch log -f`."
1520 );
1521
1522 let msg_discord = format!(
1523 "🚨 **Orch startup blocked — GitHub unreachable**\n\n\
1524 The engine has been retrying backend connections for {duration}\n\
1525 without success. All task processing, routing, and scheduled\n\
1526 jobs are paused. Check service logs with `orch log -f`."
1527 );
1528
1529 let msg_slack = format!(
1530 "🚨 *Orch startup blocked — GitHub unreachable*\n\n\
1531 The engine has been retrying backend connections for {duration}\n\
1532 without success. All task processing, routing, and scheduled\n\
1533 jobs are paused. Check service logs with `orch log -f`."
1534 );
1535
1536 let tg_token = secret_ref::config_secret("channels.telegram.bot_token").await;
1538 let tg_chat_id = config::get("channels.telegram.chat_id");
1539 if let (Ok(token), Ok(chat_id)) = (&tg_token, &tg_chat_id)
1540 && !token.is_empty()
1541 && !chat_id.is_empty()
1542 && let Ok(channel) = TelegramChannel::new(token.clone(), Some(chat_id.clone()))
1543 {
1544 let msg = OutgoingMessage {
1545 thread_id: "startup_escalation".to_string(),
1546 body: msg_telegram.clone(),
1547 reply_to: None,
1548 metadata: serde_json::json!({ "preformatted_html": true }),
1549 topic_id: config::get("channels.telegram.general_topic_id").ok(),
1550 };
1551 match channel.send(&msg).await {
1552 Err(e) => {
1553 tracing::warn!(?e, "failed to send startup escalation via telegram");
1554 }
1555 _ => {
1556 tracing::info!("startup escalation notification sent to telegram");
1557 }
1558 }
1559 }
1560
1561 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await {
1563 let channel_id = config::get("channels.discord.channel_id")
1564 .or_else(|_| config::get("channels.discord.general_channel_id"));
1565 if !token.is_empty()
1566 && let Some(cid) = channel_id.ok().filter(|s| !s.is_empty())
1567 {
1568 let shard_id = config::get("channels.discord.shard_id")
1569 .ok()
1570 .and_then(|v| v.parse::<u64>().ok())
1571 .unwrap_or(0);
1572 let shard_count = config::get("channels.discord.shard_count")
1573 .ok()
1574 .and_then(|v| v.parse::<u64>().ok())
1575 .unwrap_or(1);
1576 if let Ok(gateway) = DiscordGateway::new(token, Some(cid), shard_id, shard_count) {
1577 let msg = OutgoingMessage {
1578 thread_id: "startup_escalation".to_string(),
1579 body: msg_discord.clone(),
1580 reply_to: None,
1581 metadata: serde_json::Value::Null,
1582 topic_id: None,
1583 };
1584 match gateway.send(&msg).await {
1585 Err(e) => {
1586 tracing::warn!(?e, "failed to send startup escalation via discord");
1587 }
1588 _ => {
1589 tracing::info!("startup escalation notification sent to discord");
1590 }
1591 }
1592 }
1593 }
1594 }
1595
1596 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await {
1598 let channel_id = config::get("channels.slack.channel_id");
1599 if !token.is_empty()
1600 && let Ok(cid) = channel_id
1601 && let Ok(channel) = SlackChannel::new(token, Some(cid))
1602 {
1603 let msg = OutgoingMessage {
1604 thread_id: "startup_escalation".to_string(),
1605 body: msg_slack.clone(),
1606 reply_to: None,
1607 metadata: serde_json::Value::Null,
1608 topic_id: None,
1609 };
1610 match channel.send(&msg).await {
1611 Err(e) => {
1612 tracing::warn!(?e, "failed to send startup escalation via slack");
1613 }
1614 _ => {
1615 tracing::info!("startup escalation notification sent to slack");
1616 }
1617 }
1618 }
1619 }
1620}
1621
1622pub async fn serve() -> anyhow::Result<()> {
1626 let killed = home::kill_other_orch_serve_processes(Duration::from_secs(30));
1627 if killed > 0 {
1628 tracing::info!(count = killed, "terminated stale orch serve processes");
1629 }
1630
1631 tracing::info!("orch engine starting");
1632
1633 if let Err(e) = home::init_dirs().await {
1637 tracing::warn!("failed to initialize orch directories: {e}");
1638 }
1639
1640 if let Err(e) = config::warm_cache().await {
1643 tracing::warn!("failed to warm config cache: {e}");
1644 }
1645
1646 let mut config = EngineConfig::from_config();
1647
1648 let store = {
1651 let mut delay_secs = 5u64;
1652 let mut escalated = false;
1653 let start = Instant::now();
1654 loop {
1655 match open_shared_store().await {
1656 Ok(store) => break store,
1657 Err(e) => {
1658 let elapsed = start.elapsed().as_secs();
1659 let escalation = config.startup_failure_escalation_secs;
1660 if escalation > 0 && !escalated && elapsed >= escalation {
1662 escalated = true;
1663 tracing::error!(
1664 elapsed_secs = elapsed,
1665 "the store or the cluster has not started, escalating: {e:#}"
1666 );
1667 send_startup_escalation_notification(elapsed).await;
1668 } else {
1669 tracing::warn!(
1670 delay_secs,
1671 "could not open the store or start the cluster, retrying: {e:#}"
1672 );
1673 }
1674
1675 time::sleep(Duration::from_secs(delay_secs)).await;
1676 delay_secs = (delay_secs * 2).min(120);
1677 }
1678 }
1679 }
1680 };
1681
1682 engine_cooldown::init_cooldown_store(Arc::clone(&store)).await;
1685 tracing::info!("internal database ready");
1686
1687 let event_bus = events::EventBus::new(256);
1689 if let Err(e) = event_bus.start_ws_server().await {
1690 tracing::warn!(
1691 ?e,
1692 "failed to start event websocket server, continuing without it"
1693 );
1694 }
1695
1696 if let Err(e) = ui::serve(Arc::clone(&store), event_bus.sender()).await {
1698 tracing::warn!(?e, "failed to start the web UI, continuing without it");
1699 }
1700
1701 let startup_escalation_secs = config.startup_failure_escalation_secs;
1710 let mut project_engines = {
1711 let mut delay_secs = 5u64;
1712 let mut attempt = 0u32;
1713 let mut escalated = false;
1714 let start = Instant::now();
1715 loop {
1716 attempt += 1;
1717 match init_project_engines(&store).await {
1718 Ok(engines) => break engines,
1719 Err(e) => {
1720 let elapsed = start.elapsed().as_secs();
1721
1722 if startup_escalation_secs > 0
1728 && !escalated
1729 && elapsed >= startup_escalation_secs
1730 {
1731 escalated = true;
1732 #[expect(clippy::cast_precision_loss, reason = "a display duration")]
1733 let outage = notification::format_duration(elapsed as f64);
1734 tracing::error!(
1735 delay_secs,
1736 attempt,
1737 elapsed_secs = elapsed,
1738 threshold_secs = startup_escalation_secs,
1739 "project backends unavailable for {outage} — escalating startup failure: {e}",
1740 );
1741 send_startup_escalation_notification(elapsed).await;
1745 } else {
1746 tracing::warn!(
1747 delay_secs,
1748 attempt,
1749 "project backends unavailable, retrying: {e}"
1750 );
1751 }
1752
1753 time::sleep(Duration::from_secs(delay_secs)).await;
1754 delay_secs = (delay_secs * 2).min(120);
1755 }
1756 }
1757 }
1758 };
1759
1760 tracing::info!(
1761 project_count = project_engines.len(),
1762 "initialized project engines"
1763 );
1764
1765 let mut active_repos: HashSet<String> =
1769 project_engines.iter().map(|e| e.repo.clone()).collect();
1770
1771 {
1775 let current = env!("ORCH_VERSION").to_string();
1776 tokio::spawn(async move {
1777 if let Some(latest) = fetch_latest_release_version().await
1778 && latest != current
1779 {
1780 tracing::warn!(
1781 current_version = %current,
1782 latest_version = %latest,
1783 "orch service is behind the latest release — run: brew update && brew upgrade orch && brew services restart orch"
1784 );
1785 }
1786 });
1787 }
1788
1789 for engine in &mut project_engines {
1791 engine.task_manager = Arc::new(TaskManager::with_events(
1792 Arc::clone(&engine.backend),
1793 Arc::clone(&engine.store),
1794 engine.repo.clone(),
1795 event_bus.sender(),
1796 ));
1797 }
1798
1799 let leads = project_engines.first().is_none_or(|e| e.store.is_leader());
1801 if leads && let Err(e) = reconcile_startup_worktrees(&project_engines).await {
1802 tracing::warn!(err = %e, "startup worktree reconciliation failed");
1803 }
1804
1805 if leads {
1806 let engines_snapshot: Vec<ProjectEngine> = project_engines.clone();
1810 tokio::spawn(async move {
1811 reconcile_startup_estimates(&engines_snapshot).await;
1812 });
1813 }
1814
1815 let global_channel_config = GlobalChannelConfig {
1817 telegram_general_topic_id: config::get("channels.telegram.general_topic_id").ok(),
1818 discord_general_channel_id: config::get("channels.discord.general_channel_id").ok(),
1819 control_telegram_topic_id: config::get("control.channels.telegram.topic_id").ok(),
1820 control_discord_channel_id: config::get("control.channels.discord.channel_id").ok(),
1821 };
1822 let project_channel_configs: Vec<(String, ProjectChannelConfig)> = project_engines
1823 .iter()
1824 .map(|e| {
1825 let cfg = read_project_channel_config(&e.project_dir);
1826 (e.repo.clone(), cfg)
1827 })
1828 .collect();
1829 let channel_router = Arc::new(ChannelRouter::new(
1830 &global_channel_config,
1831 &project_channel_configs,
1832 ));
1833 tracing::info!(
1834 projects = ?channel_router.projects(),
1835 "channel router initialized"
1836 );
1837
1838 let tmux = Arc::new(TmuxManager::new());
1840 tick::startup_cleanup(&tmux).await;
1843
1844 let transport = Arc::new(Transport::new());
1846
1847 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
1849 let capture_for_tick = Arc::clone(&capture);
1850 tokio::spawn(async move {
1851 capture.start().await;
1852 });
1853
1854 let mut channel_registry = ChannelRegistry::new();
1856
1857 if let Ok(token) = secret_ref::config_secret("channels.telegram.bot_token").await
1859 && !token.is_empty()
1860 {
1861 let chat_id = config::get("channels.telegram.chat_id").ok();
1862 match TelegramChannel::new(token, chat_id) {
1863 Ok(telegram) => {
1864 if let Err(e) = telegram.health_check().await {
1865 tracing::warn!(?e, "telegram channel health check failed, skipping");
1866 } else {
1867 channel_registry.register(Box::new(telegram));
1868 tracing::info!("telegram channel registered");
1869 }
1870 }
1871 Err(e) => {
1872 tracing::warn!(?e, "failed to create telegram channel, skipping");
1873 }
1874 }
1875 }
1876
1877 if let Ok(token) = secret_ref::config_secret("channels.discord.bot_token").await
1879 && !token.is_empty()
1880 {
1881 let channel_id = config::get("channels.discord.channel_id").ok();
1882 let shard_id = config::get("channels.discord.shard_id")
1883 .ok()
1884 .and_then(|v| v.parse::<u64>().ok())
1885 .unwrap_or(0);
1886 let shard_count = config::get("channels.discord.shard_count")
1887 .ok()
1888 .and_then(|v| v.parse::<u64>().ok())
1889 .unwrap_or(1);
1890 match DiscordGateway::new(token, channel_id, shard_id, shard_count) {
1891 Ok(discord) => {
1892 if let Err(e) = discord.health_check().await {
1893 tracing::warn!(?e, "discord gateway health check failed, skipping");
1894 } else {
1895 channel_registry.register(Box::new(discord));
1896 tracing::info!(shard_id, shard_count, "discord gateway registered");
1897 }
1898 }
1899 Err(e) => {
1900 tracing::warn!(?e, "failed to create discord gateway, skipping");
1901 }
1902 }
1903 }
1904
1905 if let Ok(token) = secret_ref::config_secret("channels.slack.bot_token").await
1907 && !token.is_empty()
1908 {
1909 let channel_id = config::get("channels.slack.channel_id").ok();
1910 match SlackChannel::new(token, channel_id) {
1911 Ok(slack) => {
1912 if let Err(e) = slack.health_check().await {
1913 tracing::warn!(?e, "slack channel health check failed, skipping");
1914 } else {
1915 channel_registry.register(Box::new(slack));
1916 tracing::info!("slack channel registered");
1917 }
1918 }
1919 Err(e) => {
1920 tracing::warn!(?e, "failed to create slack channel, skipping");
1921 }
1922 }
1923 }
1924
1925 let tmux_channel = TmuxChannel::with_transport(Arc::clone(&transport));
1927 channel_registry.register(Box::new(tmux_channel));
1928 tracing::info!("tmux channel registered");
1929
1930 let mut channel_receivers: Vec<Receiver<IncomingMessage>> = Vec::new();
1932 for channel in channel_registry.iter() {
1933 match channel.start().await {
1934 Ok(rx) => {
1935 tracing::info!(channel = channel.name(), "channel started");
1936 channel_receivers.push(rx);
1937 }
1938 Err(e) => {
1939 tracing::warn!(channel = channel.name(), ?e, "failed to start channel");
1940 }
1941 }
1942 }
1943
1944 let channel_registry = Arc::new(channel_registry);
1946
1947 let transport_for_messages = Arc::clone(&transport);
1949 let tmux_for_messages = Arc::clone(&tmux);
1950 let capture_for_messages = Arc::clone(&capture_for_tick);
1951 let channels_for_messages = Arc::clone(&channel_registry);
1952 let engine_refs: Vec<EngineRef> = project_engines
1954 .iter()
1955 .map(|e| {
1956 (
1957 e.repo.clone(),
1958 Arc::clone(&e.backend),
1959 Arc::clone(&e.task_manager),
1960 Some(Arc::clone(&e.store)),
1961 )
1962 })
1963 .collect();
1964 let pending_picks: channel_handler::PendingPicks = Arc::new(Mutex::new(HashMap::new()));
1966 let router_for_messages = Arc::clone(&channel_router);
1967 for mut rx in channel_receivers {
1968 let transport = Arc::clone(&transport_for_messages);
1969 let tmux = Arc::clone(&tmux_for_messages);
1970 let capture = Arc::clone(&capture_for_messages);
1971 let channels = Arc::clone(&channels_for_messages);
1972 let engine_refs = engine_refs.clone();
1973 let ch_router = Arc::clone(&router_for_messages);
1974 let picks = Arc::clone(&pending_picks);
1975 tokio::spawn(async move {
1976 while let Some(msg) = rx.recv().await {
1977 tracing::debug!(channel = %msg.channel, thread = %msg.thread_id, "received message from channel");
1978 channel_handler::handle_channel_message(
1979 msg,
1980 &transport,
1981 &tmux,
1982 &capture,
1983 &channels,
1984 &engine_refs,
1985 &ch_router,
1986 &picks,
1987 )
1988 .await;
1989 }
1990 });
1991 }
1992
1993 {
1997 let restart_channels = Arc::clone(&channel_registry);
1998 let restart_store = Some(Arc::clone(&store));
1999 if let Some(store) = restart_store {
2000 tokio::spawn(async move {
2001 time::sleep(Duration::from_secs(5)).await;
2002 match store.kv_get("control:restart_pending").await {
2003 Ok(Some(conv_key)) => {
2004 if let Some((ch_name, thread_id, topic_id)) =
2005 transport::parse_conversation_key(&conv_key)
2006 {
2007 for ch in restart_channels.iter() {
2008 if ch.name() == ch_name {
2009 let msg = OutgoingMessage {
2010 thread_id: thread_id.to_string(),
2011 body: "orch restarted successfully.".to_string(),
2012 reply_to: None,
2013 metadata: serde_json::json!({}),
2014 topic_id: topic_id.map(ToString::to_string),
2015 };
2016 if let Err(e) = ch.send(&msg).await {
2017 tracing::warn!(
2018 channel = ch_name,
2019 error = %e,
2020 "failed to send restart confirmation"
2021 );
2022 }
2023 break;
2024 }
2025 }
2026 }
2027 if let Err(e) = store.kv_delete("control:restart_pending").await {
2028 tracing::warn!(error = %e, "failed to clear control:restart_pending");
2029 }
2030 }
2031 Ok(None) => {} Err(e) => {
2033 tracing::warn!(error = %e, "failed to read control:restart_pending");
2034 }
2035 }
2036 });
2037 }
2038 }
2039
2040 {
2051 let mut notification_rx = transport.subscribe_notifications();
2052 let channels = Arc::clone(&channel_registry);
2053 let notif_router = Arc::clone(&channel_router);
2054 let notif_store: Option<Arc<TaskStore>> = Some(Arc::clone(&store));
2056 tokio::spawn(async move {
2057 loop {
2058 match notification_rx.recv().await {
2059 Ok(notification) => {
2060 let level = NotificationLevel::from_config();
2061 if !level.should_notify(¬ification.status) {
2062 tracing::debug!(
2063 task_id = %notification.task_id,
2064 status = %notification.status,
2065 "notification suppressed by level={:?}",
2066 level
2067 );
2068 continue;
2069 }
2070
2071 tracing::info!(
2072 task_id = %notification.task_id,
2073 status = %notification.status,
2074 repo = ?notification.repo,
2075 "routing notification to channels"
2076 );
2077
2078 let mut routed = false;
2080
2081 if let Some(ref target) = notification.notify_target {
2084 let telegram = channels.iter().find(|c| c.name() == "telegram");
2085 if let Some(channel) = telegram {
2086 let body = notification.format_telegram();
2087 let metadata = serde_json::json!({
2088 "preformatted_html": true,
2089 "chat_id_override": target
2090 });
2091 let msg = OutgoingMessage {
2092 thread_id: notification.task_id.clone(),
2093 body,
2094 reply_to: None,
2095 metadata,
2096 topic_id: None,
2097 };
2098 match channel.send(&msg).await {
2099 Err(e) => {
2100 tracing::warn!(
2101 task_id = %notification.task_id,
2102 target,
2103 ?e,
2104 "failed to send to notify_target"
2105 );
2106 }
2107 _ => {
2108 routed = true;
2109 }
2110 }
2111 }
2112 }
2113
2114 if routed {
2115 } else if let Some(repo) = notification.repo.as_deref() {
2117 let mut sent: HashSet<(String, Option<String>)> = HashSet::new();
2123
2124 for channel in channels.iter() {
2126 let ch_name = channel.name();
2127 if let Some(topic_id) =
2128 notif_router.target_for_project(repo, ch_name)
2129 {
2130 let body = match ch_name {
2131 "telegram" => notification.format_telegram(),
2132 "discord" => notification.format_discord(),
2133 "slack" => notification.format_slack(),
2134 _ => continue,
2135 };
2136 let metadata = if ch_name == "telegram" {
2137 serde_json::json!({"preformatted_html": true})
2138 } else {
2139 serde_json::json!({})
2140 };
2141 let resolved_topic = Some(topic_id.to_string());
2142 let msg = OutgoingMessage {
2143 thread_id: notification.task_id.clone(),
2144 body,
2145 reply_to: None,
2146 metadata,
2147 topic_id: resolved_topic.clone(),
2148 };
2149 if let Err(e) = channel.send(&msg).await {
2150 tracing::warn!(
2151 channel = ch_name,
2152 task_id = %notification.task_id,
2153 ?e,
2154 "failed to send to dedicated channel"
2155 );
2156 } else {
2157 sent.insert((ch_name.to_string(), resolved_topic));
2158 routed = true;
2159 }
2160 }
2161 }
2162
2163 if let Some(store) = ¬if_store {
2165 match store.list_subscribers_for_repo(repo).await {
2166 Ok(subscribers) => {
2167 for (ch_name, thread_id, topic_id) in subscribers {
2168 let channel =
2169 channels.iter().find(|c| c.name() == ch_name);
2170 let Some(channel) = channel else {
2171 continue;
2172 };
2173 let resolved_topic = if topic_id.is_empty() {
2174 None
2175 } else {
2176 Some(topic_id.clone())
2177 };
2178 if sent.contains(&(
2181 ch_name.clone(),
2182 resolved_topic.clone(),
2183 )) {
2184 tracing::debug!(
2185 channel = %ch_name,
2186 task_id = %notification.task_id,
2187 topic_id = ?resolved_topic,
2188 "skipping duplicate subscriber dispatch — destination already received from dedicated path"
2189 );
2190 continue;
2191 }
2192 let body = notification.format_with_project(&ch_name);
2193 let metadata = if ch_name == "telegram" {
2194 serde_json::json!({"preformatted_html": true})
2195 } else {
2196 serde_json::json!({})
2197 };
2198 let msg = OutgoingMessage {
2199 thread_id: thread_id.clone(),
2200 body,
2201 reply_to: None,
2202 metadata,
2203 topic_id: resolved_topic.clone(),
2204 };
2205 if let Err(e) = channel.send(&msg).await {
2206 tracing::warn!(
2207 channel = %ch_name,
2208 task_id = %notification.task_id,
2209 ?e,
2210 "failed to send to subscribed channel"
2211 );
2212 } else {
2213 sent.insert((ch_name.clone(), resolved_topic));
2214 routed = true;
2215 }
2216 }
2217 }
2218 Err(e) => {
2219 tracing::warn!(
2220 repo,
2221 ?e,
2222 "failed to list subscribers for repo"
2223 );
2224 }
2225 }
2226 }
2227 }
2228
2229 if !routed {
2231 for channel in channels.iter() {
2232 let (body, should_send, is_telegram) = match channel.name() {
2233 "telegram" => (notification.format_telegram(), true, true),
2234 "discord" => (notification.format_discord(), true, false),
2235 "slack" => (notification.format_slack(), true, false),
2236 _ => (String::new(), false, false),
2239 };
2240
2241 if !should_send {
2242 continue;
2243 }
2244
2245 let metadata = if is_telegram {
2246 serde_json::json!({"preformatted_html": true})
2247 } else {
2248 serde_json::json!({})
2249 };
2250 let msg = OutgoingMessage {
2251 thread_id: notification.task_id.clone(),
2252 body,
2253 reply_to: None,
2254 metadata,
2255 topic_id: None,
2256 };
2257
2258 if let Err(e) = channel.send(&msg).await {
2259 tracing::warn!(
2260 channel = channel.name(),
2261 task_id = %notification.task_id,
2262 ?e,
2263 "failed to send notification"
2264 );
2265 }
2266 }
2267 }
2268 }
2269 Err(RecvError::Lagged(n)) => {
2270 tracing::warn!(missed = n, "notification receiver lagged");
2271 }
2272 Err(RecvError::Closed) => {
2273 tracing::debug!("notification channel closed");
2274 break;
2275 }
2276 }
2277 }
2278 });
2279 tracing::info!("notification dispatcher started");
2280 }
2281
2282 let webhook_notify = Arc::new(Notify::new());
2284
2285 let webhook_port: Option<u16>;
2288 let mut webhook_healthy: bool;
2289 let mut last_webhook_health_check = Instant::now();
2290 let webhook_enabled = config::get_global("webhook.enabled").is_ok_and(|v| v == "true");
2292 let webhook_secret = if webhook_enabled {
2294 secret_ref::config_secret("webhook.secret")
2295 .await
2296 .ok()
2297 .filter(|s| !s.trim().is_empty())
2298 } else {
2299 None
2300 };
2301 if webhook_enabled && webhook_secret.is_none() {
2302 tracing::error!(
2303 "webhook.enabled needs a readable webhook.secret, the webhook server stays off"
2304 );
2305 }
2306 let webhook_enabled = webhook_enabled && webhook_secret.is_some();
2307 let webhook_secret = webhook_secret.unwrap_or_default();
2308
2309 let webhook_status = Arc::new(Mutex::new(WebhookStatus::default()));
2311
2312 if webhook_enabled {
2313 const WEBHOOK_MAX_ATTEMPTS: u32 = 5;
2317 let port: u16 = config::get("webhook.port")
2318 .ok()
2319 .and_then(|v| v.parse().ok())
2320 .unwrap_or(8080);
2321 webhook_port = Some(port);
2322 let secret = webhook_secret.clone();
2323 let webhook_repo = project_engines
2324 .first()
2325 .map(|e| e.repo.clone())
2326 .unwrap_or_default();
2327
2328 let (tx, mut rx) = mpsc::channel::<IncomingMessage>(64);
2329
2330 let to_save = {
2332 let mut s = webhook_status.lock().await;
2333 s.configured = true;
2334 s.port = Some(port);
2335 s.healthy = false;
2336 s.fallback_mode = false;
2337 s.clone()
2338 };
2339 to_save.save().await;
2340
2341 let status_for_spawn = Arc::clone(&webhook_status);
2342 tokio::spawn(async move {
2343 let mut attempt = 0u32;
2344 loop {
2345 attempt += 1;
2346 tracing::info!(attempt, port, "attempting to start webhook server");
2347 match start_webhook_server(port, secret.clone(), webhook_repo.clone(), tx.clone())
2348 .await
2349 {
2350 Ok(()) => {
2351 tracing::info!(attempt, "webhook server exited cleanly");
2352 break;
2353 }
2354 Err(e) => {
2355 let reason = e.to_string();
2356 tracing::error!(attempt, %reason, "webhook server failed");
2357 if attempt >= WEBHOOK_MAX_ATTEMPTS || !is_transient_bind_error(&e) {
2358 let kind = if attempt >= WEBHOOK_MAX_ATTEMPTS {
2359 "max attempts reached"
2360 } else {
2361 "non-transient error"
2362 };
2363 tracing::error!(
2364 attempt,
2365 %reason,
2366 kind,
2367 orch_webhook_in_fallback = true,
2368 "webhook server giving up, switching to polling fallback"
2369 );
2370 let to_save = {
2371 let mut s = status_for_spawn.lock().await;
2372 s.fallback_mode = true;
2373 s.healthy = false;
2374 s.last_failure_reason = Some(reason);
2375 s.startup_attempts = attempt;
2376 s.clone()
2377 };
2378 to_save.save().await;
2379 break;
2380 }
2381 let delay = webhook_backoff_delay(attempt);
2382 tracing::warn!(
2383 attempt,
2384 delay_ms = delay.as_millis(),
2385 %reason,
2386 "webhook bind failed (transient), retrying with backoff"
2387 );
2388 {
2389 let to_save = {
2390 let mut s = status_for_spawn.lock().await;
2391 s.startup_attempts = attempt;
2392 s.last_failure_reason = Some(reason);
2393 s.clone()
2394 };
2395 to_save.save().await;
2396 }
2397 time::sleep(delay).await;
2398 }
2399 }
2400 }
2401 });
2402
2403 let notify = Arc::clone(&webhook_notify);
2405 let transport_for_webhook = Arc::clone(&transport);
2406 tokio::spawn(async move {
2407 while let Some(msg) = rx.recv().await {
2408 tracing::info!(
2409 channel = %msg.channel,
2410 thread = %msg.thread_id,
2411 event = %msg.metadata.get("event").and_then(|v| v.as_str()).unwrap_or("unknown"),
2412 action = %msg.metadata.get("action").and_then(|v| v.as_str()).unwrap_or("unknown"),
2413 "webhook event received, triggering immediate tick"
2414 );
2415 let route = transport_for_webhook.route(&msg).await;
2416 tracing::info!(
2417 channel = %msg.channel,
2418 route = ?route,
2419 "webhook event routed"
2420 );
2421 notify.notify_one();
2423 }
2424 });
2425
2426 webhook_healthy = true;
2427 tracing::info!(port, orch_webhook_up = true, "webhook server started");
2428 } else {
2429 webhook_port = None;
2430 webhook_healthy = false;
2431 {
2432 let to_save = {
2433 let mut s = webhook_status.lock().await;
2434 s.configured = false;
2435 s.fallback_mode = true;
2436 s.clone()
2437 };
2438 to_save.save().await;
2439 }
2440 tracing::info!(
2441 orch_webhook_in_fallback = true,
2442 "webhook server disabled, using polling fallback mode"
2443 );
2444 }
2445
2446 start_relay_client(&store, &webhook_notify);
2447 app::start_loader();
2448
2449 let router = Arc::new(RwLock::new(
2453 match task::spawn_blocking(Router::from_config).await {
2454 Ok(r) => r,
2455 Err(e) => {
2456 tracing::error!(?e, "router init panicked in spawn_blocking, using default");
2457 task::spawn_blocking(|| Router::new(RouterConfig::default()))
2458 .await
2459 .unwrap_or_else(|e2| {
2460 panic!("router default init also panicked in spawn_blocking — cannot start engine without a router: {e2:?}");
2461 })
2462 }
2463 },
2464 ));
2465 {
2466 let r = router.read().await;
2467 tracing::info!(
2468 mode = %r.config.mode,
2469 agents = ?r.available_agents,
2470 fallback = %r.config.fallback_executor,
2471 "router initialized"
2472 );
2473 }
2474
2475 let semaphore = Arc::new(Semaphore::new(config.max_parallel));
2479 let mut parallel_in_force = config.max_parallel;
2480 let review_semaphore = Arc::new(Semaphore::new(config.max_parallel));
2483 let mut review_in_force = config.max_parallel;
2484
2485 let dispatching: Arc<dashmap::DashMap<String, String>> = Arc::new(dashmap::DashMap::new());
2492
2493 let auto_merge_in_flight: Arc<dashmap::DashSet<String>> = Arc::new(dashmap::DashSet::new());
2497
2498 let mut config_rx = config::subscribe();
2500
2501 let mut last_sync = Instant::now();
2503
2504 let mut last_upgrade_check: Option<Instant> = None;
2506
2507 let mut rate_limit_reported: Option<i64> = None;
2510
2511 let (weight_tx, mut weight_rx) = mpsc::channel::<WeightSignal>(64);
2513
2514 for engine in project_engines.iter().filter(|_| leads) {
2516 let reset =
2517 reset_orphaned_reviews(&engine.task_manager, &engine.store, &engine.repo, &tmux).await;
2518
2519 if reset > 0 {
2520 tracing::info!(repo = %engine.repo, reset, "reset orphaned reviews to needs_review on startup");
2521 }
2522 }
2523
2524 for engine in &project_engines {
2526 subscribers::dispatch::spawn(
2527 event_bus.subscribe(),
2528 Arc::clone(&engine.backend),
2529 Arc::clone(&tmux),
2530 Arc::clone(&engine.runner),
2531 Arc::clone(&capture_for_tick),
2532 Arc::clone(&semaphore),
2533 Arc::clone(&engine.task_manager),
2534 weight_tx.clone(),
2535 Arc::clone(&router),
2536 Arc::clone(&dispatching),
2537 Arc::clone(&engine.store),
2538 engine.repo.clone(),
2539 );
2540 }
2541 tracing::info!("dispatch subscriber started");
2542
2543 for engine in &project_engines {
2545 subscribers::review::spawn(
2546 event_bus.subscribe(),
2547 Arc::clone(&engine.backend),
2548 Arc::clone(&tmux),
2549 Arc::clone(&review_semaphore),
2550 Arc::clone(&engine.task_manager),
2551 Arc::clone(&router),
2552 Arc::clone(&dispatching),
2553 Arc::clone(&engine.store),
2554 engine.repo.clone(),
2555 );
2556 }
2557 tracing::info!("review subscriber started");
2558
2559 for engine in &project_engines {
2561 subscribers::unblock::spawn(
2562 event_bus.subscribe(),
2563 Arc::clone(&engine.backend),
2564 Arc::clone(&engine.task_manager),
2565 Arc::clone(&engine.store),
2566 engine.repo.clone(),
2567 );
2568 }
2569 tracing::info!("unblock subscriber started");
2570
2571 subscribers::notify::spawn(event_bus.subscribe(), Arc::clone(&transport));
2574 tracing::info!("notify subscriber started");
2575
2576 tracing::info!(
2578 tick = ?config.tick_interval,
2579 sync = ?config.sync_interval,
2580 parallel = config.max_parallel,
2581 "entering main loop"
2582 );
2583 let mut restart_only = restart_only_snapshot();
2586
2587 let mut interval = time::interval(config.tick_interval);
2588
2589 let mut sigterm = unix::signal(SignalKind::terminate())?;
2591 let mut sighup = unix::signal(SignalKind::hangup())?;
2592
2593 let sync_in_progress = Arc::new(AtomicBool::new(false));
2595
2596 let last_tick_epoch = Arc::new(AtomicU64::new(
2599 SystemTime::now()
2600 .duration_since(std_time::UNIX_EPOCH)
2601 .unwrap_or_default()
2602 .as_secs(),
2603 ));
2604 {
2605 let last_tick_epoch = Arc::clone(&last_tick_epoch);
2606 let tick_interval_secs = config.tick_interval.as_secs();
2607 tokio::spawn(async move {
2608 let mut watchdog_interval = time::interval(Duration::from_secs(30));
2610 let mut suspend_detector = SuspendDetector::new();
2615 loop {
2616 watchdog_interval.tick().await;
2617 let own_gap = suspend_detector.check();
2618 let last = last_tick_epoch.load(Ordering::Relaxed);
2619 let now = SystemTime::now()
2620 .duration_since(std_time::UNIX_EPOCH)
2621 .unwrap_or_default()
2622 .as_secs();
2623 let stale_secs = now.saturating_sub(last);
2624 let threshold = (tick_interval_secs * 6).max(60);
2626 if stale_secs > threshold {
2627 let shared_gap =
2634 engine_suspend::gap_detected_within(Duration::from_secs(stale_secs));
2635 let gap = match (own_gap, shared_gap) {
2636 (Some(a), Some(b)) => Some(a.max(b)),
2637 (Some(a), None) => Some(a),
2638 (None, Some(b)) => Some(b),
2639 (None, None) => None,
2640 };
2641 if let Some(gap) = gap {
2642 let gap_secs = u64::try_from(gap.num_seconds()).unwrap_or(0);
2643 if gap_secs >= stale_secs {
2644 tracing::info!(
2645 stale_secs,
2646 threshold,
2647 suspend_gap_secs = gap_secs,
2648 "tick loop stale window explained by host suspend/resume, not a stall"
2649 );
2650 } else {
2651 tracing::error!(
2652 stale_secs,
2653 threshold,
2654 suspend_gap_secs = gap_secs,
2655 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — suspend gap of {}s only partially explains the stall",
2656 stale_secs,
2657 threshold,
2658 gap_secs,
2659 );
2660 }
2661 } else {
2662 tracing::error!(
2663 stale_secs,
2664 threshold,
2665 "WATCHDOG: tick loop has not completed a tick in {}s (threshold {}s) — possible stall",
2666 stale_secs,
2667 threshold,
2668 );
2669 }
2670 }
2671 }
2672 });
2673 }
2674
2675 loop {
2676 tokio::select! {
2677 _ = interval.tick() => {
2678 let tick_start = Instant::now();
2679
2680 if let Some(gap) = engine_suspend::checkpoint() {
2684 tracing::info!(
2685 gap_secs = gap.num_seconds(),
2686 "host suspend/resume gap detected, discounting from stuck-task age checks"
2687 );
2688 }
2689
2690 if review_in_force != config.max_parallel {
2694 review_in_force =
2695 resize_dispatch_pool(&review_semaphore, review_in_force, config.max_parallel);
2696 }
2697 if parallel_in_force != config.max_parallel {
2698 let resized =
2699 resize_dispatch_pool(&semaphore, parallel_in_force, config.max_parallel);
2700 if resized != parallel_in_force {
2701 parallel_in_force = resized;
2702 tracing::info!(
2703 in_force = parallel_in_force,
2704 target = config.max_parallel,
2705 "dispatch pool resized"
2706 );
2707 }
2708 }
2709
2710 {
2712 let mut rw = router.write().await;
2713 rw.tick_weight_recovery();
2714 }
2715
2716 if let Some(remaining) = GhHttp::is_rate_limited() {
2720 let reset_at = chrono::Utc::now()
2721 .timestamp()
2722 .saturating_add(i64::try_from(remaining.as_secs()).unwrap_or(i64::MAX));
2723 if rate_limit_reported.is_none_or(|prev| (reset_at - prev).abs() > 60) {
2725 rate_limit_reported = Some(reset_at);
2726 report_rate_limited(reset_at, Some(&store), Some(&transport)).await;
2727 }
2728 } else if let Some(prev) = rate_limit_reported.take() {
2729 tracing::info!(
2730 limited_until = prev,
2731 "GitHub API rate limit cleared"
2732 );
2733 let _ = store.kv_delete(RATE_LIMITED_UNTIL_KEY).await;
2734 }
2735
2736 tick::tick_all(
2738 &project_engines,
2739 &tmux,
2740 &capture_for_tick,
2741 &semaphore,
2742 &config,
2743 &router,
2744 &weight_tx,
2745 &dispatching,
2746 Some(&transport),
2747 &active_repos,
2748 &auto_merge_in_flight,
2749 ).await;
2750
2751 while let Ok(signal) = weight_rx.try_recv() {
2754 let mut rw = router.write().await;
2755 match signal {
2756 WeightSignal::RateLimited { ref agent } => {
2757 rw.record_rate_limit(agent);
2758 }
2759 WeightSignal::Success { ref agent } => {
2760 rw.record_success(agent);
2761 }
2762 WeightSignal::Failed { .. }
2763 | WeightSignal::Rerouted
2764 | WeightSignal::Blocked
2765 | WeightSignal::None => {}
2766 }
2767 }
2768
2769 if last_sync.elapsed() >= config.sync_interval {
2774 if sync_in_progress.compare_exchange(
2775 false,
2776 true,
2777 Ordering::AcqRel,
2778 Ordering::Relaxed,
2779 ).is_ok() {
2780 spawn_sync_pass(
2781 &project_engines,
2782 &tmux,
2783 &config,
2784 &router,
2785 &dispatching,
2786 &auto_merge_in_flight,
2787 &sync_in_progress,
2788 "periodic",
2789 );
2790 } else {
2791 tracing::debug!("sync tick still in progress, skipping");
2792 }
2793 last_sync = Instant::now();
2797 }
2798
2799 if webhook_enabled {
2801 let health_check_interval = config.webhook_health_check_interval
2802 .unwrap_or(Duration::from_secs(60));
2803
2804 if last_webhook_health_check.elapsed() >= health_check_interval {
2805 if let Some(port) = webhook_port {
2806 let (health, failure_reason) =
2807 github::check_webhook_health(port).await;
2808 if health != webhook_healthy {
2809 webhook_healthy = health;
2810 if webhook_healthy {
2811 tracing::info!(port, orch_webhook_up = true, "webhook health restored");
2812 } else {
2813 tracing::warn!(
2814 port,
2815 orch_webhook_up = false,
2816 reason = failure_reason.as_deref().unwrap_or("unknown"),
2817 "webhook health check failed"
2818 );
2819 }
2820 }
2821 {
2823 let to_save = {
2824 let mut s = webhook_status.lock().await;
2825 s.healthy = health;
2826 s.last_check_utc = Some(chrono::Utc::now());
2827 if health {
2828 s.last_failure_reason = None;
2829 } else if failure_reason.is_some() {
2830 s.last_failure_reason = failure_reason;
2831 }
2832 s.clone()
2833 };
2834 to_save.save().await;
2835 }
2836 }
2837 last_webhook_health_check = Instant::now();
2838 }
2839 }
2840
2841 if should_run_upgrade_check(last_upgrade_check, config.upgrade_check_interval) {
2843 let store = Arc::clone(&store);
2844 let channels = Arc::clone(&channel_registry);
2845 let current = env!("ORCH_VERSION").to_string();
2846 let check_interval = Duration::from_secs(config.upgrade_check_interval);
2847 tokio::spawn(async move {
2848 check_and_notify_upgrade(&store, &channels, ¤t, check_interval)
2849 .await;
2850 });
2851 last_upgrade_check = Some(Instant::now());
2852 }
2853
2854 let tick_elapsed = tick_start.elapsed();
2856 if tick_elapsed.as_secs() > 30 {
2857 tracing::warn!(
2858 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2859 "slow tick"
2860 );
2861 } else {
2862 tracing::debug!(
2863 elapsed_ms = u64::try_from(tick_elapsed.as_millis()).unwrap_or(u64::MAX),
2864 "tick complete"
2865 );
2866 }
2867 last_tick_epoch.store(
2868 SystemTime::now()
2869 .duration_since(std_time::UNIX_EPOCH)
2870 .unwrap_or_default()
2871 .as_secs(),
2872 Ordering::Relaxed,
2873 );
2874 }
2875 () = webhook_notify.notified() => {
2877 if let Some(remaining) = GhHttp::is_rate_limited() {
2878 tracing::warn!(
2879 remaining_secs = remaining.as_secs(),
2880 "GitHub API rate-limited, skipping webhook-triggered tick"
2881 );
2882 } else {
2883 tracing::info!("webhook event triggered immediate tick");
2884
2885 tick::tick_all(
2886 &project_engines,
2887 &tmux,
2888 &capture_for_tick,
2889 &semaphore,
2890 &config,
2891 &router,
2892 &weight_tx,
2893 &dispatching,
2894 Some(&transport),
2895 &active_repos,
2896 &auto_merge_in_flight,
2897 ).await;
2898
2899 while let Ok(signal) = weight_rx.try_recv() {
2901 let mut rw = router.write().await;
2902 match signal {
2903 WeightSignal::RateLimited { ref agent } => {
2904 rw.record_rate_limit(agent);
2905 }
2906 WeightSignal::Success { ref agent } => {
2907 rw.record_success(agent);
2908 }
2909 WeightSignal::Failed { .. }
2910 | WeightSignal::Rerouted
2911 | WeightSignal::Blocked
2912 | WeightSignal::None => {}
2913 }
2914 }
2915
2916 if sync_in_progress.compare_exchange(
2922 false,
2923 true,
2924 Ordering::AcqRel,
2925 Ordering::Relaxed,
2926 ).is_ok() {
2927 spawn_sync_pass(
2928 &project_engines,
2929 &tmux,
2930 &config,
2931 &router,
2932 &dispatching,
2933 &auto_merge_in_flight,
2934 &sync_in_progress,
2935 "webhook",
2936 );
2937 last_sync = Instant::now();
2939 } else {
2940 tracing::debug!("sync tick still in progress, skipping webhook-triggered immediate sync");
2941 }
2942 }
2943
2944 interval.reset();
2946 }
2947 result = config_rx.recv() => {
2948 let reload = match result {
2951 Ok(path) => {
2952 tracing::info!(path = %path.display(), "config file changed, reloading");
2953 true
2954 }
2955 Err(RecvError::Lagged(n)) => {
2956 tracing::warn!(missed = n, "config change receiver lagged, reloading");
2957 true
2958 }
2959 Err(RecvError::Closed) => {
2960 tracing::warn!("config change channel closed");
2961 false
2962 }
2963 };
2964
2965 if reload {
2966 let new_config = EngineConfig::from_config();
2968 log_engine_config_changes(&config, &new_config);
2969
2970 let new_restart_only = restart_only_snapshot();
2971 log_restart_only_changes(&restart_only, &new_restart_only);
2972 restart_only = new_restart_only;
2973
2974 let tick_changed = new_config.tick_interval != config.tick_interval;
2975 config = new_config;
2976
2977 if tick_changed {
2979 interval = time::interval(config.tick_interval);
2980 tracing::info!(tick = ?config.tick_interval, "tick interval updated");
2981 }
2982
2983 if review_in_force != config.max_parallel {
2986 review_in_force = resize_dispatch_pool(
2987 &review_semaphore,
2988 review_in_force,
2989 config.max_parallel,
2990 );
2991 }
2992 if parallel_in_force != config.max_parallel {
2993 parallel_in_force = resize_dispatch_pool(
2994 &semaphore,
2995 parallel_in_force,
2996 config.max_parallel,
2997 );
2998 tracing::info!(
2999 in_force = parallel_in_force,
3000 target = config.max_parallel,
3001 "dispatch pool resized"
3002 );
3003 }
3004
3005 {
3007 let mut router_guard = router.write().await;
3008 router_guard.reload_async().await;
3009 }
3010
3011 sync_project_engines(
3013 &store,
3014 &mut project_engines,
3015 &mut active_repos,
3016 &event_bus,
3017 &tmux,
3018 &capture_for_tick,
3019 &semaphore,
3020 &review_semaphore,
3021 &weight_tx,
3022 &router,
3023 &dispatching,
3024 ).await;
3025
3026 tracing::info!(
3027 tick = ?config.tick_interval,
3028 sync = ?config.sync_interval,
3029 parallel = config.max_parallel,
3030 projects = project_engines.len(),
3031 "config reloaded"
3032 );
3033 }
3034 }
3035 signal_name = async {
3036 tokio::select! {
3037 _ = signal::ctrl_c() => "SIGINT",
3038 _ = sigterm.recv() => "SIGTERM",
3039 _ = sighup.recv() => "SIGHUP",
3040 }
3041 } => {
3042 tracing::info!(signal = signal_name, "beginning graceful shutdown");
3043
3044 let shutdown_result = time::timeout(
3047 config.graceful_shutdown_timeout,
3048 async {
3049 let mut reset_count = 0u32;
3053 let mut review_reset_count = 0u32;
3054 for engine in &project_engines {
3055 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InProgress).await {
3056 for task in &tasks {
3057 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::Routed).await {
3058 tracing::warn!(task_id = task.id.0, ?e, "failed to reset task on shutdown");
3059 } else {
3060 if let Ok(Some(store_id)) =
3061 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3062 && let Err(e) = engine
3063 .store
3064 .finalize_incomplete_runs(
3065 store_id,
3066 "aborted",
3067 "graceful shutdown: reset in_progress task to routed",
3068 )
3069 .await
3070 {
3071 tracing::warn!(
3072 task_id = task.id.0,
3073 ?e,
3074 "failed to finalize incomplete runs on shutdown"
3075 );
3076 }
3077 reset_count += 1;
3078 }
3079 }
3080 }
3081 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InProgress).await {
3083 for task in &tasks {
3084 let task_id = format!("internal:{}", task.id);
3085 if let Err(e) = engine.task_manager.update_task_status(
3086 &ExternalId(task_id.clone()),
3087 Status::Routed,
3088 ).await {
3089 tracing::warn!(task_id, ?e, "failed to reset internal task on shutdown");
3090 } else {
3091 if let Err(e) = engine
3092 .store
3093 .finalize_incomplete_runs(
3094 task.id,
3095 "aborted",
3096 "graceful shutdown: reset in_progress task to routed",
3097 )
3098 .await
3099 {
3100 tracing::warn!(
3101 task_id,
3102 ?e,
3103 "failed to finalize incomplete internal runs on shutdown"
3104 );
3105 }
3106 reset_count += 1;
3107 }
3108 }
3109 }
3110 if let Ok(tasks) = engine.task_manager.list_external_by_status(Status::InReview).await {
3112 for task in &tasks {
3113 if let Err(e) = engine.task_manager.update_task_status(&task.id, Status::NeedsReview).await {
3114 tracing::warn!(task_id = task.id.0, ?e, "failed to reset in_review task on shutdown");
3115 } else {
3116 if let Ok(Some(store_id)) =
3117 engine.store.resolve_task_id(&engine.repo, &task.id.0).await
3118 && let Err(e) = engine
3119 .store
3120 .finalize_incomplete_runs(
3121 store_id,
3122 "aborted",
3123 "graceful shutdown: reset in_review task to needs_review",
3124 )
3125 .await
3126 {
3127 tracing::warn!(
3128 task_id = task.id.0,
3129 ?e,
3130 "failed to finalize incomplete review runs on shutdown"
3131 );
3132 }
3133 review_reset_count += 1;
3134 }
3135 }
3136 }
3137 if let Ok(tasks) = engine.store.list_internal_by_status(&engine.repo, TaskStatus::InReview).await {
3139 for task in &tasks {
3140 let task_id = format!("internal:{}", task.id);
3141 if let Err(e) = engine.task_manager.update_task_status(
3142 &ExternalId(task_id.clone()),
3143 Status::NeedsReview,
3144 ).await {
3145 tracing::warn!(task_id, ?e, "failed to reset internal in_review task on shutdown");
3146 } else {
3147 if let Err(e) = engine
3148 .store
3149 .finalize_incomplete_runs(
3150 task.id,
3151 "aborted",
3152 "graceful shutdown: reset in_review task to needs_review",
3153 )
3154 .await
3155 {
3156 tracing::warn!(
3157 task_id,
3158 ?e,
3159 "failed to finalize incomplete internal review runs on shutdown"
3160 );
3161 }
3162 review_reset_count += 1;
3163 }
3164 }
3165 }
3166 }
3167 if reset_count > 0 {
3168 tracing::info!(reset_count, "reset in_progress tasks to routed for re-dispatch");
3169 }
3170 if review_reset_count > 0 {
3171 tracing::info!(review_reset_count, "reset in_review tasks to needs_review for re-dispatch");
3172 }
3173
3174 let mut killed_sessions = Vec::new();
3177 if let Ok(sessions) = tmux.list_sessions().await {
3178 for session in &sessions {
3179 if session.name.starts_with("orch-") {
3180 match tmux.kill_session(&session.name).await { Err(e) => {
3181 tracing::warn!(session = %session.name, error = %e, "failed to kill tmux session");
3182 } _ => {
3183 killed_sessions.push(session.name.clone());
3184 }}
3185 }
3186 }
3187 if !killed_sessions.is_empty() {
3188 tracing::info!(killed = killed_sessions.len(), "killing orch tmux sessions on shutdown");
3189 }
3190 }
3191
3192 if !killed_sessions.is_empty() {
3195 let still_alive = tmux
3196 .wait_for_sessions_dead(
3197 &killed_sessions,
3198 Duration::from_millis(100),
3199 Duration::from_secs(5),
3200 )
3201 .await;
3202 if still_alive > 0 {
3203 tracing::warn!(
3204 still_alive,
3205 "some tmux sessions did not terminate cleanly - they will be cleaned up on startup"
3206 );
3207 }
3208 tracing::info!(killed = killed_sessions.len() - still_alive, "confirmed tmux sessions terminated");
3209 }
3210 },
3211 ).await;
3212
3213 if shutdown_result.is_err() {
3214 tracing::error!(
3215 timeout_secs = config.graceful_shutdown_timeout.as_secs(),
3216 "graceful shutdown timed out, forcing exit"
3217 );
3218 }
3219
3220 break;
3221 }
3222 }
3223 }
3224
3225 events::cleanup_port_file().await;
3227 local::cleanup().await;
3228 ui::cleanup().await;
3229 events::cleanup_version_file();
3230
3231 let _ = transport;
3233 tracing::info!("orch engine stopped");
3234 Ok(())
3235}
3236
3237#[cfg(test)]
3238mod tests {
3239 use std::env;
3240 use std::process;
3241 use std::slice;
3242 use std::sync::Arc;
3243
3244 use super::*;
3245 use crate::backends::ExternalBackend;
3246 use crate::backends::test_helpers::NoopBackend;
3247 use crate::channels::capture::CaptureService;
3248 use crate::channels::transport::Transport;
3249 use crate::engine::runner::git_ops;
3250 use crate::store::UpsertExternal;
3251 use crate::tmux::SessionGuard;
3252 use crate::tmux::TmuxManager;
3253
3254 async fn in_review_task(store: &TaskStore, number: &str) -> i64 {
3255 let id = store
3256 .upsert_external(&UpsertExternal {
3257 repo: "owner/repo",
3258 ext_id: number,
3259 title: "Under review",
3260 body: "",
3261 author: "",
3262 url: "",
3263 labels: &[],
3264 origin: "github",
3265 })
3266 .await
3267 .unwrap();
3268 store.update_status(id, TaskStatus::InReview).await.unwrap();
3269 id
3270 }
3271
3272 #[tokio::test]
3273 async fn a_review_without_a_session_goes_back_to_needs_review_and_a_live_one_stays() {
3274 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3275 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3276 let task_manager =
3277 TaskManager::with_store(backend, Arc::clone(&store), "owner/repo".to_string());
3278 let tmux = TmuxManager::isolated();
3279 let orphaned = in_review_task(&store, "11").await;
3280 let live = in_review_task(&store, "12").await;
3281 let session = tmux
3282 .create_session("owner/repo", "12-review", "/tmp", "sleep 30", &[])
3283 .await
3284 .unwrap();
3285 let _guard = SessionGuard(session);
3286
3287 let reset = reset_orphaned_reviews(&task_manager, &store, "owner/repo", &tmux).await;
3288
3289 assert_eq!(reset, 1);
3290 assert_eq!(
3291 store.get(orphaned).await.unwrap().status,
3292 TaskStatus::NeedsReview
3293 );
3294 assert_eq!(store.get(live).await.unwrap().status, TaskStatus::InReview);
3295 }
3296
3297 #[test]
3298 fn growing_max_parallel_lets_more_tasks_run_without_a_restart() {
3299 let semaphore = Arc::new(Semaphore::new(4));
3300 let held: Vec<_> = (0..4)
3301 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3302 .collect();
3303 assert!(
3304 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3305 "pool is full"
3306 );
3307
3308 let in_force = resize_dispatch_pool(&semaphore, 4, 8);
3309 assert_eq!(in_force, 8);
3310
3311 let grown: Vec<_> = (0..4)
3312 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3313 .collect();
3314 assert_eq!(held.len() + grown.len(), 8, "8 tasks run at once");
3315 assert!(
3316 Arc::clone(&semaphore).try_acquire_owned().is_err(),
3317 "pool is full"
3318 );
3319 }
3320
3321 #[test]
3322 fn shrinking_max_parallel_never_cancels_a_running_task() {
3323 let semaphore = Arc::new(Semaphore::new(8));
3324 let mut held: Vec<_> = (0..8)
3325 .map(|_| Arc::clone(&semaphore).try_acquire_owned().unwrap())
3326 .collect();
3327
3328 let mut in_force = resize_dispatch_pool(&semaphore, 8, 4);
3330 assert_eq!(in_force, 8);
3331 assert_eq!(held.len(), 8);
3332
3333 held.truncate(5);
3335 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3336 assert_eq!(in_force, 5);
3337 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3338
3339 held.truncate(4);
3341 in_force = resize_dispatch_pool(&semaphore, in_force, 4);
3342 assert_eq!(in_force, 4);
3343 assert!(Arc::clone(&semaphore).try_acquire_owned().is_err());
3344
3345 held.truncate(3);
3347 assert!(Arc::clone(&semaphore).try_acquire_owned().is_ok());
3348 }
3349
3350 #[test]
3351 fn a_project_added_to_the_config_is_picked_up_and_running_ones_are_not_respawned() {
3352 let configured = vec![
3353 ("owner/one".to_string(), PathBuf::from("/srv/one")),
3354 ("owner/two".to_string(), PathBuf::from("/srv/two")),
3355 ];
3356 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3357
3358 let (added, removed) = diff_configured_projects(&configured, &active);
3359 assert_eq!(
3360 added,
3361 vec![("owner/two".to_string(), PathBuf::from("/srv/two"))],
3362 "only the new project is started, so owner/one keeps its single set of subscribers"
3363 );
3364 assert!(removed.is_empty(), "{removed:?}");
3365 }
3366
3367 #[test]
3368 fn a_project_dropped_from_the_config_is_reported_and_not_restarted() {
3369 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3370 let active: HashSet<String> = ["owner/one".to_string(), "owner/gone".to_string()]
3371 .into_iter()
3372 .collect();
3373
3374 let (added, removed) = diff_configured_projects(&configured, &active);
3375 assert!(added.is_empty(), "nothing new to start");
3376 assert_eq!(removed, vec!["owner/gone".to_string()]);
3377 }
3378
3379 #[test]
3380 fn an_unchanged_project_list_starts_nothing() {
3381 let configured = vec![("owner/one".to_string(), PathBuf::from("/srv/one"))];
3382 let active: HashSet<String> = ["owner/one".to_string()].into_iter().collect();
3383
3384 assert_eq!(
3385 diff_configured_projects(&configured, &active),
3386 (vec![], vec![])
3387 );
3388 }
3389
3390 #[test]
3391 fn restart_only_keys_are_the_settings_read_once_at_startup() {
3392 assert_eq!(RESTART_ONLY_KEYS, ["channels", "webhook", "relay"]);
3395 }
3396
3397 #[test]
3398 fn a_changed_restart_only_key_is_reported_and_an_unchanged_one_is_not() {
3399 let before = vec![Some("bot_token: a\n".to_string()), None];
3400
3401 assert!(
3402 log_restart_only_changes(&before, &before.clone()).is_empty(),
3403 "a reload that did not touch these keys must stay quiet"
3404 );
3405
3406 let after = vec![
3408 Some("bot_token: b\n".to_string()),
3409 Some("port: 8080\n".to_string()),
3410 ];
3411 assert_eq!(
3412 log_restart_only_changes(&before, &after),
3413 ["channels", "webhook"]
3414 );
3415
3416 let token_only = vec![Some("bot_token: b\n".to_string()), None];
3418 assert_eq!(log_restart_only_changes(&before, &token_only), ["channels"]);
3419 }
3420
3421 #[test]
3422 fn resize_dispatch_pool_is_a_noop_at_the_current_limit() {
3423 let semaphore = Arc::new(Semaphore::new(4));
3424 assert_eq!(resize_dispatch_pool(&semaphore, 4, 4), 4);
3425 assert_eq!(semaphore.available_permits(), 4);
3426 }
3427
3428 #[test]
3429 fn live_session_stuck_limit_follows_the_run_timeout() {
3430 let config = EngineConfig {
3431 stuck_timeout: 1800,
3432 run_timeouts: RunTimeouts {
3433 base: 1800,
3434 medium: 2700,
3435 complex: 3600,
3436 },
3437 ..EngineConfig::default()
3438 };
3439 assert_eq!(
3440 config.live_session_stuck_limit(Some("simple")),
3441 1800 + RUN_TIMEOUT_GRACE_SECS
3442 );
3443 assert_eq!(
3444 config.live_session_stuck_limit(Some("medium")),
3445 2700 + RUN_TIMEOUT_GRACE_SECS
3446 );
3447 assert_eq!(
3448 config.live_session_stuck_limit(Some("complex")),
3449 3600 + RUN_TIMEOUT_GRACE_SECS
3450 );
3451
3452 let long_stuck = EngineConfig {
3453 stuck_timeout: 7200,
3454 ..config
3455 };
3456 assert_eq!(long_stuck.live_session_stuck_limit(Some("complex")), 7200);
3457 }
3458
3459 #[test]
3460 fn engine_config_defaults() {
3461 let config = EngineConfig::default();
3462 assert_eq!(config.tick_interval, Duration::from_secs(10));
3463 assert_eq!(config.sync_interval, Duration::from_secs(45));
3464 assert_eq!(config.max_parallel, 4);
3465 assert_eq!(config.stuck_timeout, 1800);
3466 assert_eq!(config.no_session_stuck_timeout, 600);
3467 assert!(
3468 config.no_session_stuck_timeout < config.stuck_timeout,
3469 "no_session_stuck_timeout must be shorter than stuck_timeout"
3470 );
3471 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3472 assert_eq!(config.upgrade_check_interval, 3600);
3473 }
3474
3475 #[test]
3476 fn engine_config_from_config_uses_defaults_when_no_config() {
3477 let config = EngineConfig::from_config();
3478 assert_eq!(config.tick_interval, Duration::from_secs(10));
3479 assert_eq!(config.sync_interval, Duration::from_secs(45));
3480 assert!(config.max_parallel >= 1 && config.max_parallel <= 64);
3482 assert_eq!(config.stuck_timeout, 1800);
3483 assert_eq!(config.no_session_stuck_timeout, 600);
3484 assert_eq!(config.graceful_shutdown_timeout, Duration::from_secs(600));
3485 assert_eq!(config.upgrade_check_interval, 3600);
3486 }
3487
3488 #[test]
3489 fn engine_config_no_session_timeout_is_under_15_minutes() {
3490 let config = EngineConfig::default();
3491 assert!(
3492 config.no_session_stuck_timeout <= 900,
3493 "no_session_stuck_timeout ({}) exceeds 15 minutes",
3494 config.no_session_stuck_timeout
3495 );
3496 }
3497
3498 #[test]
3499 fn upgrade_check_runs_immediately_when_never_checked() {
3500 assert!(should_run_upgrade_check(None, 3600));
3501 }
3502
3503 #[test]
3504 fn upgrade_check_waits_for_interval_after_first_run() {
3505 assert!(!should_run_upgrade_check(Some(Instant::now()), 3600));
3506 }
3507
3508 #[test]
3509 fn upgrade_check_can_be_disabled() {
3510 assert!(!should_run_upgrade_check(None, 0));
3511 }
3512
3513 #[test]
3514 fn configured_agents_parses_yaml_block_array() {
3515 let yaml_str = "- claude\n- codex\n- opencode\n- olm\n";
3517 let parsed: Vec<String> = serde_norway::from_str(yaml_str).unwrap();
3518 assert_eq!(parsed, vec!["claude", "codex", "opencode", "olm"]);
3519 }
3520
3521 #[test]
3522 fn configured_agents_parses_json_inline_array() {
3523 let json_str = r#"["claude","codex","olm"]"#;
3524 let parsed: Vec<String> = serde_json::from_str(json_str).unwrap();
3525 assert_eq!(parsed, vec!["claude", "codex", "olm"]);
3526 }
3527
3528 #[test]
3529 fn configured_agents_fallback_returns_default_agents() {
3530 let defaults = configured_agents();
3532 assert!(defaults.contains(&"claude".to_string()));
3533 assert!(defaults.contains(&"codex".to_string()));
3534 assert!(defaults.contains(&"opencode".to_string()));
3535 }
3536
3537 #[tokio::test]
3546 #[ignore = "capture-pane timing is unreliable on CI runners"]
3547 async fn integration_channel_to_tmux_to_capture() {
3548 let transport = Arc::new(Transport::new());
3550 let capture = Arc::new(CaptureService::new(Arc::clone(&transport)));
3551 let tmux = TmuxManager::new();
3552
3553 let task_id = "testtask1".to_string();
3554 let session_name = format!("orch-test-{task_id}");
3555
3556 let _ = tmux
3558 .command()
3559 .args(["new-session", "-d", "-s", &session_name])
3560 .output()
3561 .await
3562 .expect("failed to start tmux session");
3563
3564 let _guard = SessionGuard(session_name.clone());
3566
3567 capture
3569 .register_session("owner/repo", &task_id, &session_name)
3570 .await;
3571 transport
3572 .bind(
3573 "owner/repo",
3574 &task_id,
3575 &session_name,
3576 "telegram",
3577 "12345",
3578 None,
3579 )
3580 .await;
3581
3582 let capture_clone = Arc::clone(&capture);
3584 let capture_handle = tokio::spawn(async move { capture_clone.run().await });
3585
3586 let mut rx = transport
3588 .subscribe("owner/repo", &task_id)
3589 .await
3590 .expect("no subscription");
3591
3592 let send = tmux
3594 .command()
3595 .args([
3596 "send-keys",
3597 "-t",
3598 &session_name,
3599 "echo from-channel",
3600 "Enter",
3601 ])
3602 .output()
3603 .await
3604 .expect("failed to send keys");
3605 assert!(send.status.success());
3606
3607 let mut got = false;
3609 for _ in 0..20 {
3610 if let Ok(Ok(c)) = time::timeout(Duration::from_millis(500), rx.recv()).await
3611 && c.content.contains("from-channel")
3612 {
3613 got = true;
3614 break;
3615 }
3616 }
3617
3618 capture.unregister_session("owner/repo", &task_id).await;
3620
3621 let _ = capture_handle.await;
3623
3624 assert!(got, "did not observe tmux output via capture/transport");
3625 }
3626
3627 #[test]
3630 fn read_project_channel_config_returns_defaults_when_no_file() {
3631 let dir = tempfile::tempdir().expect("tempdir");
3632 let cfg = read_project_channel_config(dir.path());
3633 assert!(cfg.telegram_topic_id.is_none());
3634 assert!(cfg.discord_channel_id.is_none());
3635 }
3636
3637 #[test]
3638 fn read_project_channel_config_parses_telegram_and_discord() {
3639 let dir = tempfile::tempdir().expect("tempdir");
3640 let orch_yml = dir.path().join(".orch.yml");
3641 fs::write(
3642 &orch_yml,
3643 "channels:\n telegram:\n topic_id: \"42\"\n discord:\n channel_id: \"9999\"\n",
3644 )
3645 .expect("write .orch.yml");
3646 let cfg = read_project_channel_config(dir.path());
3647 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("42"));
3648 assert_eq!(cfg.discord_channel_id.as_deref(), Some("9999"));
3649 }
3650
3651 #[test]
3652 fn read_project_channel_config_handles_partial_channels() {
3653 let dir = tempfile::tempdir().expect("tempdir");
3654 let orch_yml = dir.path().join(".orch.yml");
3655 fs::write(&orch_yml, "channels:\n telegram:\n topic_id: \"7\"\n")
3656 .expect("write .orch.yml");
3657 let cfg = read_project_channel_config(dir.path());
3658 assert_eq!(cfg.telegram_topic_id.as_deref(), Some("7"));
3659 assert!(cfg.discord_channel_id.is_none());
3660 }
3661
3662 #[test]
3663 fn read_project_channel_config_handles_malformed_yaml() {
3664 let dir = tempfile::tempdir().expect("tempdir");
3665 let orch_yml = dir.path().join(".orch.yml");
3666 fs::write(&orch_yml, "channels: [\nbad yaml").expect("write .orch.yml");
3667 let cfg = read_project_channel_config(dir.path());
3668 assert!(cfg.telegram_topic_id.is_none());
3670 assert!(cfg.discord_channel_id.is_none());
3671 }
3672
3673 #[test]
3674 #[cfg(unix)]
3675 fn read_project_channel_config_handles_unreadable_file() {
3676 let dir = tempfile::tempdir().expect("tempdir");
3677 fs::create_dir(dir.path().join(".orch.yml")).expect("create .orch.yml dir");
3679 let cfg = read_project_channel_config(dir.path());
3680 assert!(cfg.telegram_topic_id.is_none());
3682 assert!(cfg.discord_channel_id.is_none());
3683 }
3684
3685 fn git(dir: &Path, args: &[&str]) {
3686 let mut cmd = process::Command::new("git");
3687 cmd.current_dir(dir);
3688 if args.first() == Some(&"commit") {
3689 cmd.args(["-c", "commit.gpgsign=false"]);
3690 }
3691 let out = cmd
3692 .args(args)
3693 .output()
3694 .unwrap_or_else(|e| panic!("git {args:?} failed to start: {e}"));
3695 assert!(
3696 out.status.success(),
3697 "git {:?} failed:\nstdout: {}\nstderr: {}",
3698 args,
3699 String::from_utf8_lossy(&out.stdout),
3700 String::from_utf8_lossy(&out.stderr)
3701 );
3702 }
3703
3704 fn restore_orch_home(prev: Option<String>) {
3705 match prev {
3706 Some(v) => unsafe { env::set_var("ORCH_HOME", v) },
3708 None => unsafe { env::remove_var("ORCH_HOME") },
3710 }
3711 }
3712
3713 struct StartupConflict {
3714 _tmp: tempfile::TempDir,
3715 _orch_home: tempfile::TempDir,
3716 prev_orch_home: Option<String>,
3717 project_dir: PathBuf,
3718 worktree_dir: PathBuf,
3719 branch: String,
3720 store: Arc<TaskStore>,
3721 store_id: i64,
3722 engine: ProjectEngine,
3723 }
3724
3725 impl Drop for StartupConflict {
3726 fn drop(&mut self) {
3727 restore_orch_home(self.prev_orch_home.take());
3728 }
3729 }
3730
3731 async fn startup_conflict_fixture(
3732 retries: i32,
3733 pr_number: Option<i32>,
3734 start_rebase: bool,
3735 ) -> StartupConflict {
3736 let orch_home = tempfile::tempdir().unwrap();
3737 let prev_orch_home = env::var("ORCH_HOME").ok();
3738 unsafe { env::set_var("ORCH_HOME", orch_home.path()) };
3740
3741 let tmp = tempfile::tempdir().unwrap();
3742 let origin = tmp.path().join("origin.git");
3743 fs::create_dir_all(&origin).unwrap();
3744 git(&origin, &["init", "--bare", "-b", "main"]);
3745 let project_dir = tmp.path().join("repo");
3746 git(
3747 tmp.path(),
3748 &[
3749 "clone",
3750 origin.to_str().unwrap(),
3751 project_dir.to_str().unwrap(),
3752 ],
3753 );
3754 git(&project_dir, &["config", "user.email", "t@t.com"]);
3755 git(&project_dir, &["config", "user.name", "T"]);
3756 fs::write(project_dir.join("f.txt"), "base\n").unwrap();
3757 git(&project_dir, &["add", "."]);
3758 git(&project_dir, &["commit", "-m", "base"]);
3759 git(&project_dir, &["push", "origin", "HEAD:main"]);
3760 git(&project_dir, &["remote", "set-head", "origin", "main"]);
3761
3762 let branch = "gh-issue-4087-conflict".to_string();
3763 git(&project_dir, &["checkout", "-b", &branch]);
3764 fs::write(project_dir.join("f.txt"), "agent\n").unwrap();
3765 git(&project_dir, &["commit", "-am", "agent"]);
3766 git(&project_dir, &["checkout", "main"]);
3767 fs::write(project_dir.join("f.txt"), "main\n").unwrap();
3768 git(&project_dir, &["commit", "-am", "main"]);
3769 git(&project_dir, &["push", "origin", "HEAD:main"]);
3770
3771 let wt_base = orch_home.path().join("worktrees").join("repo");
3772 fs::create_dir_all(&wt_base).unwrap();
3773 let worktree_dir = wt_base.join(&branch);
3774 git(
3775 &project_dir,
3776 &["worktree", "add", worktree_dir.to_str().unwrap(), &branch],
3777 );
3778 git(&worktree_dir, &["config", "user.email", "t@t.com"]);
3779 git(&worktree_dir, &["config", "user.name", "T"]);
3780 git(&worktree_dir, &["fetch", "origin"]);
3781 if start_rebase {
3782 let rebase = process::Command::new("git")
3783 .args(["rebase", "origin/main"])
3784 .current_dir(&worktree_dir)
3785 .output()
3786 .unwrap();
3787 assert!(!rebase.status.success(), "precondition: rebase conflicts");
3788 }
3789
3790 let store = Arc::new(TaskStore::open_memory().await.unwrap());
3791 let store_id = store
3792 .upsert_external(&UpsertExternal {
3793 repo: "owner/repo",
3794 ext_id: "4087",
3795 title: "Conflict",
3796 body: "",
3797 author: "",
3798 url: "",
3799 labels: &[],
3800 origin: "github",
3801 })
3802 .await
3803 .unwrap();
3804 store
3805 .update_status(store_id, TaskStatus::Routed)
3806 .await
3807 .unwrap();
3808 let mut fields = vec![
3809 ("branch", serde_json::json!(branch)),
3810 (
3811 "worktree",
3812 serde_json::json!(worktree_dir.to_string_lossy().as_ref()),
3813 ),
3814 ("merge_conflict_retries", serde_json::json!(retries)),
3815 ];
3816 if let Some(pr) = pr_number {
3817 fields.push(("pr_number", serde_json::json!(pr)));
3818 }
3819 store.set_fields(store_id, &fields).await.unwrap();
3820
3821 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
3822 let engine = ProjectEngine {
3823 repo: "owner/repo".to_string(),
3824 project_dir: project_dir.clone(),
3825 backend: Arc::clone(&backend),
3826 task_manager: Arc::new(TaskManager::with_store(
3827 backend,
3828 Arc::clone(&store),
3829 "owner/repo".to_string(),
3830 )),
3831 runner: Arc::new(
3832 runner::TaskRunner::new("owner/repo".to_string()).with_store(Arc::clone(&store)),
3833 ),
3834 store: Arc::clone(&store),
3835 };
3836
3837 StartupConflict {
3838 _tmp: tmp,
3839 _orch_home: orch_home,
3840 prev_orch_home,
3841 project_dir,
3842 worktree_dir,
3843 branch,
3844 store,
3845 store_id,
3846 engine,
3847 }
3848 }
3849
3850 #[tokio::test]
3851 #[serial_test::serial(orch_home)]
3852 async fn startup_keeps_a_content_conflict_when_the_task_retried_a_merge() {
3853 let fx = startup_conflict_fixture(2, Some(4043), false).await;
3854 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3855 .await
3856 .unwrap();
3857
3858 assert!(fx.worktree_dir.exists(), "worktree must stay");
3859 assert!(
3860 git_ops::rebase_in_progress(&fx.worktree_dir).await,
3861 "the content conflict must stay for the agent"
3862 );
3863 let task = fx.store.get(fx.store_id).await.unwrap();
3864 assert_eq!(task.status, TaskStatus::Routed);
3865 assert_eq!(task.branch, fx.branch);
3866 assert!(!task.worktree.is_empty());
3867 }
3868
3869 #[tokio::test]
3870 #[serial_test::serial(orch_home)]
3871 async fn startup_keeps_an_in_progress_rebase_when_the_task_has_a_pr() {
3872 let fx = startup_conflict_fixture(0, Some(12), true).await;
3873 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3874 .await
3875 .unwrap();
3876
3877 assert!(fx.worktree_dir.exists(), "worktree must stay");
3878 assert!(
3879 git_ops::rebase_in_progress(&fx.worktree_dir).await,
3880 "an in-progress rebase for a PR task must stay"
3881 );
3882 let task = fx.store.get(fx.store_id).await.unwrap();
3883 assert_eq!(task.status, TaskStatus::Routed);
3884 assert_eq!(task.branch, fx.branch);
3885 }
3886
3887 #[tokio::test]
3888 #[serial_test::serial(orch_home)]
3889 async fn startup_resets_a_conflict_without_retries_or_a_pr() {
3890 let fx = startup_conflict_fixture(0, None, false).await;
3891 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3892 .await
3893 .unwrap();
3894
3895 assert!(
3896 !fx.worktree_dir.exists(),
3897 "worktree is removed when there is nothing to keep"
3898 );
3899 let task = fx.store.get(fx.store_id).await.unwrap();
3900 assert_eq!(task.status, TaskStatus::New);
3901 }
3902
3903 #[tokio::test]
3904 #[serial_test::serial(orch_home)]
3905 async fn startup_reset_keeps_branch_when_rescue_cannot_push() {
3906 let fx = startup_conflict_fixture(0, None, false).await;
3907 git(
3908 &fx.project_dir,
3909 &["remote", "set-url", "origin", "/nonexistent/origin.git"],
3910 );
3911 reconcile_startup_worktrees(slice::from_ref(&fx.engine))
3912 .await
3913 .unwrap();
3914
3915 let task = fx.store.get(fx.store_id).await.unwrap();
3916 assert_eq!(task.status, TaskStatus::New);
3917 assert_eq!(
3918 task.branch, fx.branch,
3919 "the task still names the local branch cleanup kept"
3920 );
3921 assert!(
3922 fx.worktree_dir.exists(),
3923 "rescue failure keeps the worktree on disk"
3924 );
3925 }
3926}