1pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38 finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39 PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46 ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47 StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51 MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55 MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71const HELLO_CORR: u64 = 1;
73
74const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106 MaintenanceDrainKind::Watcher,
107 MaintenanceDrainKind::Lsp,
108 MaintenanceDrainKind::ConfigureTail,
109 MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135 channel: u16,
136 epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 write!(f, "{}@{}", self.channel, self.epoch)
142 }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156 build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157 DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158 ReapBlockerCensus, ResponseTaskGuard,
159};
160pub(crate) use self::manifest::is_subc_native_plumbing_tool;
161use self::manifest::{
162 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163 is_subc_agent_core_tool,
164};
165pub use self::wire::SubcError;
166
167#[doc(hidden)]
172#[derive(Clone, Debug, PartialEq, Eq)]
173pub enum SubcLifecycleEvent {
174 AttachDecision {
175 attempt: u32,
176 will_retry: bool,
177 },
178 RouteDetached {
179 route_channel: u16,
180 route_epoch: u32,
181 session_id: String,
182 },
183 ReliableCompletionRetained {
184 task_id: String,
185 session_id: String,
186 },
187 ReliableCompletionReplayed {
188 route_channel: u16,
189 route_epoch: u32,
190 task_id: String,
191 session_id: String,
192 },
193}
194
195#[doc(hidden)]
197#[derive(Clone)]
198pub struct SubcTestLifecycleProbe {
199 events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
200}
201
202impl SubcTestLifecycleProbe {
203 #[doc(hidden)]
204 pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
205 Self { events_tx }
206 }
207
208 fn attach_decision(&self, attempt: u32, will_retry: bool) {
209 let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
210 attempt,
211 will_retry,
212 });
213 }
214
215 fn route_detached(&self, route: RouteChannel, session_id: &str) {
216 let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
217 route_channel: route.channel,
218 route_epoch: route.epoch,
219 session_id: session_id.to_string(),
220 });
221 }
222
223 fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
224 let _ = self
225 .events_tx
226 .send(SubcLifecycleEvent::ReliableCompletionRetained {
227 task_id: task_id.to_string(),
228 session_id: session_id.to_string(),
229 });
230 }
231
232 fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
233 let _ = self
234 .events_tx
235 .send(SubcLifecycleEvent::ReliableCompletionReplayed {
236 route_channel: route.channel,
237 route_epoch: route.epoch,
238 task_id: task_id.to_string(),
239 session_id: session_id.to_string(),
240 });
241 }
242}
243
244pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
248 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
249}
250use self::wire::{
251 build_error_frame, build_goodbye_frame, build_tool_response_frame,
252 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
253 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
254 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
255};
256
257struct DecodedFrame {
258 frame: Frame,
259 phase_trace: PhaseTrace,
260}
261
262struct ToolCallCompletion {
263 text: String,
264 phase_trace: PhaseTrace,
265}
266
267#[derive(Clone, Copy, Debug, PartialEq, Eq)]
268enum RouteDetachPolicy {
269 RetainForReplay,
270 CancelOnDetach,
271}
272
273#[derive(Clone)]
274struct ActiveToolCall {
275 root_id: ProjectRootId,
276 cancellation: JobCancellation,
277 detach_policy: RouteDetachPolicy,
278}
279
280type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
281
282struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
283
284impl PendingDeferredSetupGuard {
285 fn new(count: Arc<AtomicUsize>) -> Self {
286 count.fetch_add(1, Ordering::SeqCst);
287 Self(count)
288 }
289}
290
291impl Drop for PendingDeferredSetupGuard {
292 fn drop(&mut self) {
293 self.0.fetch_sub(1, Ordering::SeqCst);
294 }
295}
296
297enum DeferredSetupOutcome {
298 Immediate {
299 text: String,
300 phase_trace: PhaseTrace,
301 },
302 Deferred {
303 pending: PendingResponse,
304 surface_downgraded: bool,
305 phase_trace: PhaseTrace,
306 },
307}
308
309struct PendingSubcResponse {
310 route: RouteChannel,
311 corr: u64,
312 flags: Flags,
313 ver: u8,
314 root: ProjectRootId,
315 session_id: String,
316 bare_name: String,
317 format_context: crate::subc_format::FormatContext,
318 bind_trust: BindTrust,
319 pending: PendingResponse,
320 surface_downgraded: bool,
321 phase_trace: PhaseTrace,
322}
323
324struct ResolvedSubcResponse {
325 entry: PendingSubcResponse,
326 response: Response,
327}
328
329#[derive(Default)]
330struct PendingSubcResponses {
331 entries: Vec<PendingSubcResponse>,
332}
333
334impl PendingSubcResponses {
335 fn register(&mut self, pending: PendingSubcResponse) {
336 self.entries.retain(|entry| {
337 let keep = entry.route != pending.route || entry.corr != pending.corr;
338 if !keep {
339 if let Some(cancellation) = &entry.pending.cancellation {
340 cancellation.request_cancel();
341 }
342 }
343 keep
344 });
345 self.entries.push(pending);
346 }
347
348 fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
349 let mut ready = Vec::new();
350 let mut waiting = Vec::with_capacity(self.entries.len());
351 for mut entry in self.entries.drain(..) {
352 let response = executor
353 .actor_context(&entry.root)
354 .and_then(|ctx| (entry.pending.poll)(&ctx));
355 if let Some(response) = response {
356 ready.push(ResolvedSubcResponse { entry, response });
357 } else {
358 waiting.push(entry);
359 }
360 }
361 self.entries = waiting;
362 ready
363 }
364
365 fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
366 let mut cancelled = false;
367 self.entries.retain(|entry| {
368 let keep = entry.route != route || entry.corr != corr;
369 if !keep {
370 cancelled = true;
371 if let Some(cancellation) = &entry.pending.cancellation {
372 cancellation.request_cancel();
373 }
374 }
375 keep
376 });
377 cancelled
378 }
379
380 fn drain_route(
381 &mut self,
382 route: RouteChannel,
383 executor: &Executor,
384 ) -> Vec<ResolvedSubcResponse> {
385 self.drain_matching(executor, |entry| entry.route == route)
386 }
387
388 fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
389 self.drain_matching(executor, |_| true)
390 }
391
392 fn drain_matching(
393 &mut self,
394 executor: &Executor,
395 matches: impl Fn(&PendingSubcResponse) -> bool,
396 ) -> Vec<ResolvedSubcResponse> {
397 let mut resolved = Vec::new();
398 let mut waiting = Vec::with_capacity(self.entries.len());
399 for mut entry in self.entries.drain(..) {
400 if !matches(&entry) {
401 waiting.push(entry);
402 continue;
403 }
404 if let Some(cancellation) = &entry.pending.cancellation {
405 cancellation.request_cancel();
406 }
407 if let Some(ctx) = executor.actor_context(&entry.root) {
408 if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
409 let response = on_shutdown(&ctx);
410 resolved.push(ResolvedSubcResponse { entry, response });
411 }
412 }
413 }
414 self.entries = waiting;
415 resolved
416 }
417
418 fn is_empty(&self) -> bool {
419 self.entries.is_empty()
420 }
421}
422
423#[derive(Clone)]
424struct PushSenders {
425 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
426 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
427 lossy_overflow: Arc<push::LossyOverflow>,
428 lossy_seq: Arc<AtomicU64>,
429 fleet_status_client: FleetStatusClient,
430}
431
432#[derive(Clone)]
433struct PersistentCancelSignal {
434 inner: Arc<PersistentCancelInner>,
435}
436
437struct PersistentCancelInner {
438 cancelled: AtomicBool,
439 notify: Notify,
440}
441
442impl PersistentCancelSignal {
443 fn new() -> Self {
444 Self {
445 inner: Arc::new(PersistentCancelInner {
446 cancelled: AtomicBool::new(false),
447 notify: Notify::new(),
448 }),
449 }
450 }
451
452 fn cancel(&self) {
453 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
454 self.inner.notify.notify_waiters();
455 }
456 }
457
458 fn is_cancelled(&self) -> bool {
459 self.inner.cancelled.load(Ordering::SeqCst)
460 }
461
462 async fn cancelled(&self) {
463 loop {
471 let notified = self.inner.notify.notified();
472 tokio::pin!(notified);
473 notified.as_mut().enable();
474 if self.is_cancelled() {
475 return;
476 }
477 notified.await;
478 }
479 }
480}
481
482fn submit_active_tool_call(
483 executor: &Executor,
484 active: &ActiveToolCalls,
485 route: RouteChannel,
486 corr: u64,
487 root_id: ProjectRootId,
488 lane: Lane,
489 request_id: String,
490 detach_policy: RouteDetachPolicy,
491 job: crate::executor::ExecutorJob,
492) -> oneshot::Receiver<Response> {
493 let (rx, cancellation) =
494 executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
495 active
496 .lock()
497 .unwrap_or_else(std::sync::PoisonError::into_inner)
498 .insert(
499 (route, corr),
500 ActiveToolCall {
501 root_id,
502 cancellation,
503 detach_policy,
504 },
505 );
506 rx
507}
508
509fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
510 active
511 .lock()
512 .unwrap_or_else(std::sync::PoisonError::into_inner)
513 .remove(&(route, corr))
514 .is_some()
515}
516
517fn active_tool_call_is_registered(
518 active: &ActiveToolCalls,
519 route: RouteChannel,
520 corr: u64,
521) -> bool {
522 active
523 .lock()
524 .unwrap_or_else(std::sync::PoisonError::into_inner)
525 .contains_key(&(route, corr))
526}
527
528fn cancel_active_tool_call(
529 active: &ActiveToolCalls,
530 executor: &Executor,
531 route: RouteChannel,
532 corr: u64,
533 reason: &str,
534) -> bool {
535 let call = active
536 .lock()
537 .unwrap_or_else(std::sync::PoisonError::into_inner)
538 .remove(&(route, corr));
539 let Some(call) = call else {
540 return false;
541 };
542 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
543 log::debug!(
544 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
545 );
546 true
547}
548
549#[derive(Clone, Copy, Debug, PartialEq, Eq)]
550enum RouteWorkDisposition {
551 RetainForReplay,
552 RetainStartedForReplay,
553 Abandon,
554}
555
556fn apply_route_work_disposition(
557 active: &ActiveToolCalls,
558 executor: &Executor,
559 route: RouteChannel,
560 disposition: RouteWorkDisposition,
561 reason: &str,
562) -> usize {
563 if disposition != RouteWorkDisposition::Abandon {
564 let route_calls = active
565 .lock()
566 .unwrap_or_else(std::sync::PoisonError::into_inner)
567 .iter()
568 .filter(|((call_route, _), _)| *call_route == route)
569 .map(|(key, call)| (*key, call.clone()))
570 .collect::<Vec<_>>();
571 let mut retained = 0usize;
572 let mut cancelled_before_execution = 0usize;
573 let mut cancelled_terminal = 0usize;
574
575 for (key, call) in route_calls {
576 let remove = match (call.detach_policy, disposition) {
577 (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
578 retained += 1;
579 false
580 }
581 (
582 RouteDetachPolicy::RetainForReplay,
583 RouteWorkDisposition::RetainStartedForReplay,
584 ) => {
585 match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
586 PreExecutionCancelOutcome::AlreadyStarted => {
587 retained += 1;
588 false
589 }
590 PreExecutionCancelOutcome::QueuedRemoved
591 | PreExecutionCancelOutcome::DispatchedCancelled => {
592 cancelled_before_execution += 1;
593 true
594 }
595 }
596 }
597 (RouteDetachPolicy::CancelOnDetach, _) => {
598 executor.cancel_job(&call.root_id, &call.cancellation);
599 cancelled_terminal += 1;
600 true
601 }
602 (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
603 };
604 if remove {
605 active
606 .lock()
607 .unwrap_or_else(std::sync::PoisonError::into_inner)
608 .remove(&key);
609 }
610 }
611 log::debug!(
612 "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
613 );
614 return retained;
615 }
616
617 let cancelled = {
618 let mut calls = active
619 .lock()
620 .unwrap_or_else(std::sync::PoisonError::into_inner);
621 let mut cancelled = Vec::new();
622 calls.retain(|(call_route, _), call| {
623 if *call_route == route {
624 cancelled.push(call.clone());
625 false
626 } else {
627 true
628 }
629 });
630 cancelled
631 };
632 for call in &cancelled {
633 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
634 log::debug!(
635 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
636 );
637 }
638 cancelled.len()
639}
640
641fn cancel_all_active_tool_calls(
642 active: &ActiveToolCalls,
643 executor: &Executor,
644 reason: &str,
645) -> usize {
646 let cancelled = {
647 let mut calls = active
648 .lock()
649 .unwrap_or_else(std::sync::PoisonError::into_inner);
650 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
651 };
652 for call in &cancelled {
653 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
654 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
655 }
656 cancelled.len()
657}
658
659#[derive(Debug, Clone, Copy, PartialEq, Eq)]
660pub enum BindTrust {
661 FirstParty,
662 Untrusted,
663}
664
665impl BindTrust {
666 fn allows_bash_observation(self) -> bool {
667 matches!(self, Self::FirstParty)
668 }
669
670 fn label(self) -> &'static str {
671 match self {
672 Self::FirstParty => "first_party",
673 Self::Untrusted => "untrusted",
674 }
675 }
676
677 fn sandbox_trust(self) -> PrincipalTrust {
678 match self {
679 Self::FirstParty => PrincipalTrust::FirstParty,
680 Self::Untrusted => PrincipalTrust::Untrusted,
681 }
682 }
683}
684
685pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
686 match principal {
687 Some(Principal::Direct) => BindTrust::FirstParty,
688 Some(Principal::Reserved { module_id })
699 if module_id == "llm-runner"
700 || module_id == "aft"
701 || module_id == "broca"
702 || module_id == "alfonso-core"
703 || module_id == "prefrontal"
704 || module_id == "prefrontal-core" =>
705 {
706 BindTrust::FirstParty
707 }
708 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
709 BindTrust::Untrusted
710 }
711 }
712}
713
714fn harness_forces_untrusted(harness: &str) -> bool {
715 harness.starts_with("fed:")
716}
717
718pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
719 if harness_forces_untrusted(harness) {
720 BindTrust::Untrusted
721 } else {
722 trust_for_principal(principal)
723 }
724}
725
726fn principal_id(principal: &Option<Principal>) -> Option<String> {
727 match principal {
728 Some(Principal::Direct) => Some("direct".to_string()),
729 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
730 Some(Principal::Unverified) => Some("unverified".to_string()),
731 None => None,
732 }
733}
734
735fn principal_label(principal: &Option<Principal>) -> String {
736 principal_id(principal).unwrap_or_else(|| "absent".to_string())
737}
738
739#[derive(Debug)]
740struct RootMeta {
746 maintenance_pending: bool,
747 maintenance_jobs_in_flight: usize,
748 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
749 maintenance_last_submitted: Option<Instant>,
750 maintenance_poisoned: bool,
751 last_touched: Instant,
752 diagnostics_on_edit: bool,
753 active_bash_waits: usize,
754 idle_artifacts_evicted: bool,
755 unbound_quiesced: bool,
756 consecutive_missing_sweeps: u8,
757}
758
759#[derive(Debug)]
760struct PendingBind {
761 bind_root_id: ProjectRootId,
762 inserted_new_actor: bool,
763 cancelled: bool,
764 configure_request_id: String,
765 started_at: Instant,
766 warned_half_deadline: bool,
767 deadline_reported: bool,
768 corr: u64,
769 ver: u8,
770 flags: Flags,
771 cancellation: crate::executor::JobCancellation,
776}
777
778struct RouteBindCompletion {
779 route: RouteChannel,
780 identity: RouteIdentity,
781 bind_root_id: ProjectRootId,
782 inserted_new_actor: bool,
783 configure_response: Response,
784 diagnostics_on_edit: bool,
785 ver: u8,
786 corr: u64,
787 flags: Flags,
788}
789
790#[derive(Debug, Clone)]
791struct RouteIdentity(Arc<RouteIdentityData>);
792
793#[derive(Debug)]
794struct RouteIdentityData {
795 root: ProjectRootId,
796 project_root: PathBuf,
797 harness: String,
798 session: String,
799 trust: BindTrust,
800 spawn_principal: AuthenticatedPrincipal,
801 consumer_elicitation_capable: bool,
802}
803
804impl Deref for RouteIdentity {
805 type Target = RouteIdentityData;
806
807 fn deref(&self) -> &Self::Target {
808 &self.0
809 }
810}
811
812#[derive(Debug, Clone)]
813struct RetainedSessionIdentity {
814 harness: String,
815 trust: BindTrust,
816}
817
818#[derive(Clone)]
819struct BgSub {
820 corr: u64,
821 ver: u8,
822 flags: Flags,
823 root: ProjectRootId,
824 session: String,
825}
826
827#[derive(Clone, Copy, Debug)]
828struct BgWakeState {
829 next_nudge_at: Instant,
830 nudges_sent: u32,
831}
832
833impl BgWakeState {
834 fn armed(now: Instant) -> Self {
835 Self {
836 next_nudge_at: now,
837 nudges_sent: 0,
838 }
839 }
840}
841
842type BgWakePending = HashMap<RouteChannel, BgWakeState>;
843
844type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
847
848struct MaintenanceCompletion {
849 root_id: ProjectRootId,
850 kind: MaintenanceDrainKind,
851 response: Response,
852 empty_bg_sessions: Vec<(String, u64)>,
853 unacked_bg_keys: Option<HashSet<String>>,
854 requeue_kind: Option<MaintenanceDrainKind>,
855}
856
857#[derive(Clone, Copy, Debug, PartialEq, Eq)]
858enum MaintenanceDrainKind {
859 Watcher,
860 Lsp,
861 ConfigureTail,
862 CompletionDrains,
863}
864
865impl MaintenanceDrainKind {
866 fn label(self) -> &'static str {
867 match self {
868 Self::Watcher => "watcher",
869 Self::Lsp => "lsp",
870 Self::ConfigureTail => "configure-tail",
871 Self::CompletionDrains => "completion-drains",
872 }
873 }
874}
875
876#[derive(Debug, Default)]
877struct MaintenanceJobOutcome {
878 empty_bg_sessions: Vec<(String, u64)>,
879 unacked_bg_keys: Option<HashSet<String>>,
880 requeue_kind: Option<MaintenanceDrainKind>,
881}
882
883#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
884struct ReverseCorrKey {
885 route: RouteChannel,
886 corr: u64,
887}
888
889struct PendingBashAsk {
890 route: RouteChannel,
891 tool_corr: u64,
892 tool_flags: Flags,
893 tool_ver: u8,
894 root: ProjectRootId,
895 project_root: PathBuf,
896 session_id: String,
897 spawn_principal: AuthenticatedPrincipal,
898 edit_slot_survives: Option<bool>,
899 request_id: String,
900 arguments: Value,
901 format_context: crate::subc_format::FormatContext,
902 cancel: bash::BashWaitCancel,
903 grants: Vec<String>,
904 expires_at: Instant,
905}
906
907impl RootMeta {
908 fn new(now: Instant) -> Self {
909 Self {
910 maintenance_pending: false,
911 maintenance_jobs_in_flight: 0,
912 maintenance_queued_kinds: VecDeque::new(),
913 maintenance_last_submitted: None,
914 maintenance_poisoned: false,
915 last_touched: now,
916 diagnostics_on_edit: false,
917 active_bash_waits: 0,
918 idle_artifacts_evicted: false,
919 unbound_quiesced: false,
920 consecutive_missing_sweeps: 0,
921 }
922 }
923
924 fn note_activity(&mut self) {
925 self.last_touched = Instant::now();
926 }
927
928 fn reactivate_bound(&mut self) {
929 self.note_activity();
930 self.idle_artifacts_evicted = false;
931 self.unbound_quiesced = false;
932 }
933}
934
935fn due_maintenance_jobs(
936 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
937 executor: Option<&Executor>,
938 bg_sub_by_session: &BgSubsBySession,
939 bg_wake_pending: &BgWakePending,
940 budget: usize,
941 pending_bind_roots: &HashSet<ProjectRootId>,
942) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
943 let mut jobs = Vec::new();
944 let mut deferred = false;
945 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
946 roots.sort_by(|left, right| {
947 let left_last = live_roots
948 .get(left)
949 .and_then(|meta| meta.maintenance_last_submitted);
950 let right_last = live_roots
951 .get(right)
952 .and_then(|meta| meta.maintenance_last_submitted);
953 left_last
954 .cmp(&right_last)
955 .then_with(|| left.as_path().cmp(right.as_path()))
956 });
957
958 for root_id in roots {
959 let Some(meta) = live_roots.get_mut(&root_id) else {
960 continue;
961 };
962 if meta.maintenance_poisoned {
963 continue;
964 }
965
966 if pending_bind_roots.contains(&root_id) {
967 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
968 deferred = true;
969 }
970 continue;
971 }
972
973 if !meta.maintenance_pending {
974 if jobs.len() >= budget {
975 deferred = true;
976 continue;
977 }
978 let executor_actor_context =
982 executor.and_then(|executor| executor.actor_context(&root_id));
983 let root_has_pending_bg_wake =
984 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
985 sub_root == &root_id
986 && channels
987 .iter()
988 .any(|channel| bg_wake_pending.contains_key(channel))
989 });
990 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
991 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
992 .into_iter()
993 .filter(|kind| {
994 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
995 return false;
996 }
997 match kind {
998 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
999 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
1000 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1001 MaintenanceDrainKind::CompletionDrains => {
1007 root_has_pending_bg_wake || ctx.completion_drains_have_work()
1008 }
1009 }
1010 })
1011 .collect(),
1012 None if meta.unbound_quiesced => Vec::new(),
1013 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1015 };
1016 if kinds_with_work.is_empty() {
1017 continue;
1018 }
1019 meta.maintenance_pending = true;
1020 meta.maintenance_queued_kinds.extend(kinds_with_work);
1021 }
1022
1023 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1024 if jobs.len() >= budget {
1025 meta.maintenance_queued_kinds.push_front(kind);
1026 deferred = true;
1027 break;
1028 }
1029 meta.maintenance_jobs_in_flight += 1;
1030 meta.maintenance_last_submitted = Some(Instant::now());
1031 jobs.push((root_id.clone(), kind));
1032 }
1033
1034 meta.maintenance_pending =
1035 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1036 }
1037
1038 (jobs, deferred)
1039}
1040
1041fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1042 match estimate.estimated_bytes {
1043 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1044 None if estimate.status == "busy" => "busy".to_string(),
1045 None => "not estimated".to_string(),
1046 }
1047}
1048
1049fn optional_memory_label(bytes: Option<u64>) -> String {
1050 bytes.map_or_else(
1051 || "not estimated".to_string(),
1052 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1053 )
1054}
1055
1056fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1057 format!(
1058 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1059 optional_memory_label(relief.rss_before_bytes),
1060 optional_memory_label(relief.rss_after_bytes),
1061 optional_memory_label(relief.allocator_before.bytes_in_use),
1062 optional_memory_label(relief.allocator_after.bytes_in_use),
1063 optional_memory_label(relief.allocator_before.size_allocated),
1064 optional_memory_label(relief.allocator_after.size_allocated),
1065 optional_memory_label(relief.allocator_before.retained_slack_bytes),
1066 optional_memory_label(relief.allocator_after.retained_slack_bytes),
1067 relief.bytes_released as f64 / (1024.0 * 1024.0),
1068 )
1069}
1070
1071fn idle_root_eviction_message(
1072 root_id: &ProjectRootId,
1073 memory: &crate::memory::RootMemorySnapshot,
1074 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1075) -> String {
1076 let freed_bytes = [
1079 &memory.semantic,
1080 &memory.trigram,
1081 &memory.symbols,
1082 &memory.callgraph,
1083 &memory.inspect,
1084 ]
1085 .iter()
1086 .filter_map(|estimate| estimate.estimated_bytes)
1087 .fold(0u64, u64::saturating_add);
1088 let mut message = format!(
1089 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1090 root_id.as_path().display(),
1091 freed_bytes as f64 / (1024.0 * 1024.0),
1092 eviction_estimate_label(&memory.semantic),
1093 eviction_estimate_label(&memory.trigram),
1094 eviction_estimate_label(&memory.symbols),
1095 eviction_estimate_label(&memory.callgraph),
1096 eviction_estimate_label(&memory.inspect),
1097 eviction_estimate_label(&memory.bash),
1098 eviction_estimate_label(&memory.lsp),
1099 eviction_estimate_label(&memory.parser_pool),
1100 );
1101 if let Some(pressure_relief) = pressure_relief {
1102 message.push_str(&pressure_relief_label(pressure_relief));
1103 }
1104 message
1105}
1106
1107fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1108 executor
1109 .actor_context(root_id)
1110 .map(|ctx| ctx.config().idle.root_ttl())
1111 .unwrap_or(IDLE_ROOT_TTL)
1112}
1113
1114fn process_has_been_idle(
1115 now: Instant,
1116 live_roots: &HashMap<ProjectRootId, RootMeta>,
1117 executor: &Executor,
1118) -> bool {
1119 !live_roots.is_empty()
1120 && live_roots.iter().all(|(root_id, meta)| {
1121 now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1122 && meta.active_bash_waits == 0
1123 && !meta.maintenance_pending
1124 && meta.maintenance_queued_kinds.is_empty()
1125 })
1126}
1127
1128fn allocator_pressure_relief_after_idle_sweep(
1129 now: Instant,
1130 live_roots: &HashMap<ProjectRootId, RootMeta>,
1131 executor: &Executor,
1132) -> Option<crate::memory::AllocatorPressureRelief> {
1133 if !process_has_been_idle(now, live_roots, executor)
1134 || live_roots.keys().any(|root_id| {
1135 executor
1136 .actor_context(root_id)
1137 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1138 })
1139 {
1140 return None;
1141 }
1142
1143 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1144 {
1145 Some(crate::memory::relieve_allocator_pressure())
1146 }
1147 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1148 {
1149 None
1150 }
1151}
1152
1153fn quiesce_unbound_root(
1154 root_id: &ProjectRootId,
1155 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1156 executor: &Arc<Executor>,
1157) {
1158 let Some(meta) = live_roots.get_mut(root_id) else {
1159 return;
1160 };
1161
1162 let ctx = executor.actor_context(root_id);
1163 if let Some(ctx) = ctx.as_ref() {
1164 ctx.mark_subc_unbound();
1168 ctx.bash_background()
1169 .replace_live_delivery_sessions(HashSet::new());
1170 }
1171 let cancelled = executor.cancel_queued_maintenance(root_id);
1172 let discarded = ctx
1180 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1181 .unwrap_or(0);
1182 meta.unbound_quiesced = true;
1183 meta.maintenance_queued_kinds.clear();
1184 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1185 log::info!(
1186 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1187 root_id.as_path().display(),
1188 cancelled,
1189 discarded
1190 );
1191}
1192
1193#[allow(clippy::too_many_arguments)]
1194fn quiesce_connection_roots(
1195 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1196 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1197 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1198 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1199 installed_route_epochs: &mut HashMap<u16, u32>,
1200 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1201 active_tool_calls: &ActiveToolCalls,
1202 executor: &Arc<Executor>,
1203) {
1204 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1205 for cancel in route_bash_cancels.values() {
1206 cancel.token.cancel();
1207 }
1208 route_bash_cancels.clear();
1209
1210 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1211 for pending in pending_binds.values_mut() {
1212 pending.cancelled = true;
1213 roots.insert(pending.bind_root_id.clone());
1214 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1215 }
1216
1217 for root_id in roots {
1221 if live_roots.contains_key(&root_id) {
1222 quiesce_unbound_root(&root_id, live_roots, executor);
1223 } else if let Some(ctx) = executor.actor_context(&root_id) {
1224 ctx.mark_subc_unbound();
1225 executor.cancel_queued_maintenance(&root_id);
1226 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1227 }
1228 }
1229
1230 routes.clear();
1231 root_channels.clear();
1232 installed_route_epochs.clear();
1233}
1234
1235#[derive(Debug, Default)]
1239struct ReclaimedRoutes {
1240 highest_epoch_by_channel: HashMap<u16, u32>,
1241}
1242
1243impl ReclaimedRoutes {
1244 fn insert(&mut self, route: RouteChannel) {
1245 self.highest_epoch_by_channel
1246 .entry(route.channel)
1247 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1248 .or_insert(route.epoch);
1249 }
1250
1251 fn contains(&self, route: RouteChannel) -> bool {
1252 self.highest_epoch_by_channel
1253 .get(&route.channel)
1254 .is_some_and(|epoch| route.epoch <= *epoch)
1255 }
1256}
1257
1258#[derive(Debug, Default)]
1259struct IdleReapOutcome {
1260 evicted: usize,
1261 forgotten_deleted_roots: Vec<ProjectRootId>,
1262}
1263
1264fn reap_idle_roots(
1265 now: Instant,
1266 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1267 pending_binds: &HashMap<RouteChannel, PendingBind>,
1268 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1269 executor: &Arc<Executor>,
1270 metrics: &DispatchPathMetrics,
1271) -> IdleReapOutcome {
1272 let pending_bind_roots = pending_binds
1273 .values()
1274 .map(|pending| pending.bind_root_id.clone())
1275 .collect::<HashSet<_>>();
1276 let mut census = ReapBlockerCensus::default();
1277 let mut candidates = Vec::new();
1278
1279 for (root_id, meta) in live_roots.iter_mut() {
1280 let deleted = !root_id.as_path().exists();
1281 if deleted {
1282 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1292 } else {
1293 meta.consecutive_missing_sweeps = 0;
1294 }
1295 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1296 let has_bound_route = root_channels
1297 .get(root_id)
1298 .is_some_and(|channels| !channels.is_empty());
1299 let has_pending_bind = pending_bind_roots.contains(root_id);
1300
1301 if deleted {
1302 let mut retained = false;
1303 if !deletion_confirmed {
1304 census.absence_unconfirmed += 1;
1305 retained = true;
1306 }
1307 if meta.active_bash_waits > 0 {
1311 census.bash_waits += 1;
1312 retained = true;
1313 }
1314 if meta.maintenance_pending {
1315 census.maintenance_pending += 1;
1316 retained = true;
1317 }
1318 if !meta.maintenance_queued_kinds.is_empty() {
1319 census.maintenance_queued += 1;
1320 retained = true;
1321 }
1322 if has_pending_bind {
1323 census.pending_binds += 1;
1324 retained = true;
1325 }
1326 match executor.try_actor_is_idle(root_id) {
1327 Some(true) => {}
1328 Some(false) => {
1329 census.actor_busy += 1;
1330 retained = true;
1331 }
1332 None => {
1333 census.actor_state_busy += 1;
1334 retained = true;
1335 }
1336 }
1337 if retained {
1338 census.deleted_retained += 1;
1339 continue;
1340 }
1341 } else {
1342 if has_bound_route
1346 || !meta.unbound_quiesced
1347 || meta.idle_artifacts_evicted
1348 || now.saturating_duration_since(meta.last_touched)
1349 < root_idle_ttl(executor, root_id)
1350 || meta.active_bash_waits > 0
1351 || meta.maintenance_pending
1352 || !meta.maintenance_queued_kinds.is_empty()
1353 || has_pending_bind
1354 || !executor.actor_is_idle(root_id)
1355 {
1356 continue;
1357 }
1358 }
1359 candidates.push((root_id.clone(), deleted));
1360 }
1361
1362 let mut reaped = Vec::new();
1363 let mut forgotten_deleted_roots = Vec::new();
1364 for (root_id, deleted) in candidates {
1365 let Some(ctx) = executor.actor_context(&root_id) else {
1366 if deleted {
1367 census.deleted_retained += 1;
1368 census.actor_busy += 1;
1369 }
1370 continue;
1371 };
1372 if deleted {
1383 ctx.bash_background()
1384 .kill_running_tasks_for_root(root_id.as_path());
1385 }
1386 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1387 if ctx.artifact_eviction_blocked() {
1388 if let Some(pending) = taken_pending {
1389 ctx.restore_pending_reconciliation_state(pending);
1390 }
1391 if deleted {
1392 census.deleted_retained += 1;
1393 census.artifact_eviction_blocked += 1;
1394 }
1395 continue;
1396 }
1397 let memory_before = ctx.memory_root_snapshot();
1398 if !ctx.evict_idle_artifacts() {
1399 if let Some(pending) = taken_pending {
1400 ctx.restore_pending_reconciliation_state(pending);
1401 }
1402 if deleted {
1403 census.deleted_retained += 1;
1404 census.artifact_eviction_failed += 1;
1405 }
1406 continue;
1407 }
1408 drop(taken_pending);
1409 ctx.stop_watcher_runtime_in_background();
1410 ctx.invalidate_artifacts_after_watcher_gap();
1413
1414 if deleted {
1415 if executor.retire_idle_actor_in_background(&root_id) {
1416 live_roots.remove(&root_id);
1417 forgotten_deleted_roots.push(root_id.clone());
1418 } else {
1419 census.deleted_retained += 1;
1420 census.actor_busy += 1;
1421 }
1422 } else {
1423 if let Some(meta) = live_roots.get_mut(&root_id) {
1424 meta.idle_artifacts_evicted = true;
1425 }
1426 ctx.release_idle_reopenable_resources_in_background();
1427 }
1428 reaped.push((root_id, memory_before));
1429 }
1430
1431 let census_changed = metrics.record_reap(census);
1434 if census_changed {
1435 log::info!(
1436 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1437 census.deleted_retained,
1438 census.blocker_histogram()
1439 );
1440 }
1441
1442 let pressure_relief = (!reaped.is_empty())
1443 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1444 .flatten();
1445 for (root_id, memory_before) in &reaped {
1446 log::info!(
1447 "{}",
1448 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1449 );
1450 }
1451 IdleReapOutcome {
1452 evicted: reaped.len(),
1453 forgotten_deleted_roots,
1454 }
1455}
1456
1457fn reap_idle_lsp_servers(
1465 now: Instant,
1466 live_roots: &HashMap<ProjectRootId, RootMeta>,
1467 executor: &Executor,
1468) {
1469 for (root_id, meta) in live_roots {
1470 let Some(ctx) = executor.actor_context(root_id) else {
1471 continue;
1472 };
1473 crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1474 }
1475}
1476
1477#[allow(clippy::too_many_arguments)]
1478fn purge_deleted_root_residents(
1479 root_id: &ProjectRootId,
1480 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1481 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1482 installed_route_epochs: &mut HashMap<u16, u32>,
1483 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1484 active_tool_calls: &ActiveToolCalls,
1485 executor: &Executor,
1486 retry_buffer: &mut RetryBuffer,
1487 reclaimed_routes: &mut ReclaimedRoutes,
1488 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1489 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1490 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1491 bg_sub_by_session: &mut BgSubsBySession,
1492 bg_wake_pending: &mut BgWakePending,
1493 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1494 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1495 metrics: &DispatchPathMetrics,
1496) {
1497 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1498 stale_routes.extend(
1499 routes
1500 .iter()
1501 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1502 );
1503 stale_routes.extend(
1504 bg_sub_by_session
1505 .iter()
1506 .filter(|((root, _), _)| root == root_id)
1507 .flat_map(|(_, routes)| routes.iter().copied()),
1508 );
1509 stale_routes.extend(
1510 pending_bash_asks
1511 .values()
1512 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1513 );
1514
1515 for route in stale_routes {
1516 reclaimed_routes.insert(route);
1517 remove_installed_route(installed_route_epochs, route);
1518 remove_route_channel(routes, root_channels, route);
1519 if let Some(cancel) = route_bash_cancels.remove(&route) {
1520 cancel.token.cancel();
1521 }
1522 apply_route_work_disposition(
1523 active_tool_calls,
1524 executor,
1525 route,
1526 RouteWorkDisposition::Abandon,
1527 "root reclaim",
1528 );
1529 retry_buffer.remove(&route);
1530 if let Some(sub) = bg_subs.remove(&route) {
1531 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1532 }
1533 bg_wake_pending.remove(&route);
1534 }
1535 root_channels.remove(root_id);
1536 session_identity.retain(|(root, _), _| root != root_id);
1537 push_buffer.retain(|key, _| &key.root != root_id);
1538 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1539 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1540 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1541 sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1542
1543 log::info!(
1544 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1545 root_id.as_path().display()
1546 );
1547}
1548
1549#[allow(clippy::too_many_arguments)]
1550fn submit_due_maintenance_jobs(
1551 executor: &Arc<Executor>,
1552 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1553 pending_binds: &HashMap<RouteChannel, PendingBind>,
1554 bg_sub_by_session: &BgSubsBySession,
1555 bg_wake_pending: &BgWakePending,
1556 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1557 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1558 metrics: &Arc<DispatchPathMetrics>,
1559) {
1560 let pending_bind_roots = pending_binds
1561 .values()
1562 .map(|pending| pending.bind_root_id.clone())
1563 .collect::<HashSet<_>>();
1564 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1565 live_roots,
1566 Some(executor),
1567 bg_sub_by_session,
1568 bg_wake_pending,
1569 MAINTENANCE_SUBMIT_BUDGET,
1570 &pending_bind_roots,
1571 );
1572 if deferred_jobs {
1573 metrics
1574 .maintenance_budget_deferrals
1575 .fetch_add(1, Ordering::Relaxed);
1576 }
1577 for (root_id, kind) in due_jobs {
1578 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1579 bg_sub_by_session
1580 .iter()
1581 .filter_map(|((root, session), _)| {
1582 if root == &root_id {
1583 Some((
1584 session.clone(),
1585 bg_wake_epoch
1586 .get(&(root_id.clone(), session.clone()))
1587 .copied()
1588 .unwrap_or(0),
1589 ))
1590 } else {
1591 None
1592 }
1593 })
1594 .collect()
1595 } else {
1596 Vec::new()
1597 };
1598 submit_maintenance_job(
1599 executor,
1600 root_id,
1601 kind,
1602 bg_sessions_to_check,
1603 maintenance_tx,
1604 metrics,
1605 );
1606 }
1607}
1608
1609fn should_requiesce_after_maintenance(
1610 meta: &RootMeta,
1611 completed_kind: MaintenanceDrainKind,
1612 bind_pending: bool,
1613) -> bool {
1614 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1615}
1616
1617fn note_maintenance_completion(
1618 meta: &mut RootMeta,
1619 requeue_kind: Option<MaintenanceDrainKind>,
1620 fatal: bool,
1621 defer_requeue: bool,
1622) {
1623 if fatal {
1624 meta.maintenance_poisoned = true;
1625 }
1626
1627 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1628 meta.maintenance_queued_kinds.push_back(kind);
1629 }
1630
1631 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1632 meta.maintenance_pending =
1633 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1634}
1635
1636fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1637 RouteChannel { channel, epoch }
1638}
1639
1640fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1641 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1642 installed_epochs.remove(&route.channel);
1643 }
1644}
1645
1646fn ingress_route_should_be_processed(
1647 installed_epochs: &HashMap<u16, u32>,
1648 reclaimed_routes: &ReclaimedRoutes,
1649 frame: &Frame,
1650) -> bool {
1651 if frame.header.channel == 0
1652 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1653 {
1654 return true;
1655 }
1656
1657 frame.header.ty == FrameType::Request
1662 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1663}
1664
1665fn bash_elicitation_timeout() -> Duration {
1666 if cfg!(debug_assertions) {
1667 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1668 if let Ok(ms) = raw.parse::<u64>() {
1669 if ms > 0 {
1670 return Duration::from_millis(ms);
1671 }
1672 }
1673 }
1674 }
1675 BASH_ELICITATION_TIMEOUT
1676}
1677
1678fn allocate_reverse_corr(
1679 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1680 route: RouteChannel,
1681 next_corr: &mut u64,
1682) -> u64 {
1683 loop {
1684 let corr = *next_corr;
1685 *next_corr = (*next_corr).wrapping_add(1).max(1);
1686 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1687 return corr;
1688 }
1689 }
1690}
1691
1692fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1693 match kind {
1694 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1695 crate::bash_permissions::PermissionKind::Bash => "bash",
1696 }
1697}
1698
1699fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1700 let mut patterns = Vec::new();
1701 let mut seen = HashSet::new();
1702 for ask in asks {
1703 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1704 if seen.insert(pattern.clone()) {
1705 patterns.push(pattern.clone());
1706 }
1707 }
1708 }
1709 patterns
1710}
1711
1712fn bash_elicitation_message(
1713 command: &str,
1714 asks: &[crate::bash_permissions::PermissionAsk],
1715) -> String {
1716 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1717 let patterns = bash_elicitation_patterns(asks);
1718 let pattern_text = if patterns.is_empty() {
1719 "no matched permission patterns".to_string()
1720 } else {
1721 patterns.join(", ")
1722 };
1723 let ask_kinds = asks
1724 .iter()
1725 .map(|ask| bash_permission_kind_label(&ask.kind))
1726 .collect::<HashSet<_>>()
1727 .into_iter()
1728 .collect::<Vec<_>>()
1729 .join(", ");
1730 if ask_kinds.is_empty() {
1731 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1732 } else {
1733 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1734 }
1735}
1736
1737fn bash_elicitation_request_body(
1738 command: &str,
1739 asks: &[crate::bash_permissions::PermissionAsk],
1740) -> Value {
1741 json!({
1742 "method": BASH_ELICITATION_CREATE_METHOD,
1743 "params": {
1744 "mode": "form",
1745 "message": bash_elicitation_message(command, asks),
1746 "requestedSchema": {
1747 "type": "object",
1748 "properties": {
1749 "decision": {
1750 "type": "string",
1751 "enum": ["allow", "deny"],
1752 "description": "Choose allow to run this bash command once, or deny to block it."
1753 }
1754 },
1755 "required": ["decision"],
1756 "additionalProperties": false
1757 },
1758 "_meta": {
1759 "aft": {
1760 "tool": "bash",
1761 "command": command,
1762 "asks": asks
1763 }
1764 }
1765 }
1766 })
1767}
1768
1769fn build_bash_elicitation_request_frame(
1770 ver: u8,
1771 route: RouteChannel,
1772 corr: u64,
1773 flags: Flags,
1774 command: &str,
1775 asks: &[crate::bash_permissions::PermissionAsk],
1776) -> Result<Frame, SubcError> {
1777 let body = bash_elicitation_request_body(command, asks);
1778 Frame::build_with_version(
1779 ver,
1780 FrameType::Request,
1781 flags,
1782 route.channel,
1783 route.epoch,
1784 corr,
1785 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1786 )
1787 .map_err(SubcError::FrameBuild)
1788}
1789
1790fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1791 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1792 return false;
1793 };
1794 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1795}
1796
1797fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1798 let Some(object) = value.as_object() else {
1799 return false;
1800 };
1801 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1802}
1803
1804fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1805 let Some(object) = value.as_object() else {
1806 return false;
1807 };
1808 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1809 return false;
1810 }
1811 let Some(content) = object.get("content").and_then(Value::as_object) else {
1812 return false;
1813 };
1814 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1815}
1816
1817#[allow(clippy::too_many_arguments)]
1818async fn settle_pending_bash_ask_denied(
1819 tx: &WriterSender,
1820 pending: PendingBashAsk,
1821 routes: &HashMap<RouteChannel, RouteIdentity>,
1822 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1823 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1824 shutdown: &Arc<Notify>,
1825 metrics: &DispatchPathMetrics,
1826) -> Result<(), SubcError> {
1827 let completion = bash::bash_denied_untrusted_completion(
1828 pending.route,
1829 pending.tool_corr,
1830 pending.tool_flags,
1831 pending.tool_ver,
1832 pending.root,
1833 pending.request_id,
1834 pending.format_context,
1835 );
1836 bash::handle_bash_deferred_completion(
1837 tx,
1838 completion,
1839 routes,
1840 live_roots,
1841 route_bash_cancels,
1842 shutdown,
1843 metrics,
1844 )
1845 .await
1846}
1847
1848fn take_pending_bash_asks_for_route(
1849 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1850 route: RouteChannel,
1851) -> Vec<PendingBashAsk> {
1852 let keys = pending_bash_asks
1853 .keys()
1854 .copied()
1855 .filter(|key| key.route == route)
1856 .collect::<Vec<_>>();
1857 keys.into_iter()
1858 .filter_map(|key| pending_bash_asks.remove(&key))
1859 .collect()
1860}
1861
1862#[allow(clippy::too_many_arguments)]
1863async fn settle_pending_bash_asks_for_route(
1864 tx: &WriterSender,
1865 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1866 route: RouteChannel,
1867 routes: &HashMap<RouteChannel, RouteIdentity>,
1868 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1869 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1870 shutdown: &Arc<Notify>,
1871 metrics: &DispatchPathMetrics,
1872) -> Result<(), SubcError> {
1873 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1874 settle_pending_bash_ask_denied(
1875 tx,
1876 pending,
1877 routes,
1878 live_roots,
1879 route_bash_cancels,
1880 shutdown,
1881 metrics,
1882 )
1883 .await?;
1884 }
1885 Ok(())
1886}
1887
1888#[allow(clippy::too_many_arguments)]
1889async fn settle_all_pending_bash_asks(
1890 tx: &WriterSender,
1891 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1892 routes: &HashMap<RouteChannel, RouteIdentity>,
1893 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1894 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1895 shutdown: &Arc<Notify>,
1896 metrics: &DispatchPathMetrics,
1897) -> Result<(), SubcError> {
1898 let pending = pending_bash_asks
1899 .drain()
1900 .map(|(_, pending)| pending)
1901 .collect::<Vec<_>>();
1902 for pending in pending {
1903 settle_pending_bash_ask_denied(
1904 tx,
1905 pending,
1906 routes,
1907 live_roots,
1908 route_bash_cancels,
1909 shutdown,
1910 metrics,
1911 )
1912 .await?;
1913 }
1914 Ok(())
1915}
1916
1917#[allow(clippy::too_many_arguments)]
1918async fn expire_pending_bash_asks(
1919 tx: &WriterSender,
1920 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1921 routes: &HashMap<RouteChannel, RouteIdentity>,
1922 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1923 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1924 shutdown: &Arc<Notify>,
1925 metrics: &DispatchPathMetrics,
1926) -> Result<(), SubcError> {
1927 let now = Instant::now();
1928 let expired = pending_bash_asks
1929 .iter()
1930 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1931 .collect::<Vec<_>>();
1932 for key in expired {
1933 if let Some(pending) = pending_bash_asks.remove(&key) {
1934 log::debug!(
1935 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1936 key.corr,
1937 pending.route
1938 );
1939 settle_pending_bash_ask_denied(
1940 tx,
1941 pending,
1942 routes,
1943 live_roots,
1944 route_bash_cancels,
1945 shutdown,
1946 metrics,
1947 )
1948 .await?;
1949 }
1950 }
1951 Ok(())
1952}
1953
1954#[allow(clippy::too_many_arguments)]
1955async fn handle_bash_elicitation_reply(
1956 tx: &WriterSender,
1957 frame: &Frame,
1958 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1959 routes: &HashMap<RouteChannel, RouteIdentity>,
1960 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1961 executor: &Arc<Executor>,
1962 shutdown: &Arc<Notify>,
1963 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1964 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1965 metrics: &Arc<DispatchPathMetrics>,
1966 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1967 dispatch: DispatchFn,
1968) -> Result<(), SubcError> {
1969 let key = ReverseCorrKey {
1970 route: route_key(frame.header.channel, frame.header.epoch),
1971 corr: frame.header.corr,
1972 };
1973 let Some(pending) = pending_bash_asks.remove(&key) else {
1974 return Ok(());
1975 };
1976
1977 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1978 if routes.contains_key(&key.route) {
1979 bash::submit_deferred_bash(
1980 executor,
1981 bash_deferred_tx,
1982 bash_poll_touch_tx,
1983 metrics,
1984 dispatch,
1985 pending.root,
1986 pending.project_root,
1987 pending.session_id,
1988 pending.request_id,
1989 pending.route,
1990 pending.tool_corr,
1991 pending.tool_flags,
1992 pending.tool_ver,
1993 pending.arguments,
1994 pending.format_context,
1995 pending.cancel,
1996 BindTrust::Untrusted,
1997 pending.spawn_principal,
1998 pending.edit_slot_survives,
1999 Some(pending.grants),
2000 );
2001 return Ok(());
2002 }
2003 log::debug!(
2004 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2005 key.corr,
2006 pending.route
2007 );
2008 }
2009
2010 settle_pending_bash_ask_denied(
2011 tx,
2012 pending,
2013 routes,
2014 live_roots,
2015 route_bash_cancels,
2016 shutdown,
2017 metrics,
2018 )
2019 .await
2020}
2021
2022#[allow(clippy::too_many_arguments)]
2023async fn cancel_pending_bash_ask_for_tool_call(
2024 tx: &WriterSender,
2025 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2026 route: RouteChannel,
2027 tool_corr: u64,
2028 routes: &HashMap<RouteChannel, RouteIdentity>,
2029 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2030 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2031 shutdown: &Arc<Notify>,
2032 metrics: &DispatchPathMetrics,
2033) -> Result<(), SubcError> {
2034 let keys = pending_bash_asks
2035 .iter()
2036 .filter_map(|(key, pending)| {
2037 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2038 })
2039 .collect::<Vec<_>>();
2040 for key in keys {
2041 if let Some(pending) = pending_bash_asks.remove(&key) {
2042 settle_pending_bash_ask_denied(
2043 tx,
2044 pending,
2045 routes,
2046 live_roots,
2047 route_bash_cancels,
2048 shutdown,
2049 metrics,
2050 )
2051 .await?;
2052 }
2053 }
2054 Ok(())
2055}
2056
2057fn remove_root_channel(
2058 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2059 root: &ProjectRootId,
2060 channel: RouteChannel,
2061) {
2062 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2063 channels.remove(&channel);
2064 channels.is_empty()
2065 } else {
2066 false
2067 };
2068 if remove_root {
2069 root_channels.remove(root);
2070 }
2071}
2072
2073fn remove_route_channel(
2074 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2075 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2076 channel: RouteChannel,
2077) -> Option<RouteIdentity> {
2078 let removed = routes.remove(&channel);
2079 if let Some(identity) = &removed {
2080 remove_root_channel(root_channels, &identity.root, channel);
2081 }
2082 removed
2083}
2084
2085fn insert_route_channel(
2086 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2087 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2088 channel: RouteChannel,
2089 identity: RouteIdentity,
2090) {
2091 if let Some(previous) = routes.insert(channel, identity.clone()) {
2092 remove_root_channel(root_channels, &previous.root, channel);
2093 }
2094 root_channels
2095 .entry(identity.root.clone())
2096 .or_default()
2097 .insert(channel);
2098}
2099
2100fn sync_bg_live_delivery_sessions(
2101 executor: &Executor,
2102 routes: &HashMap<RouteChannel, RouteIdentity>,
2103 additional_root: Option<&ProjectRootId>,
2104) {
2105 let sessions = routes
2109 .values()
2110 .filter(|identity| identity.trust.allows_bash_observation())
2111 .map(|identity| identity.session.clone())
2112 .collect::<HashSet<_>>();
2113 let mut roots = routes
2114 .values()
2115 .map(|identity| identity.root.clone())
2116 .collect::<HashSet<_>>();
2117 roots.extend(additional_root.cloned());
2118 for root in roots {
2119 if let Some(ctx) = executor.actor_context(&root) {
2120 ctx.bash_background()
2121 .replace_live_delivery_sessions(sessions.clone());
2122 }
2123 }
2124}
2125
2126fn insert_bg_subscription_index(
2127 bg_sub_by_session: &mut BgSubsBySession,
2128 root: ProjectRootId,
2129 session: String,
2130 channel: RouteChannel,
2131) {
2132 bg_sub_by_session
2133 .entry((root, session))
2134 .or_default()
2135 .insert(channel);
2136}
2137
2138fn remove_bg_subscription_index(
2139 bg_sub_by_session: &mut BgSubsBySession,
2140 channel: RouteChannel,
2141 identity: Option<&RouteIdentity>,
2142) {
2143 if let Some(identity) = identity {
2144 let key = (identity.root.clone(), identity.session.clone());
2145 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2146 channels.remove(&channel);
2147 channels.is_empty()
2148 });
2149 if remove_key {
2150 bg_sub_by_session.remove(&key);
2151 }
2152 } else {
2153 bg_sub_by_session.retain(|_, channels| {
2154 channels.remove(&channel);
2155 !channels.is_empty()
2156 });
2157 }
2158}
2159
2160fn route_removal_will_quiesce_root(
2161 root: &ProjectRootId,
2162 route: RouteChannel,
2163 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2164 has_pending_bind: bool,
2165 replacement_root: Option<&ProjectRootId>,
2166) -> bool {
2167 let removes_last_route = root_channels
2168 .get(root)
2169 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2170 removes_last_route && !has_pending_bind && replacement_root != Some(root)
2171}
2172
2173fn should_quiesce_removed_root(
2174 root: &ProjectRootId,
2175 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2176 has_pending_bind: bool,
2177 replacement_root: Option<&ProjectRootId>,
2178) -> bool {
2179 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2180}
2181
2182async fn end_bg_subscription(
2183 writer_tx: &WriterSender,
2184 metrics: &DispatchPathMetrics,
2185 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2186 bg_sub_by_session: &mut BgSubsBySession,
2187 bg_wake_pending: &mut BgWakePending,
2188 channel: RouteChannel,
2189 identity: Option<&RouteIdentity>,
2190 cause: &str,
2191) -> Result<(), SubcError> {
2192 if let Some(sub) = bg_subs.remove(&channel) {
2193 bg_wake_pending.remove(&channel);
2194 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2195 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2196 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2197 }
2198 Ok(())
2199}
2200
2201#[allow(clippy::too_many_arguments)]
2202async fn teardown_installed_route(
2203 tx: &WriterSender,
2204 metrics: &DispatchPathMetrics,
2205 executor: &Arc<Executor>,
2206 channel: RouteChannel,
2207 cancellation_reason: &str,
2208 replacement_root: Option<&ProjectRootId>,
2209 installed_route_epochs: &mut HashMap<u16, u32>,
2210 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2211 management_routes: &mut HashSet<RouteChannel>,
2212 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2213 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2214 bg_sub_by_session: &mut BgSubsBySession,
2215 bg_wake_pending: &mut BgWakePending,
2216 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2217 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2218 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2219 active_tool_calls: &ActiveToolCalls,
2220 pending_responses: &mut PendingSubcResponses,
2221 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2222 retry_buffer: &mut RetryBuffer,
2223 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2224 shutdown: &Arc<Notify>,
2225 tool_response_body_limit: usize,
2226 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2227) -> Result<(), SubcError> {
2228 remove_installed_route(installed_route_epochs, channel);
2229 management_routes.remove(&channel);
2230 let bg_end_cause = match cancellation_reason {
2231 "Goodbye" => "goodbye",
2232 "higher-epoch RouteBind" => "higher-epoch",
2233 other => other,
2234 };
2235 end_bg_subscription(
2236 tx,
2237 metrics,
2238 bg_subs,
2239 bg_sub_by_session,
2240 bg_wake_pending,
2241 channel,
2242 routes.get(&channel),
2243 bg_end_cause,
2244 )
2245 .await?;
2246 settle_pending_bash_asks_for_route(
2247 tx,
2248 pending_bash_asks,
2249 channel,
2250 routes,
2251 live_roots,
2252 route_bash_cancels,
2253 shutdown,
2254 metrics,
2255 )
2256 .await?;
2257 if let Some(cancel) = route_bash_cancels.remove(&channel) {
2258 cancel.token.cancel();
2259 }
2260 for resolved in pending_responses.drain_route(channel, executor) {
2261 deliver_resolved_subc_response(
2262 tx,
2263 resolved,
2264 routes,
2265 live_roots,
2266 executor.as_ref(),
2267 active_tool_calls,
2268 shutdown,
2269 metrics,
2270 tool_response_body_limit,
2271 )
2272 .await?;
2273 }
2274 let work_disposition = if replacement_root.is_some() {
2279 RouteWorkDisposition::RetainForReplay
2280 } else {
2281 RouteWorkDisposition::RetainStartedForReplay
2282 };
2283 apply_route_work_disposition(
2284 active_tool_calls,
2285 executor,
2286 channel,
2287 work_disposition,
2288 cancellation_reason,
2289 );
2290 if let Some(pending) = pending_binds.get_mut(&channel) {
2291 pending.cancelled = true;
2292 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2293 log::debug!(
2294 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2295 channel.channel
2296 );
2297 }
2298 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2299 if let Some(identity) = routes.get(&channel) {
2300 let has_pending_bind = pending_binds
2301 .values()
2302 .any(|pending| pending.bind_root_id == identity.root);
2303 if route_removal_will_quiesce_root(
2304 &identity.root,
2305 channel,
2306 root_channels,
2307 has_pending_bind,
2308 replacement_root,
2309 ) {
2310 if let Some(ctx) = executor.actor_context(&identity.root) {
2311 ctx.mark_subc_unbound();
2314 }
2315 }
2316 }
2317 delay_route_detach_for_test(lifecycle_probe).await;
2320 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2321 sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2322 if let Some(probe) = lifecycle_probe {
2323 probe.route_detached(channel, &identity.session);
2324 }
2325 let session_still_routed = routes
2326 .values()
2327 .any(|route| route.root == identity.root && route.session == identity.session);
2328 if !session_still_routed {
2329 if let Some(ctx) = executor.actor_context(&identity.root) {
2330 ctx.hashline_bindings()
2331 .teardown(identity.root.as_path(), &identity.session);
2332 }
2333 }
2334 if migrated > 0 {
2335 log::debug!(
2336 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2337 channel.channel
2338 );
2339 }
2340 if let Some(meta) = live_roots.get_mut(&identity.root) {
2341 let idle_for = meta.last_touched.elapsed();
2342 meta.note_activity();
2343 log::debug!(
2344 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2345 channel.channel,
2346 identity.root.as_path().display(),
2347 identity.harness,
2348 identity.session,
2349 idle_for
2350 );
2351 } else {
2352 log::debug!(
2353 "subc attach: route {} torn down for root {} harness {} session {}",
2354 channel.channel,
2355 identity.root.as_path().display(),
2356 identity.harness,
2357 identity.session
2358 );
2359 }
2360 let has_pending_bind = pending_binds
2361 .values()
2362 .any(|pending| pending.bind_root_id == identity.root);
2363 if should_quiesce_removed_root(
2364 &identity.root,
2365 root_channels,
2366 has_pending_bind,
2367 replacement_root,
2368 ) {
2369 quiesce_unbound_root(&identity.root, live_roots, executor);
2370 }
2371 } else {
2372 if migrated > 0 {
2373 log::debug!(
2374 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2375 channel.channel
2376 );
2377 }
2378 log::debug!("subc attach: unbound route {} torn down", channel.channel);
2379 }
2380 Ok(())
2381}
2382
2383async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2384 if lifecycle_probe.is_none() {
2385 return;
2386 }
2387 let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2388 .ok()
2389 .and_then(|raw| raw.parse::<u64>().ok())
2390 else {
2391 return;
2392 };
2393 tokio::time::sleep(Duration::from_millis(delay)).await;
2394}
2395
2396fn remember_session_identity(
2397 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2398 identity: &RouteIdentity,
2399) {
2400 let key = (identity.root.clone(), identity.session.clone());
2401 if matches!(identity.trust, BindTrust::Untrusted)
2402 && session_identity
2403 .get(&key)
2404 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2405 {
2406 return;
2407 }
2408
2409 session_identity.insert(
2414 key,
2415 RetainedSessionIdentity {
2416 harness: identity.harness.clone(),
2417 trust: identity.trust,
2418 },
2419 );
2420}
2421
2422fn replay_key_for_session(
2423 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2424 root: &ProjectRootId,
2425 session: &str,
2426) -> Option<(push::ReplayKey, BindTrust)> {
2427 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2428 Some((
2429 push::ReplayKey {
2430 root: root.clone(),
2431 harness: retained.harness.clone(),
2432 session: session.to_string(),
2433 },
2434 retained.trust,
2435 ))
2436}
2437pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2440
2441#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2442enum ModuleLoopExit {
2443 Graceful,
2446 ConnectionLost,
2449 SkipSearchFlush,
2454}
2455
2456pub fn run_subc_mode(
2461 connection_file_path: &Path,
2462 ctx: Arc<AppContext>,
2463 executor: Arc<Executor>,
2464 dispatch: DispatchFn,
2465 user_config_path: Option<PathBuf>,
2466) -> Result<(), SubcError> {
2467 run_subc_mode_inner(
2471 connection_file_path,
2472 ctx,
2473 executor,
2474 dispatch,
2475 user_config_path,
2476 false,
2477 MAX_FRAME_BODY_LEN as usize,
2478 None,
2479 )
2480}
2481
2482fn run_subc_mode_inner(
2483 connection_file_path: &Path,
2484 ctx: Arc<AppContext>,
2485 executor: Arc<Executor>,
2486 dispatch: DispatchFn,
2487 user_config_path: Option<PathBuf>,
2488 allow_native_passthrough: bool,
2489 tool_response_body_limit: usize,
2490 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2491) -> Result<(), SubcError> {
2492 let runtime = tokio::runtime::Builder::new_current_thread()
2493 .enable_all()
2494 .build()
2495 .map_err(SubcError::Runtime)?;
2496
2497 let executor_for_loop = Arc::clone(&executor);
2498 let loop_result = runtime.block_on(async move {
2499 let shared_app = ctx.app();
2500 drop(ctx);
2501 let stream =
2502 connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2503 log::info!(
2504 "subc attach: authenticated to daemon via {}",
2505 connection_file_path.display()
2506 );
2507 let (read_half, write_half) = tokio::io::split(stream);
2508 run_module_loop(
2509 read_half,
2510 write_half,
2511 connection_file_path,
2512 shared_app,
2513 executor_for_loop,
2514 dispatch,
2515 user_config_path,
2516 allow_native_passthrough,
2517 tool_response_body_limit,
2518 lifecycle_probe,
2519 )
2520 .await
2521 });
2522
2523 let actor_contexts = executor.actor_contexts();
2524 if matches!(
2525 loop_result,
2526 Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2527 ) {
2528 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2531 }
2532 for actor_ctx in &actor_contexts {
2533 actor_ctx.lsp().shutdown_all();
2534 actor_ctx.bash_background().detach();
2535 }
2536
2537 match loop_result {
2538 Ok(exit) => module_loop_exit_result(exit),
2539 Err(error) => Err(error),
2540 }
2541}
2542
2543fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2547 match exit {
2548 ModuleLoopExit::Graceful => Ok(()),
2549 ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2550 ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2551 }
2552}
2553
2554fn note_fatal_panic_response(response: &Response) -> bool {
2558 let fatal = response_is_fatal_panic(response);
2559 if fatal {
2560 log::error!(
2561 "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2562 response.id,
2563 response
2564 .data
2565 .get("message")
2566 .and_then(Value::as_str)
2567 .unwrap_or("(no message)")
2568 );
2569 }
2570 fatal
2571}
2572
2573fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2574 for actor_ctx in actor_contexts {
2575 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2576 }
2577 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2578}
2579
2580#[doc(hidden)]
2585pub fn run_subc_mode_for_test(
2586 connection_file_path: &Path,
2587 ctx: Arc<AppContext>,
2588 executor: Arc<Executor>,
2589 dispatch: DispatchFn,
2590 user_config_path: Option<PathBuf>,
2591) -> Result<(), SubcError> {
2592 run_subc_mode_inner(
2593 connection_file_path,
2594 ctx,
2595 executor,
2596 dispatch,
2597 user_config_path,
2598 true,
2599 MAX_FRAME_BODY_LEN as usize,
2600 None,
2601 )
2602}
2603
2604#[doc(hidden)]
2606pub fn run_subc_mode_for_test_with_lifecycle_probe(
2607 connection_file_path: &Path,
2608 ctx: Arc<AppContext>,
2609 executor: Arc<Executor>,
2610 dispatch: DispatchFn,
2611 user_config_path: Option<PathBuf>,
2612 lifecycle_probe: SubcTestLifecycleProbe,
2613) -> Result<(), SubcError> {
2614 run_subc_mode_inner(
2615 connection_file_path,
2616 ctx,
2617 executor,
2618 dispatch,
2619 user_config_path,
2620 true,
2621 MAX_FRAME_BODY_LEN as usize,
2622 Some(lifecycle_probe),
2623 )
2624}
2625
2626#[doc(hidden)]
2629pub fn run_subc_mode_for_test_with_response_body_limit(
2630 connection_file_path: &Path,
2631 ctx: Arc<AppContext>,
2632 executor: Arc<Executor>,
2633 dispatch: DispatchFn,
2634 user_config_path: Option<PathBuf>,
2635 tool_response_body_limit: usize,
2636) -> Result<(), SubcError> {
2637 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2638 run_subc_mode_inner(
2639 connection_file_path,
2640 ctx,
2641 executor,
2642 dispatch,
2643 user_config_path,
2644 true,
2645 tool_response_body_limit,
2646 None,
2647 )
2648}
2649
2650#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2651enum AttachErrorClass {
2652 Transient,
2653 Permanent,
2654}
2655
2656impl fmt::Display for AttachErrorClass {
2657 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2658 match self {
2659 Self::Transient => f.write_str("transient"),
2660 Self::Permanent => f.write_str("permanent"),
2661 }
2662 }
2663}
2664
2665#[derive(Clone, Copy)]
2666struct AttachRetryPolicy {
2667 budget: Duration,
2668 initial_backoff: Duration,
2669 max_backoff: Duration,
2670 jitter_percent: u64,
2671}
2672
2673const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2674 budget: ATTACH_RETRY_BUDGET,
2675 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2676 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2677 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2678};
2679
2680fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2683 let transient = match error {
2684 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2685 SubcError::Auth { source, .. } => match source {
2686 subc_transport::AuthError::Timeout { .. }
2687 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2688 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2689 _ => false,
2690 },
2691 _ => false,
2692 };
2693 if transient {
2694 AttachErrorClass::Transient
2695 } else {
2696 AttachErrorClass::Permanent
2697 }
2698}
2699
2700fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2701 matches!(
2702 kind,
2703 io::ErrorKind::ConnectionRefused
2704 | io::ErrorKind::TimedOut
2705 | io::ErrorKind::ConnectionReset
2706 | io::ErrorKind::ConnectionAborted
2707 | io::ErrorKind::BrokenPipe
2708 | io::ErrorKind::UnexpectedEof
2709 )
2710}
2711
2712async fn connect_and_authenticate(
2716 connection_file_path: &Path,
2717 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2718) -> Result<TcpStream, SubcError> {
2719 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2720 .await
2721}
2722
2723async fn connect_and_authenticate_with_policy(
2724 connection_file_path: &Path,
2725 policy: AttachRetryPolicy,
2726 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2727) -> Result<TcpStream, SubcError> {
2728 let started_at = Instant::now();
2729 let deadline = started_at + policy.budget;
2730 let mut attempt = 0_u32;
2731 let mut backoff = policy.initial_backoff;
2732 let mut history = Vec::new();
2733
2734 loop {
2735 attempt = attempt.saturating_add(1);
2736 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2737 Ok(stream) => return Ok(stream),
2738 Err(error) => error,
2739 };
2740 let class = classify_attach_error(&error);
2741 let will_retry = class != AttachErrorClass::Permanent;
2742 if let Some(probe) = lifecycle_probe {
2743 probe.attach_decision(attempt, will_retry);
2744 }
2745 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2746 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2747
2748 if !will_retry {
2749 log_attach_final_failure(started_at.elapsed(), &history);
2750 return Err(error);
2751 }
2752
2753 let remaining = deadline.saturating_duration_since(Instant::now());
2754 if remaining.is_zero() {
2755 log_attach_final_failure(started_at.elapsed(), &history);
2756 return Err(error);
2757 }
2758
2759 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2760 log::info!(
2761 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2762 );
2763 tokio::time::sleep(delay).await;
2764
2765 if Instant::now() >= deadline {
2766 log_attach_final_failure(started_at.elapsed(), &history);
2767 return Err(error);
2768 }
2769 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2770 }
2771}
2772
2773fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2774 let jitter_percent = jitter_percent.min(100);
2775 if jitter_percent == 0 {
2776 return base;
2777 }
2778
2779 let mut random_bytes = [0_u8; 8];
2780 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2781 u64::from_le_bytes(random_bytes)
2782 } else {
2783 let timestamp = std::time::SystemTime::now()
2784 .duration_since(std::time::UNIX_EPOCH)
2785 .unwrap_or_default()
2786 .subsec_nanos();
2787 u64::from(timestamp) ^ u64::from(attempt)
2788 };
2789 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2790 let multiplier_percent = 100 - jitter_percent + random % span;
2791 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2792 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2793}
2794
2795fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2796 log::error!(
2797 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2798 history.len(),
2799 history.join(" | ")
2800 );
2801}
2802
2803async fn connect_and_authenticate_once(
2804 connection_file_path: &Path,
2805 deadline: Instant,
2806) -> Result<TcpStream, SubcError> {
2807 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2810 SubcError::ConnectionFile {
2811 path: connection_file_path.to_path_buf(),
2812 source,
2813 }
2814 })?;
2815
2816 let endpoint = conn
2817 .endpoints
2818 .first()
2819 .ok_or_else(|| SubcError::NoEndpoint {
2820 path: connection_file_path.to_path_buf(),
2821 })?;
2822 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2823 let ip = endpoint
2824 .host
2825 .parse::<IpAddr>()
2826 .map_err(|_| SubcError::InvalidEndpoint {
2827 path: connection_file_path.to_path_buf(),
2828 endpoint: endpoint_label.clone(),
2829 })?;
2830 let addr = SocketAddr::new(ip, endpoint.port);
2831
2832 let connect_budget = deadline.saturating_duration_since(Instant::now());
2833 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2834 .await
2835 .map_err(|_| SubcError::Connect {
2836 endpoint: endpoint_label.clone(),
2837 source: io::Error::new(
2838 io::ErrorKind::TimedOut,
2839 "initial subc attach retry budget elapsed during TCP connect",
2840 ),
2841 })?
2842 .map_err(|source| SubcError::Connect {
2843 endpoint: endpoint_label.clone(),
2844 source,
2845 })?;
2846 stream
2847 .set_nodelay(true)
2848 .map_err(|source| SubcError::Connect {
2849 endpoint: endpoint_label.clone(),
2850 source,
2851 })?;
2852
2853 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2854 authenticate_client(&mut stream, &conn, auth_budget)
2855 .await
2856 .map_err(|source| SubcError::Auth {
2857 endpoint: endpoint_label,
2858 source,
2859 })?;
2860
2861 Ok(stream)
2862}
2863
2864#[allow(clippy::too_many_arguments)]
2865async fn process_route_bind_completion(
2866 writer_tx: &WriterSender,
2867 completion: RouteBindCompletion,
2868 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2869 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2870 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2871 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2872 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2873 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2874 installed_route_epochs: &mut HashMap<u16, u32>,
2875 executor: &Arc<Executor>,
2876 standing_actor: &standing::StandingActor,
2877 shutdown: &Arc<Notify>,
2878 metrics: &Arc<DispatchPathMetrics>,
2879 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2880) -> Result<(), SubcError> {
2881 decrement_counted_channel(&metrics.control_completion_queued);
2882 handle_route_bind_completion(
2883 writer_tx,
2884 completion,
2885 routes,
2886 root_channels,
2887 session_identity,
2888 push_buffer,
2889 live_roots,
2890 pending_binds,
2891 installed_route_epochs,
2892 executor,
2893 standing_actor,
2894 shutdown,
2895 metrics,
2896 lifecycle_probe,
2897 )
2898 .await
2899}
2900
2901#[allow(clippy::too_many_arguments)]
2902async fn drain_pending_route_bind_completions(
2903 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2904 writer_tx: &WriterSender,
2905 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2906 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2907 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2908 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2909 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2910 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2911 installed_route_epochs: &mut HashMap<u16, u32>,
2912 executor: &Arc<Executor>,
2913 standing_actor: &standing::StandingActor,
2914 shutdown: &Arc<Notify>,
2915 metrics: &Arc<DispatchPathMetrics>,
2916 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2917) -> Result<usize, SubcError> {
2918 let mut drained = 0;
2919 while let Ok(completion) = control_completion_rx.try_recv() {
2920 process_route_bind_completion(
2921 writer_tx,
2922 completion,
2923 routes,
2924 root_channels,
2925 session_identity,
2926 push_buffer,
2927 live_roots,
2928 pending_binds,
2929 installed_route_epochs,
2930 executor,
2931 standing_actor,
2932 shutdown,
2933 metrics,
2934 lifecycle_probe,
2935 )
2936 .await?;
2937 drained += 1;
2938 }
2939 Ok(drained)
2940}
2941
2942async fn run_module_loop<R, W>(
2946 mut read: R,
2947 mut write: W,
2948 connection_file_path: &Path,
2949 shared_app: Arc<App>,
2950 executor: Arc<Executor>,
2951 dispatch: DispatchFn,
2952 user_config_path: Option<PathBuf>,
2953 allow_native_passthrough: bool,
2954 tool_response_body_limit: usize,
2955 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2956) -> Result<ModuleLoopExit, SubcError>
2957where
2958 R: AsyncRead + Unpin + Send + 'static,
2959 W: AsyncWrite + Unpin + Send + 'static,
2960{
2961 let hello = ModuleHelloBody {
2966 manifest: build_manifest(),
2967 protocol_ver: PROTOCOL_VERSION,
2968 control_ops: control_ops(),
2969 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2970 };
2971 let hello_frame = Frame::build(
2972 FrameType::Hello,
2973 control_flags(),
2974 0,
2975 0,
2976 HELLO_CORR,
2977 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2978 )
2979 .map_err(SubcError::FrameBuild)?;
2980 write_frame(&mut write, &hello_frame)
2981 .await
2982 .map_err(SubcError::FrameIo)?;
2983
2984 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2986 None => return Err(SubcError::ClosedBeforeHelloAck),
2987 Some(frame) => match frame.header.ty {
2988 FrameType::HelloAck => {
2989 log::info!("subc attach: registered (HelloAck received)");
2990 }
2991 FrameType::Error => {
2992 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2993 return Err(SubcError::HelloRejected { body });
2994 }
2995 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2996 },
2997 }
2998
2999 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
3000 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3001 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3002 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3009 let reader_task = spawn_reader_task(read, reader_tx);
3010 let shutdown = Arc::new(Notify::new());
3011 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3020 let mut next_maintenance_at = next_drain_at;
3021 let standing_actor =
3022 standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3023 standing_actor.reconcile_at_startup();
3026 let mut next_standing_pass_at = tokio::time::Instant::now();
3027 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3028 let (bash_deferred_tx, mut bash_deferred_rx) =
3029 mpsc::channel::<bash::BashDeferredCompletion>(256);
3030 let (deferred_response_tx, mut deferred_response_rx) =
3031 mpsc::unbounded_channel::<PendingSubcResponse>();
3032 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3033 let (control_completion_tx, mut control_completion_rx) =
3034 mpsc::channel::<RouteBindCompletion>(256);
3035 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3036 let lossy_overflow = Arc::new(push::LossyOverflow::default());
3037 let lossy_seq = Arc::new(AtomicU64::new(0));
3038 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3039 let (fleet_status_client, fleet_status_task) =
3040 spawn_fleet_status_dial(connection_file_path, 64);
3041 let push_senders = PushSenders {
3042 lossy_tx,
3043 reliable_tx,
3044 lossy_overflow: Arc::clone(&lossy_overflow),
3045 lossy_seq,
3046 fleet_status_client: fleet_status_client.clone(),
3047 };
3048 let connection_cancel = PersistentCancelSignal::new();
3049 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3050 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3051 let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3052 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3053 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3054 let mut bg_wake_pending = BgWakePending::new();
3055 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3056 let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3057 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3058 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3059 HashMap::new();
3060 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3061 let mut retry_buffer: RetryBuffer = HashMap::new();
3062 let mut reclaimed_routes = ReclaimedRoutes::default();
3063 let mut completed_tasks = push::CompletedTaskIds::default();
3064 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3065 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3066 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3067 let mut next_bash_ask_corr: u64 = 1;
3068 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3069 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3070 let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3071 let mut pending_responses = PendingSubcResponses::default();
3072 let health_rollup_cache = Arc::new(HealthRollupCache::new());
3073 let health_rollup_worker = HealthRollupWorker::start(
3074 Arc::clone(&health_rollup_cache),
3075 Arc::clone(&executor),
3076 Arc::clone(&shared_app),
3077 );
3078
3079 let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3080 shared_app.set_open_route_count(routes.len() + management_routes.len());
3081 crate::logging::perf_tick(Some(&executor));
3082 dispatch_path_metrics.mark_frame_loop_tick();
3083 let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3084 for resolved in ready_inspects {
3085 if let Err(error) = deliver_resolved_subc_response(
3086 &writer_tx,
3087 resolved,
3088 &routes,
3089 &mut live_roots,
3090 executor.as_ref(),
3091 &active_tool_calls,
3092 &shutdown,
3093 &dispatch_path_metrics,
3094 tool_response_body_limit,
3095 )
3096 .await
3097 {
3098 break 'module_loop Err(error);
3099 }
3100 }
3101 if let Err(error) = expire_pending_bash_asks(
3102 &writer_tx,
3103 &mut pending_bash_asks,
3104 &routes,
3105 &mut live_roots,
3106 &mut route_bash_cancels,
3107 &shutdown,
3108 &dispatch_path_metrics,
3109 )
3110 .await
3111 {
3112 break Err(error);
3113 }
3114
3115 match drain_pending_route_bind_completions(
3119 &mut control_completion_rx,
3120 &writer_tx,
3121 &mut routes,
3122 &mut root_channels,
3123 &mut session_identity,
3124 &mut push_buffer,
3125 &mut live_roots,
3126 &mut pending_binds,
3127 &mut installed_route_epochs,
3128 &executor,
3129 &standing_actor,
3130 &shutdown,
3131 &dispatch_path_metrics,
3132 lifecycle_probe.as_ref(),
3133 )
3134 .await
3135 {
3136 Ok(drained) => {
3137 if drained > 0 {
3138 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3139 health_rollup_worker.request_refresh();
3140 }
3141 }
3142 Err(error) => break Err(error),
3143 }
3144
3145 if tokio::time::Instant::now() >= next_drain_at {
3146 push::emit_bg_event_wakes(
3147 &writer_tx,
3148 &dispatch_path_metrics,
3149 &bg_subs,
3150 &mut bg_wake_pending,
3151 );
3152 dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3153 warn_slow_pending_binds(&mut pending_binds, &executor);
3154 warn_slow_running_interactive_jobs(&executor);
3155 if let Err(error) = expire_overdue_route_binds(
3156 &writer_tx,
3157 &executor,
3158 &mut pending_binds,
3159 &mut installed_route_epochs,
3160 &dispatch_path_metrics,
3161 )
3162 .await
3163 {
3164 break Err(error);
3165 }
3166
3167 let retried = push::drain_retry_buffers_for_bound_routes(
3168 &writer_tx,
3169 &dispatch_path_metrics,
3170 &routes,
3171 &mut retry_buffer,
3172 );
3173 if retried > 0 {
3174 log::debug!(
3175 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3176 );
3177 }
3178
3179 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3180 }
3181
3182 let overflow_batch = lossy_overflow.drain();
3188 if !overflow_batch.is_empty() {
3189 let (_, deferred) = push::drain_reliable_push_turn(
3190 &writer_tx,
3191 &dispatch_path_metrics,
3192 &routes,
3193 &root_channels,
3194 &session_identity,
3195 &mut retry_buffer,
3196 &mut push_buffer,
3197 &mut completed_tasks,
3198 &bg_sub_by_session,
3199 &mut bg_wake_pending,
3200 &mut bg_wake_epoch,
3201 &mut reliable_rx,
3202 None,
3203 lifecycle_probe.as_ref(),
3204 );
3205 if deferred {
3206 tokio::task::yield_now().await;
3207 }
3208
3209 let mut batch = Vec::new();
3210 while let Ok(item) = lossy_rx.try_recv() {
3211 batch.push(item);
3212 }
3213 batch.extend(overflow_batch);
3214 push::process_lossy_push_envelope_batch(
3215 &writer_tx,
3216 &dispatch_path_metrics,
3217 &routes,
3218 &root_channels,
3219 &completed_tasks,
3220 batch,
3221 );
3222 }
3223
3224 tokio::select! {
3225 biased;
3226 Some(completion) = control_completion_rx.recv() => {
3227 if let Err(error) = process_route_bind_completion(
3228 &writer_tx,
3229 completion,
3230 &mut routes,
3231 &mut root_channels,
3232 &mut session_identity,
3233 &mut push_buffer,
3234 &mut live_roots,
3235 &mut pending_binds,
3236 &mut installed_route_epochs,
3237 &executor,
3238 &standing_actor,
3239 &shutdown,
3240 &dispatch_path_metrics,
3241 lifecycle_probe.as_ref(),
3242 )
3243 .await
3244 {
3245 break Err(error);
3246 }
3247 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3248 health_rollup_worker.request_refresh();
3249 }
3250 _ = shutdown.notified() => {
3251 log::warn!("subc attach: fatal executor response requested teardown");
3252 break Ok(ModuleLoopExit::SkipSearchFlush);
3253 }
3254 maybe_frame = reader_rx.recv() => {
3255 let frame = match maybe_frame {
3256 None => {
3257 log::warn!(
3258 "subc attach: daemon connection ended without Goodbye; exiting for restart"
3259 );
3260 break Ok(ModuleLoopExit::ConnectionLost);
3261 }
3262 Some(Err(error)) => break Err(error),
3263 Some(Ok(frame)) => frame,
3264 };
3265 let phase_trace = frame.phase_trace;
3266 let frame = frame.frame;
3267
3268 if !ingress_route_should_be_processed(
3269 &installed_route_epochs,
3270 &reclaimed_routes,
3271 &frame,
3272 ) {
3273 log::debug!(
3274 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3275 frame.header.ty,
3276 frame.header.channel,
3277 frame.header.epoch
3278 );
3279 continue;
3280 }
3281
3282 match frame.header.ty {
3283 FrameType::Ping if frame.header.channel == 0 => {
3284 let pong = match Frame::build_with_version(
3285 frame.header.ver,
3286 FrameType::Pong,
3287 frame.header.flags,
3288 0,
3289 0,
3290 frame.header.corr,
3291 Vec::new(),
3292 ) {
3293 Ok(pong) => pong,
3294 Err(error) => break Err(SubcError::FrameBuild(error)),
3295 };
3296 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3297 break Err(error);
3298 }
3299 }
3300 FrameType::Goodbye if frame.header.channel == 0 => {
3301 log::info!("subc attach: received channel-0 Goodbye");
3302 break Ok(ModuleLoopExit::Graceful);
3303 }
3304 FrameType::Goodbye => {
3305 let channel = route_key(frame.header.channel, frame.header.epoch);
3306 if let Err(error) = teardown_installed_route(
3307 &writer_tx,
3308 &dispatch_path_metrics,
3309 &executor,
3310 channel,
3311 "Goodbye",
3312 None,
3313 &mut installed_route_epochs,
3314 &mut routes,
3315 &mut management_routes,
3316 &mut root_channels,
3317 &mut bg_subs,
3318 &mut bg_sub_by_session,
3319 &mut bg_wake_pending,
3320 &mut pending_bash_asks,
3321 &mut live_roots,
3322 &mut route_bash_cancels,
3323 &active_tool_calls,
3324 &mut pending_responses,
3325 &mut pending_binds,
3326 &mut retry_buffer,
3327 &mut push_buffer,
3328 &shutdown,
3329 tool_response_body_limit,
3330 lifecycle_probe.as_ref(),
3331 )
3332 .await
3333 {
3334 break Err(error);
3335 }
3336 }
3337 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3338 if let Err(error) = handle_bash_elicitation_reply(
3339 &writer_tx,
3340 &frame,
3341 &mut pending_bash_asks,
3342 &routes,
3343 &mut live_roots,
3344 &executor,
3345 &shutdown,
3346 &bash_deferred_tx,
3347 &bash_poll_touch_tx,
3348 &dispatch_path_metrics,
3349 &mut route_bash_cancels,
3350 dispatch,
3351 )
3352 .await
3353 {
3354 break Err(error);
3355 }
3356 }
3357 FrameType::Request if frame.header.channel == 0 => {
3358 if let Err(error) = handle_control_request(
3359 &writer_tx,
3360 &frame,
3361 &shared_app,
3362 &executor,
3363 &mut live_roots,
3364 &mut pending_binds,
3365 &mut installed_route_epochs,
3366 &mut routes,
3367 &mut management_routes,
3368 &mut root_channels,
3369 &mut bg_subs,
3370 &mut bg_sub_by_session,
3371 &mut bg_wake_pending,
3372 &mut pending_bash_asks,
3373 &mut route_bash_cancels,
3374 &active_tool_calls,
3375 &mut pending_responses,
3376 &mut retry_buffer,
3377 &mut push_buffer,
3378 &shutdown,
3379 &control_completion_tx,
3380 &dispatch_path_metrics,
3381 lifecycle_probe.as_ref(),
3382 &health_rollup_cache,
3383 &push_senders,
3384 dispatch,
3385 user_config_path.as_deref(),
3386 tool_response_body_limit,
3387 )
3388 .await
3389 {
3390 break Err(error);
3391 }
3392 }
3393 FrameType::Request => {
3394 let route = route_key(frame.header.channel, frame.header.epoch);
3395 let result = if management_routes.contains(&route) {
3396 handle_management_request(
3397 &writer_tx,
3398 &frame,
3399 &shared_app,
3400 &executor,
3401 &live_roots,
3402 &root_channels,
3403 &health_rollup_cache,
3404 &dispatch_path_metrics,
3405 )
3406 .await
3407 } else {
3408 handle_tool_call(
3409 &writer_tx,
3410 &frame,
3411 phase_trace,
3412 &routes,
3413 &pending_binds,
3414 &mut live_roots,
3415 &executor,
3416 &active_tool_calls,
3417 &pending_deferred_setups,
3418 &shutdown,
3419 &connection_cancel,
3420 &bash_deferred_tx,
3421 &bash_poll_touch_tx,
3422 &dispatch_path_metrics,
3423 &mut route_bash_cancels,
3424 &mut pending_bash_asks,
3425 &mut next_bash_ask_corr,
3426 &mut bg_subs,
3427 &mut bg_sub_by_session,
3428 &mut bg_wake_pending,
3429 &mut bg_wake_epoch,
3430 dispatch,
3431 &deferred_response_tx,
3432 allow_native_passthrough,
3433 tool_response_body_limit,
3434 )
3435 .await
3436 };
3437 if let Err(error) = result {
3438 break Err(error);
3439 }
3440 }
3441 FrameType::Cancel => {
3442 let channel = route_key(frame.header.channel, frame.header.epoch);
3443 cancel_active_tool_call(
3444 &active_tool_calls,
3445 executor.as_ref(),
3446 channel,
3447 frame.header.corr,
3448 "Cancel frame",
3449 );
3450 pending_responses.cancel_request(channel, frame.header.corr);
3451 if bg_subs.contains_key(&channel) {
3452 if let Err(error) = end_bg_subscription(
3453 &writer_tx,
3454 &dispatch_path_metrics,
3455 &mut bg_subs,
3456 &mut bg_sub_by_session,
3457 &mut bg_wake_pending,
3458 channel,
3459 routes.get(&channel),
3460 "cancel",
3461 )
3462 .await
3463 {
3464 break Err(error);
3465 }
3466 }
3467 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3468 &writer_tx,
3469 &mut pending_bash_asks,
3470 channel,
3471 frame.header.corr,
3472 &routes,
3473 &mut live_roots,
3474 &mut route_bash_cancels,
3475 &shutdown,
3476 &dispatch_path_metrics,
3477 )
3478 .await
3479 {
3480 break Err(error);
3481 }
3482 }
3483 _ => {}
3487 }
3488 }
3489 Some(pending) = deferred_response_rx.recv() => {
3490 if routes.contains_key(&pending.route)
3491 && active_tool_call_is_registered(
3492 &active_tool_calls,
3493 pending.route,
3494 pending.corr,
3495 )
3496 {
3497 pending_responses.register(pending);
3498 } else {
3499 if let Some(cancellation) = &pending.pending.cancellation {
3500 cancellation.request_cancel();
3501 }
3502 finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3503 }
3504 }
3505 Some((root_id, frame)) = reliable_rx.recv() => {
3506 let (_, deferred) = push::drain_reliable_push_turn(
3510 &writer_tx,
3511 &dispatch_path_metrics,
3512 &routes,
3513 &root_channels,
3514 &session_identity,
3515 &mut retry_buffer,
3516 &mut push_buffer,
3517 &mut completed_tasks,
3518 &bg_sub_by_session,
3519 &mut bg_wake_pending,
3520 &mut bg_wake_epoch,
3521 &mut reliable_rx,
3522 Some((root_id, frame)),
3523 lifecycle_probe.as_ref(),
3524 );
3525 if deferred {
3526 tokio::task::yield_now().await;
3527 }
3528 }
3529 Some((order, root_id, frame)) = lossy_rx.recv() => {
3530 let (_, deferred) = push::drain_reliable_push_turn(
3534 &writer_tx,
3535 &dispatch_path_metrics,
3536 &routes,
3537 &root_channels,
3538 &session_identity,
3539 &mut retry_buffer,
3540 &mut push_buffer,
3541 &mut completed_tasks,
3542 &bg_sub_by_session,
3543 &mut bg_wake_pending,
3544 &mut bg_wake_epoch,
3545 &mut reliable_rx,
3546 None,
3547 lifecycle_probe.as_ref(),
3548 );
3549 if deferred {
3550 tokio::task::yield_now().await;
3551 }
3552
3553 let mut batch = vec![(order, root_id, frame)];
3560 while let Ok(item) = lossy_rx.try_recv() {
3561 batch.push(item);
3562 }
3563 batch.extend(lossy_overflow.drain());
3564 push::process_lossy_push_envelope_batch(
3565 &writer_tx,
3566 &dispatch_path_metrics,
3567 &routes,
3568 &root_channels,
3569 &completed_tasks,
3570 batch,
3571 );
3572 }
3573 Some(done) = bash_deferred_rx.recv() => {
3574 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3575 if let Err(error) = bash::handle_bash_deferred_completion(
3576 &writer_tx,
3577 done,
3578 &routes,
3579 &mut live_roots,
3580 &mut route_bash_cancels,
3581 &shutdown,
3582 &dispatch_path_metrics,
3583 )
3584 .await
3585 {
3586 break Err(error);
3587 }
3588 }
3589 Some(root_id) = bash_poll_touch_rx.recv() => {
3590 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3591 if let Some(meta) = live_roots.get_mut(&root_id) {
3592 meta.note_activity();
3593 }
3594 }
3595 Some(completion) = maintenance_rx.recv() => {
3596 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3597 let root_id = completion.root_id.clone();
3598 let response = completion.response;
3599 let response_is_fatal = response_is_fatal_panic(&response);
3600 let bind_pending = pending_binds
3601 .values()
3602 .any(|pending| pending.bind_root_id == root_id);
3603 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3604 let defer_requeue = meta.unbound_quiesced || bind_pending;
3605 note_maintenance_completion(
3606 meta,
3607 completion.requeue_kind,
3608 response_is_fatal,
3609 defer_requeue,
3610 );
3611 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3612 } else {
3613 false
3614 };
3615 if requiesce {
3616 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3617 }
3618 push::clear_stale_bg_wakes_for_empty_sessions(
3619 &root_id,
3620 &completion.empty_bg_sessions,
3621 &bg_sub_by_session,
3622 &mut bg_wake_pending,
3623 &bg_wake_epoch,
3624 );
3625 if let Some(keys) = completion.unacked_bg_keys {
3626 bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3627 }
3628 record_bg_runtime_from_snapshots(
3629 &dispatch_path_metrics,
3630 bg_subs.len(),
3631 bg_wake_pending.len(),
3632 &bg_unacked_keys_by_root,
3633 );
3634 if response_is_fatal {
3635 if let Some(meta) = live_roots.get_mut(&root_id) {
3636 meta.maintenance_poisoned = true;
3637 }
3638 log::warn!(
3639 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3640 );
3641 }
3642 }
3643 _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3644 }
3647 _ = tokio::time::sleep_until(next_drain_at) => {
3648 }
3651 _ = tokio::time::sleep_until(next_maintenance_at) => {
3652 crate::logging::maybe_sweep_logs();
3657 crate::db::compression_events::maybe_spawn_retention(shared_app.db());
3658 let reaped_lsp_children = shared_app
3659 .lsp_child_registry()
3660 .reap_children_with_gone_cwd_or_reclaimed_root();
3661 if reaped_lsp_children > 0 {
3662 log::warn!(
3663 "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3664 );
3665 }
3666 let now = Instant::now();
3667 reap_idle_lsp_servers(now, &live_roots, &executor);
3668 let reap = reap_idle_roots(
3669 now,
3670 &mut live_roots,
3671 &pending_binds,
3672 &root_channels,
3673 &executor,
3674 &dispatch_path_metrics,
3675 );
3676 for root_id in &reap.forgotten_deleted_roots {
3677 bg_unacked_keys_by_root.remove(root_id);
3678 purge_deleted_root_residents(
3679 root_id,
3680 &mut routes,
3681 &mut root_channels,
3682 &mut installed_route_epochs,
3683 &mut route_bash_cancels,
3684 &active_tool_calls,
3685 executor.as_ref(),
3686 &mut retry_buffer,
3687 &mut reclaimed_routes,
3688 &mut session_identity,
3689 &mut push_buffer,
3690 &mut bg_subs,
3691 &mut bg_sub_by_session,
3692 &mut bg_wake_pending,
3693 &mut bg_wake_epoch,
3694 &mut pending_bash_asks,
3695 &dispatch_path_metrics,
3696 );
3697 }
3698 if reap.evicted > 0 {
3699 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3700 }
3701 record_bg_runtime_from_snapshots(
3702 &dispatch_path_metrics,
3703 bg_subs.len(),
3704 bg_wake_pending.len(),
3705 &bg_unacked_keys_by_root,
3706 );
3707 submit_due_maintenance_jobs(
3708 &executor,
3709 &mut live_roots,
3710 &pending_binds,
3711 &bg_sub_by_session,
3712 &bg_wake_pending,
3713 &bg_wake_epoch,
3714 &maintenance_tx,
3715 &dispatch_path_metrics,
3716 );
3717 if tokio::time::Instant::now() >= next_standing_pass_at {
3718 standing_actor.tick();
3719 next_standing_pass_at = tokio::time::Instant::now()
3720 + standing::STANDING_MAINTENANCE_INTERVAL;
3721 }
3722 #[cfg(any(target_os = "macos", target_os = "linux"))]
3729 {
3730 let now_std = std::time::Instant::now();
3731 let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3732 }
3733 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3734 }
3735 }
3736 };
3737
3738 shared_app.set_open_route_count(0);
3739 health_rollup_worker.shutdown();
3740
3741 connection_cancel.cancel();
3742 cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3743 let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3744 while pending_deferred_setups.load(Ordering::SeqCst) != 0
3745 && tokio::time::Instant::now() < setup_drain_deadline
3746 {
3747 tokio::select! {
3748 biased;
3749 Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3750 _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3751 }
3752 }
3753 if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3754 log::warn!(
3755 "subc attach: timed out waiting for deferred response setup registration during shutdown"
3756 );
3757 }
3758 while let Ok(pending) = deferred_response_rx.try_recv() {
3759 pending_responses.register(pending);
3760 }
3761 for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3762 if let Err(error) = deliver_resolved_subc_response(
3763 &writer_tx,
3764 resolved,
3765 &routes,
3766 &mut live_roots,
3767 executor.as_ref(),
3768 &active_tool_calls,
3769 &shutdown,
3770 &dispatch_path_metrics,
3771 tool_response_body_limit,
3772 )
3773 .await
3774 {
3775 log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3776 }
3777 }
3778 quiesce_connection_roots(
3781 &mut live_roots,
3782 &mut pending_binds,
3783 &mut routes,
3784 &mut root_channels,
3785 &mut installed_route_epochs,
3786 &mut route_bash_cancels,
3787 &active_tool_calls,
3788 &executor,
3789 );
3790
3791 fleet_status_client.set_route_live(false);
3792 fleet_status_task.abort();
3793 let _ = fleet_status_task.await;
3794
3795 let mut loop_result = loop_result;
3796 if !pending_bash_asks.is_empty() {
3797 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3798 if let Err(error) = settle_all_pending_bash_asks(
3799 &writer_tx,
3800 &mut pending_bash_asks,
3801 &no_routes,
3802 &mut live_roots,
3803 &mut route_bash_cancels,
3804 &shutdown,
3805 &dispatch_path_metrics,
3806 )
3807 .await
3808 {
3809 loop_result = loop_result.and(Err(error));
3810 }
3811 }
3812
3813 reader_task.abort();
3816 drop(writer_tx);
3817 let writer_result = finish_writer_task(writer_task).await;
3818 loop_result.and_then(|exit| writer_result.map(|_| exit))
3819}
3820
3821fn spawn_writer_task<W>(
3822 mut write: W,
3823 mut rx: mpsc::Receiver<WriterFrame>,
3824 metrics: Arc<DispatchPathMetrics>,
3825) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3826where
3827 W: AsyncWrite + Unpin + Send + 'static,
3828{
3829 tokio::spawn(async move {
3830 let mut write_buffer = Vec::new();
3831 while let Some(mut queued) = rx.recv().await {
3832 let measure = queued.tool_response_trace.is_some();
3833 let dequeued = measure.then(Instant::now);
3834 metrics.writer_active.store(true, Ordering::Relaxed);
3835 decrement_counted_channel(&metrics.writer_queued);
3836 let write_timing = write_frame_contiguous(
3837 &mut write,
3838 queued.frame(),
3839 queued.body(),
3840 &mut write_buffer,
3841 measure,
3842 )
3843 .await;
3844 metrics.writer_active.store(false, Ordering::Relaxed);
3845 let write_timing = write_timing?;
3846
3847 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3848 (queued.tool_response_trace.take(), dequeued, write_timing)
3849 {
3850 if let Some(completed) = trace.finish(
3851 dequeued,
3852 write_timing.write_started,
3853 write_timing.write_finished,
3854 write_timing.frame_bytes,
3855 ) {
3856 log_ctx::with_session(Some(completed.session), || {
3857 crate::logging::note_tool_call_trace(
3858 &completed.name,
3859 &completed.root,
3860 completed.channel,
3861 completed.corr,
3862 completed.phases,
3863 );
3864 });
3865 }
3866 }
3867 }
3868 Ok(())
3869 })
3870}
3871
3872struct FrameWriteTiming {
3873 write_started: Instant,
3874 write_finished: Instant,
3875 frame_bytes: usize,
3876}
3877
3878async fn write_frame_contiguous<W>(
3882 writer: &mut W,
3883 frame: &Frame,
3884 body: &[u8],
3885 buffer: &mut Vec<u8>,
3886 measure: bool,
3887) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3888where
3889 W: AsyncWrite + Unpin,
3890{
3891 if frame.header.len as usize != body.len() {
3892 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3893 header_len: frame.header.len,
3894 body_len: body.len(),
3895 });
3896 }
3897
3898 let header = frame.header.encode();
3899 buffer.clear();
3900 buffer.reserve(header.len() + body.len());
3901 buffer.extend_from_slice(&header);
3902 buffer.extend_from_slice(body);
3903 let write_started = measure.then(Instant::now);
3904 writer
3905 .write_all(buffer)
3906 .await
3907 .map_err(subc_transport::FrameIoError::Io)?;
3908 Ok(write_started.map(|write_started| FrameWriteTiming {
3909 write_started,
3910 write_finished: Instant::now(),
3911 frame_bytes: buffer.len(),
3912 }))
3913}
3914
3915fn spawn_reader_task<R>(
3916 mut read: R,
3917 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3918) -> JoinHandle<()>
3919where
3920 R: AsyncRead + Unpin + Send + 'static,
3921{
3922 tokio::spawn(async move {
3923 loop {
3924 match read_frame(&mut read).await {
3925 Ok(Some(frame)) => {
3926 let decoded = DecodedFrame {
3927 frame,
3928 phase_trace: PhaseTrace::new(Instant::now()),
3929 };
3930 if tx.send(Ok(decoded)).await.is_err() {
3931 return;
3932 }
3933 }
3934 Ok(None) => {
3935 return;
3937 }
3938 Err(error) => {
3939 if let subc_transport::FrameIoError::Io(io_error) = &error {
3946 if matches!(
3947 io_error.kind(),
3948 std::io::ErrorKind::ConnectionReset
3949 | std::io::ErrorKind::ConnectionAborted
3950 ) {
3951 log::info!(
3952 "subc attach: connection reset by daemon; treating as close"
3953 );
3954 return;
3955 }
3956 }
3957 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3958 return;
3959 }
3960 }
3961 }
3962 })
3963}
3964
3965async fn finish_writer_task(
3966 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3967) -> Result<(), SubcError> {
3968 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3969 Ok(Ok(Ok(()))) => Ok(()),
3970 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3971 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3972 Err(_) => {
3973 writer_task.abort();
3974 Ok(())
3975 }
3976 }
3977}
3978
3979fn register_actor_for_bind(
3980 shared_app: &Arc<App>,
3981 executor: &Arc<Executor>,
3982 push_senders: &PushSenders,
3983 bind_root_id: &ProjectRootId,
3984 route_channel: u16,
3985 root_was_live: bool,
3986) -> bool {
3987 if executor.actor_registered(bind_root_id) {
3988 log::debug!(
3989 "subc attach: reusing actor for route {} root {}",
3990 route_channel,
3991 bind_root_id.as_path().display()
3992 );
3993 return false;
3994 }
3995
3996 if root_was_live {
3997 log::warn!(
3998 "subc attach: recreating missing actor for live root {} on route {}",
3999 bind_root_id.as_path().display(),
4000 route_channel
4001 );
4002 }
4003
4004 let actor_ctx = Arc::new(AppContext::from_app(
4005 Arc::clone(shared_app),
4006 Config::default(),
4007 ));
4008 install_bash_compressor(&actor_ctx);
4009 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4010 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4011 push_senders.clone(),
4012 bind_root_id.clone(),
4013 )));
4014 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4015 drop(actor_ctx);
4016 if inserted {
4017 log::debug!(
4021 "subc attach: registered actor for route {} root {}",
4022 route_channel,
4023 bind_root_id.as_path().display()
4024 );
4025 } else {
4026 log::debug!(
4027 "subc attach: actor appeared while binding route {} root {}; reusing it",
4028 route_channel,
4029 bind_root_id.as_path().display()
4030 );
4031 }
4032 inserted
4033}
4034
4035fn rollback_pending_bind_actor(
4036 executor: &Arc<Executor>,
4037 live_roots: &HashMap<ProjectRootId, RootMeta>,
4038 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4039 root_id: &ProjectRootId,
4040 inserted_new_actor: bool,
4041) {
4042 if !inserted_new_actor || live_roots.contains_key(root_id) {
4043 return;
4044 }
4045
4046 if let Some((route, pending)) = pending_binds
4047 .iter_mut()
4048 .find(|(_, pending)| &pending.bind_root_id == root_id)
4049 {
4050 pending.inserted_new_actor = true;
4051 log::debug!(
4052 "subc attach: transferred rollback ownership for root {} to pending route {}",
4053 root_id.as_path().display(),
4054 route
4055 );
4056 return;
4057 }
4058
4059 executor.remove_actor(root_id);
4060}
4061
4062fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4063 match response.data.get("code").and_then(|code| code.as_str()) {
4064 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4069 Some("cache_key_probe_failed") => "cache_key_probe_failed",
4073 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4077 _ => "config_divergence",
4078 }
4079}
4080
4081fn queue_post_bind_configure_and_completion_maintenance(
4082 root_id: &ProjectRootId,
4083 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4084) {
4085 let Some(meta) = live_roots.get_mut(root_id) else {
4086 return;
4087 };
4088 if meta.maintenance_poisoned || meta.maintenance_pending {
4089 return;
4090 }
4091
4092 meta.maintenance_pending = true;
4093 meta.maintenance_queued_kinds
4094 .push_back(MaintenanceDrainKind::ConfigureTail);
4095 meta.maintenance_queued_kinds
4096 .push_back(MaintenanceDrainKind::CompletionDrains);
4097}
4098
4099#[allow(clippy::too_many_arguments)]
4100async fn handle_route_bind_completion(
4101 tx: &WriterSender,
4102 completion: RouteBindCompletion,
4103 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4104 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4105 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4106 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4107 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4108 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4109 installed_route_epochs: &mut HashMap<u16, u32>,
4110 executor: &Arc<Executor>,
4111 standing_actor: &standing::StandingActor,
4112 shutdown: &Arc<Notify>,
4113 metrics: &Arc<DispatchPathMetrics>,
4114 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4115) -> Result<(), SubcError> {
4116 let route_id = completion.route;
4117 let Some(pending) = pending_binds.remove(&route_id) else {
4118 log::warn!(
4119 "subc attach: dropping RouteBind completion for non-pending route {}",
4120 completion.route
4121 );
4122 rollback_pending_bind_actor(
4123 executor,
4124 live_roots,
4125 pending_binds,
4126 &completion.bind_root_id,
4127 completion.inserted_new_actor,
4128 );
4129 let has_pending_bind = pending_binds
4130 .values()
4131 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4132 if !root_channels
4133 .get(&completion.bind_root_id)
4134 .is_some_and(|channels| !channels.is_empty())
4135 && !has_pending_bind
4136 {
4137 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4138 }
4139 remove_installed_route(installed_route_epochs, route_id);
4140 return Ok(());
4141 };
4142
4143 if pending.bind_root_id != completion.bind_root_id {
4144 log::warn!(
4145 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4146 completion.route,
4147 pending.bind_root_id.as_path().display(),
4148 completion.bind_root_id.as_path().display()
4149 );
4150 }
4151
4152 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4153 if pending.cancelled {
4154 rollback_pending_bind_actor(
4155 executor,
4156 live_roots,
4157 pending_binds,
4158 &completion.bind_root_id,
4159 inserted_new_actor,
4160 );
4161 let has_pending_bind = pending_binds
4162 .values()
4163 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4164 if !root_channels
4165 .get(&completion.bind_root_id)
4166 .is_some_and(|channels| !channels.is_empty())
4167 && !has_pending_bind
4168 {
4169 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4170 }
4171 log::debug!(
4172 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4173 completion.route,
4174 completion.bind_root_id.as_path().display()
4175 );
4176 remove_installed_route(installed_route_epochs, route_id);
4177 return Ok(());
4178 }
4179
4180 let failure = if !completion.configure_response.success {
4181 Some((
4182 &completion.configure_response,
4183 "configure failed during route bind",
4184 ))
4185 } else {
4186 None
4187 };
4188
4189 if let Some((response, fallback)) = failure {
4190 rollback_pending_bind_actor(
4191 executor,
4192 live_roots,
4193 pending_binds,
4194 &completion.bind_root_id,
4195 inserted_new_actor,
4196 );
4197 let has_pending_bind = pending_binds
4198 .values()
4199 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4200 if !root_channels
4201 .get(&completion.bind_root_id)
4202 .is_some_and(|channels| !channels.is_empty())
4203 && !has_pending_bind
4204 {
4205 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4206 }
4207 let message = response_message(response, fallback);
4208 let fatal = response_is_fatal_panic(response);
4209 let error_code = route_bind_error_code_for_configure_response(response);
4210 send_route_bind_error_parts(
4211 tx,
4212 completion.ver,
4213 completion.corr,
4214 completion.flags,
4215 error_code,
4216 &message,
4217 metrics,
4218 )
4219 .await?;
4220 remove_installed_route(installed_route_epochs, route_id);
4221 if fatal {
4222 signal_fatal_teardown(
4223 tx,
4224 Some(completion.route),
4225 completion.ver,
4226 completion.corr,
4227 shutdown,
4228 metrics,
4229 )
4230 .await;
4231 }
4232 return Ok(());
4233 }
4234
4235 remember_session_identity(session_identity, &completion.identity);
4236 let replay_key = push::ReplayKey::from_identity(&completion.identity);
4237 let bind_trust = completion.identity.trust;
4238 insert_route_channel(routes, root_channels, route_id, completion.identity);
4239 sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4240 let restore_watcher = live_roots
4241 .get(&completion.bind_root_id)
4242 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4243 live_roots
4244 .entry(completion.bind_root_id.clone())
4245 .and_modify(|meta| {
4246 meta.reactivate_bound();
4247 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4248 meta.maintenance_poisoned = false;
4249 })
4250 .or_insert_with(|| RootMeta::new(Instant::now()));
4251 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4252 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4253 meta.maintenance_poisoned = false;
4254 }
4255 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4256 standing_actor.begin_session_bind(&ctx);
4259 ctx.mark_subc_bound();
4260 if restore_watcher {
4261 crate::commands::configure::ensure_project_watcher(&ctx);
4262 }
4263 }
4264
4265 let ack =
4266 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4267 let response = Frame::build_with_version(
4268 completion.ver,
4269 FrameType::Response,
4270 control_flags(),
4271 0,
4272 0,
4273 completion.corr,
4274 ack,
4275 )
4276 .map_err(SubcError::FrameBuild)?;
4277 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4278 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4279 let replayed = push::replay_buffered_push_frames(
4280 tx,
4281 metrics,
4282 route_id,
4283 push_buffer,
4284 &replay_key,
4285 bind_trust,
4286 lifecycle_probe,
4287 );
4288 if replayed > 0 {
4289 log::debug!(
4290 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4291 replayed,
4292 completion.route,
4293 replay_key.root.as_path().display(),
4294 replay_key.harness,
4295 replay_key.session
4296 );
4297 }
4298 log::info!(
4299 "subc attach: route {} bound to root {}",
4300 completion.route,
4301 completion.bind_root_id.as_path().display()
4302 );
4303 Ok(())
4304}
4305
4306async fn expire_overdue_route_binds(
4307 tx: &WriterSender,
4308 executor: &Arc<Executor>,
4309 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4310 installed_route_epochs: &mut HashMap<u16, u32>,
4311 metrics: &DispatchPathMetrics,
4312) -> Result<(), SubcError> {
4313 let now = Instant::now();
4314 let expired: Vec<_> = pending_binds
4315 .iter()
4316 .filter_map(|(route, pending)| {
4317 let age = now.saturating_duration_since(pending.started_at);
4318 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4319 (
4320 *route,
4321 pending.corr,
4322 pending.ver,
4323 pending.flags,
4324 pending.bind_root_id.clone(),
4325 pending.configure_request_id.clone(),
4326 age,
4327 )
4328 })
4329 })
4330 .collect();
4331
4332 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4333 if let Some(pending) = pending_binds.get_mut(&route) {
4334 pending.cancelled = true;
4335 pending.deadline_reported = true;
4336 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4337 log::debug!(
4338 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4339 );
4340 }
4341 remove_installed_route(installed_route_epochs, route);
4342 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4343 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4344 send_route_bind_error_parts(
4345 tx,
4346 ver,
4347 corr,
4348 flags,
4349 "actor_not_ready",
4350 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4351 metrics,
4352 )
4353 .await?;
4354 log::warn!(
4355 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4356 route,
4357 root_id.as_path().display(),
4358 deadline_ms,
4359 configure_request_id
4360 );
4361 }
4362
4363 Ok(())
4364}
4365
4366fn record_bg_runtime_from_snapshots(
4367 metrics: &DispatchPathMetrics,
4368 subscriptions: usize,
4369 wake_pending: usize,
4370 unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4371) {
4372 let unacked_total = unacked_keys_by_root
4373 .values()
4374 .flatten()
4375 .collect::<HashSet<_>>()
4376 .len();
4377 metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4378}
4379
4380async fn send_cached_health_response(
4381 tx: &WriterSender,
4382 frame: &Frame,
4383 shared_app: &App,
4384 executor: &Executor,
4385 pending_binds: &HashMap<RouteChannel, PendingBind>,
4386 metrics: &DispatchPathMetrics,
4387 health_rollup_cache: &HealthRollupCache,
4388) -> Result<(), SubcError> {
4389 let report = build_health_report(
4390 health_rollup_cache,
4391 executor,
4392 pending_binds,
4393 metrics,
4394 shared_app,
4395 );
4396 let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4397 let response = Frame::build_with_version(
4398 frame.header.ver,
4399 FrameType::Response,
4400 frame.header.flags,
4401 0,
4402 0,
4403 frame.header.corr,
4404 body,
4405 )
4406 .map_err(SubcError::FrameBuild)?;
4407 send_frame(tx, metrics, response).await
4408}
4409
4410#[allow(clippy::too_many_arguments)]
4414async fn handle_control_request(
4415 tx: &WriterSender,
4416 frame: &Frame,
4417 shared_app: &Arc<App>,
4418 executor: &Arc<Executor>,
4419 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4420 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4421 installed_route_epochs: &mut HashMap<u16, u32>,
4422 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4423 management_routes: &mut HashSet<RouteChannel>,
4424 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4425 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4426 bg_sub_by_session: &mut BgSubsBySession,
4427 bg_wake_pending: &mut BgWakePending,
4428 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4429 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4430 active_tool_calls: &ActiveToolCalls,
4431 pending_responses: &mut PendingSubcResponses,
4432 retry_buffer: &mut RetryBuffer,
4433 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4434 shutdown: &Arc<Notify>,
4435 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4436 metrics: &Arc<DispatchPathMetrics>,
4437 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4438 health_rollup_cache: &HealthRollupCache,
4439 push_senders: &PushSenders,
4440 dispatch: DispatchFn,
4441 user_config_path: Option<&Path>,
4442 tool_response_body_limit: usize,
4443) -> Result<(), SubcError> {
4444 let request =
4445 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4446 match request {
4447 ModuleControlRequest::RouteBind {
4448 route_channel,
4449 epoch,
4450 target,
4451 identity,
4452 principal,
4453 consumer_capabilities,
4454 admission_facts: _,
4455 } => {
4456 let route_id = route_key(route_channel, epoch);
4457 if epoch == 0 {
4458 return send_route_bind_error(
4459 tx,
4460 frame,
4461 "config_divergence",
4462 "route bind uses an invalid channel generation",
4463 metrics,
4464 )
4465 .await;
4466 }
4467
4468 let bind_trust = trust_for_bind(&identity.harness, &principal);
4469 if let RouteTarget::ManagementSurface { module_id } = &target {
4470 if module_id != "aft" {
4471 return send_route_bind_error(
4472 tx,
4473 frame,
4474 "route_refused",
4475 "management route target is not AFT",
4476 metrics,
4477 )
4478 .await;
4479 }
4480 if !matches!(bind_trust, BindTrust::FirstParty) {
4481 return send_route_bind_error(
4482 tx,
4483 frame,
4484 "route_refused",
4485 "AFT management routes require a first-party principal",
4486 metrics,
4487 )
4488 .await;
4489 }
4490 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4491 if installed_epoch >= epoch {
4492 return send_route_bind_error(
4493 tx,
4494 frame,
4495 "config_divergence",
4496 "route bind generation is not newer than the installed generation",
4497 metrics,
4498 )
4499 .await;
4500 }
4501 teardown_installed_route(
4502 tx,
4503 metrics,
4504 executor,
4505 route_key(route_channel, installed_epoch),
4506 "higher-epoch RouteBind",
4507 None,
4508 installed_route_epochs,
4509 routes,
4510 management_routes,
4511 root_channels,
4512 bg_subs,
4513 bg_sub_by_session,
4514 bg_wake_pending,
4515 pending_bash_asks,
4516 live_roots,
4517 route_bash_cancels,
4518 active_tool_calls,
4519 pending_responses,
4520 pending_binds,
4521 retry_buffer,
4522 push_buffer,
4523 shutdown,
4524 tool_response_body_limit,
4525 lifecycle_probe,
4526 )
4527 .await?;
4528 }
4529 if pending_binds.contains_key(&route_id) {
4530 return send_route_bind_error(
4531 tx,
4532 frame,
4533 "config_divergence",
4534 "route bind is already pending for channel",
4535 metrics,
4536 )
4537 .await;
4538 }
4539
4540 installed_route_epochs.insert(route_channel, epoch);
4541 management_routes.insert(route_id);
4542 return send_route_bind_ack(
4543 tx,
4544 frame.header.ver,
4545 frame.header.corr,
4546 frame.header.flags,
4547 metrics,
4548 )
4549 .await;
4550 }
4551 if matches!(&target, RouteTarget::InternalService { .. }) {
4552 return send_route_bind_error(
4553 tx,
4554 frame,
4555 "route_refused",
4556 "AFT does not provide an internal-service route",
4557 metrics,
4558 )
4559 .await;
4560 }
4561
4562 let mut bind_root_id = None;
4563 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4564 if installed_epoch >= epoch {
4565 return send_route_bind_error(
4566 tx,
4567 frame,
4568 "config_divergence",
4569 "route bind generation is not newer than the installed generation",
4570 metrics,
4571 )
4572 .await;
4573 }
4574
4575 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4576 Ok(root_id) => root_id,
4577 Err(error) => {
4578 return send_route_bind_error(
4579 tx,
4580 frame,
4581 "config_divergence",
4582 &format!("invalid route project root: {error}"),
4583 metrics,
4584 )
4585 .await;
4586 }
4587 };
4588 teardown_installed_route(
4589 tx,
4590 metrics,
4591 executor,
4592 route_key(route_channel, installed_epoch),
4593 "higher-epoch RouteBind",
4594 Some(&replacement_root),
4595 installed_route_epochs,
4596 routes,
4597 management_routes,
4598 root_channels,
4599 bg_subs,
4600 bg_sub_by_session,
4601 bg_wake_pending,
4602 pending_bash_asks,
4603 live_roots,
4604 route_bash_cancels,
4605 active_tool_calls,
4606 pending_responses,
4607 pending_binds,
4608 retry_buffer,
4609 push_buffer,
4610 shutdown,
4611 tool_response_body_limit,
4612 lifecycle_probe,
4613 )
4614 .await?;
4615 bind_root_id = Some(replacement_root);
4616 }
4617 if pending_binds.contains_key(&route_id) {
4618 return send_route_bind_error(
4619 tx,
4620 frame,
4621 "config_divergence",
4622 "route bind is already pending for channel",
4623 metrics,
4624 )
4625 .await;
4626 }
4627 let bind_root_id = match bind_root_id {
4628 Some(root_id) => root_id,
4629 None => match ProjectRootId::from_path(&identity.project_root) {
4630 Ok(root_id) => root_id,
4631 Err(error) => {
4632 return send_route_bind_error(
4633 tx,
4634 frame,
4635 "config_divergence",
4636 &format!("invalid route project root: {error}"),
4637 metrics,
4638 )
4639 .await;
4640 }
4641 },
4642 };
4643
4644 let request_id = format!("subc-bind-{route_channel}");
4647 let bind_project_root = identity.project_root.clone();
4648 let bind_harness = identity.harness.clone();
4649 let bind_session = identity.session.clone();
4650 let bind_principal_id = principal_id(&principal);
4651 let consumer_elicitation_capable = consumer_capabilities
4656 .as_ref()
4657 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4658 log::info!(
4659 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4660 route_channel,
4661 bind_harness,
4662 principal_label(&principal),
4663 bind_trust.label(),
4664 consumer_elicitation_capable
4665 );
4666
4667 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4672 user_config_path,
4673 Path::new(&bind_project_root),
4674 );
4675 let config_tiers: Vec<Value> = local_tiers
4676 .iter()
4677 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4678 .collect();
4679 let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4682 let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4683 &local_tiers,
4684 active_harness.as_ref(),
4685 )
4686 .config
4687 .diagnostics_on_edit;
4688 let configure_json = json!({
4689 "id": request_id,
4690 "command": "configure",
4691 "project_root": bind_project_root,
4692 "harness": bind_harness,
4693 "session_id": bind_session.clone(),
4694 "config": config_tiers,
4695 });
4696 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4697 Ok(req) => req,
4698 Err(error) => {
4699 return send_route_bind_error(
4700 tx,
4701 frame,
4702 "config_divergence",
4703 &format!("failed to build configure request: {error}"),
4704 metrics,
4705 )
4706 .await;
4707 }
4708 };
4709
4710 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4711 root: bind_root_id.clone(),
4712 project_root: PathBuf::from(&bind_project_root),
4713 harness: bind_harness.clone(),
4714 session: bind_session.clone(),
4715 trust: bind_trust,
4716 spawn_principal: AuthenticatedPrincipal::RouteBind {
4717 trust: bind_trust.sandbox_trust(),
4718 route_channel,
4719 route_epoch: epoch,
4720 project_root: PathBuf::from(&bind_project_root),
4721 harness: bind_harness.clone(),
4722 session_id: bind_session.clone(),
4723 principal_id: bind_principal_id,
4724 },
4725 consumer_elicitation_capable,
4726 }));
4727 let configure_session = route_identity.session.clone();
4728 let root_was_live = live_roots.contains_key(&bind_root_id);
4729 let inserted_new_actor = register_actor_for_bind(
4730 shared_app,
4731 executor,
4732 push_senders,
4733 &bind_root_id,
4734 route_channel,
4735 root_was_live,
4736 );
4737
4738 sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4739 let configure_request_id = configure_req.id.clone();
4740 installed_route_epochs.insert(route_channel, epoch);
4741 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4742 meta.maintenance_queued_kinds.clear();
4743 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4744 }
4745 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4746 bind_root_id.clone(),
4747 Lane::Mutating,
4748 configure_request_id.clone(),
4749 Box::new(move |ctx| {
4750 log_ctx::with_session(Some(configure_session.clone()), || {
4751 dispatch(configure_req, ctx)
4752 })
4753 }),
4754 );
4755 pending_binds.insert(
4756 route_id,
4757 PendingBind {
4758 bind_root_id: bind_root_id.clone(),
4759 inserted_new_actor,
4760 cancelled: false,
4761 configure_request_id: configure_request_id.clone(),
4762 started_at: Instant::now(),
4763 warned_half_deadline: false,
4764 deadline_reported: false,
4765 corr: frame.header.corr,
4766 ver: frame.header.ver,
4767 flags: frame.header.flags,
4768 cancellation: configure_cancellation,
4769 },
4770 );
4771
4772 let completion_tx = control_completion_tx.clone();
4773 let completion_identity = route_identity;
4774 let completion_root = bind_root_id.clone();
4775 let completion_route_channel = route_channel;
4776 let completion_ver = frame.header.ver;
4777 let completion_corr = frame.header.corr;
4778 let completion_flags = frame.header.flags;
4779 let completion_metrics = Arc::clone(metrics);
4780 tokio::spawn(async move {
4781 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4782 let configure_response =
4783 await_executor_response(configure_rx, configure_request_id.clone()).await;
4784 let completion = RouteBindCompletion {
4789 route: route_key(completion_route_channel, epoch),
4790 identity: completion_identity,
4791 bind_root_id: completion_root,
4792 inserted_new_actor,
4793 configure_response,
4794 diagnostics_on_edit,
4795 ver: completion_ver,
4796 corr: completion_corr,
4797 flags: completion_flags,
4798 };
4799 if send_counted_channel(
4800 &completion_tx,
4801 &completion_metrics.control_completion_queued,
4802 completion,
4803 )
4804 .await
4805 .is_err()
4806 {
4807 log::debug!(
4808 "subc attach: dropped RouteBind completion for route {} after loop exit",
4809 completion_route_channel
4810 );
4811 }
4812 });
4813
4814 Ok(())
4817 }
4818 ModuleControlRequest::HealthCheck {} => {
4819 send_cached_health_response(
4820 tx,
4821 frame,
4822 shared_app,
4823 executor,
4824 pending_binds,
4825 metrics,
4826 health_rollup_cache,
4827 )
4828 .await
4829 }
4830 }
4831}
4832
4833async fn handle_management_request(
4834 tx: &WriterSender,
4835 frame: &Frame,
4836 shared_app: &App,
4837 executor: &Executor,
4838 live_roots: &HashMap<ProjectRootId, RootMeta>,
4839 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4840 health_rollup_cache: &HealthRollupCache,
4841 metrics: &DispatchPathMetrics,
4842) -> Result<(), SubcError> {
4843 let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4844 let operation = decoded
4845 .as_ref()
4846 .and_then(|value| value.get("op"))
4847 .and_then(Value::as_str);
4848 let Some(operation) = operation else {
4849 let error = build_error_frame(
4850 frame.header.ver,
4851 frame.header.channel,
4852 frame.header.epoch,
4853 frame.header.corr,
4854 frame.header.flags,
4855 "unknown_management_op",
4856 "management routes accept only declared operation envelopes",
4857 )?;
4858 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4859 };
4860
4861 let result = match operation {
4862 crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4863 "management-memory-census",
4864 memory_census_with_lifecycle(
4865 health_rollup_cache,
4866 shared_app,
4867 executor,
4868 live_roots,
4869 root_channels,
4870 ),
4871 ),
4872 crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4873 let params = decoded
4874 .as_ref()
4875 .and_then(|value| value.get("params"))
4876 .cloned()
4877 .unwrap_or_else(|| json!({}));
4878 let Some(params) = params.as_object() else {
4879 return send_management_response(
4880 tx,
4881 frame,
4882 operation,
4883 Response::error(
4884 "management-health-digest",
4885 "invalid_request",
4886 "health.digest params must be an object",
4887 ),
4888 metrics,
4889 )
4890 .await;
4891 };
4892 let root = params
4893 .get("project_root")
4894 .or_else(|| params.get("root"))
4895 .and_then(Value::as_str);
4896 let Some(root) = root else {
4897 return send_management_response(
4898 tx,
4899 frame,
4900 operation,
4901 Response::error(
4902 "management-health-digest",
4903 "invalid_request",
4904 "health.digest requires params.project_root",
4905 ),
4906 metrics,
4907 )
4908 .await;
4909 };
4910
4911 let mut request = params.clone();
4912 request.insert("id".to_string(), json!("management-health-digest"));
4913 request.insert(
4914 "command".to_string(),
4915 json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4916 );
4917 let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4918 .map_err(SubcError::Json)?;
4919 match ProjectRootId::from_path(Path::new(root))
4920 .ok()
4921 .and_then(|root_id| executor.actor_context(&root_id))
4922 {
4923 Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4924 None => crate::commands::health_digest::root_not_bound_response(&request, root),
4925 }
4926 }
4927 _ => {
4928 let error = build_error_frame(
4929 frame.header.ver,
4930 frame.header.channel,
4931 frame.header.epoch,
4932 frame.header.corr,
4933 frame.header.flags,
4934 "unknown_management_op",
4935 &format!("management operation {operation:?} is not declared by AFT"),
4936 )?;
4937 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4938 }
4939 };
4940
4941 send_management_response(tx, frame, operation, result, metrics).await
4942}
4943
4944fn memory_census_with_lifecycle(
4945 health_rollup_cache: &HealthRollupCache,
4946 shared_app: &App,
4947 executor: &Executor,
4948 live_roots: &HashMap<ProjectRootId, RootMeta>,
4949 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4950) -> Value {
4951 let mut census = health_rollup_cache.memory_census();
4952 if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4953 let lifecycle = shared_app.lifecycle_census_snapshot();
4954 for (root_id, meta) in live_roots {
4955 let root = root_id.as_path().display().to_string();
4956 let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4957 let age_ms = Instant::now()
4958 .saturating_duration_since(meta.last_touched)
4959 .as_millis()
4960 .min(u128::from(u64::MAX)) as u64;
4961 let ttl_ms = root_idle_ttl(executor, root_id)
4962 .as_millis()
4963 .min(u128::from(u64::MAX)) as u64;
4964 let lsp = lifecycle
4965 .lsp
4966 .children_by_root
4967 .iter()
4968 .find(|child| child.root == root);
4969 if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4970 let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4971 row.insert("root_id".to_string(), json!(root));
4972 row.insert("bound_routes".to_string(), json!(bound_routes));
4973 row.insert("last_request_age_ms".to_string(), json!(age_ms));
4974 row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4975 row.insert(
4976 "lsp_idle_ttl_ms".to_string(),
4977 json!(executor
4978 .actor_context(root_id)
4979 .map(|ctx| ctx
4980 .config()
4981 .idle
4982 .lsp_ttl()
4983 .as_millis()
4984 .min(u128::from(u64::MAX)) as u64)
4985 .unwrap_or(0)),
4986 );
4987 row.insert(
4988 "evictable_in_ms".to_string(),
4989 crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4990 .map_or(Value::Null, |value| json!(value)),
4991 );
4992 row.insert(
4993 "evictable_bytes".to_string(),
4994 if bound_routes == 0 {
4995 evictable_bytes
4996 } else {
4997 json!(0)
4998 },
4999 );
5000 row.insert(
5001 "lsp_children".to_string(),
5002 json!({
5003 "count": lsp.map_or(0, |child| child.count),
5004 "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5005 }),
5006 );
5007 }
5008 }
5009 }
5010 census
5011}
5012
5013async fn send_management_response(
5014 tx: &WriterSender,
5015 request: &Frame,
5016 operation: &str,
5017 result: Response,
5018 metrics: &DispatchPathMetrics,
5019) -> Result<(), SubcError> {
5020 let status = if result.success { "ok" } else { "error" };
5021 let body = json!({ "op": operation, "status": status, "data": result.data });
5022 let response = Frame::build_with_version(
5023 request.header.ver,
5024 FrameType::Response,
5025 request.header.flags,
5026 request.header.channel,
5027 request.header.epoch,
5028 request.header.corr,
5029 serde_json::to_vec(&body).map_err(SubcError::Json)?,
5030 )
5031 .map_err(SubcError::FrameBuild)?;
5032 send_reliable_writer_frame(tx, metrics, response, "management response").await
5033}
5034
5035fn install_bash_compressor(ctx: &AppContext) {
5036 let filter_registry_handle = ctx.shared_filter_registry();
5038 let compress_flag = ctx.bash_compress_flag();
5039 ctx.bash_background().set_compressor_with_exit_code(
5040 move |command: &str, output: String, exit_code: Option<i32>| {
5041 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5042 return crate::compress::CompressionResult::new(output);
5043 }
5044 let registry_guard = match filter_registry_handle.read() {
5045 Ok(g) => g,
5046 Err(poisoned) => poisoned.into_inner(),
5047 };
5048 crate::compress::compress_with_registry_exit_code(
5049 command,
5050 &output,
5051 exit_code,
5052 ®istry_guard,
5053 )
5054 },
5055 );
5056}
5057
5058async fn send_route_bind_ack(
5059 tx: &WriterSender,
5060 ver: u8,
5061 corr: u64,
5062 flags: Flags,
5063 metrics: &DispatchPathMetrics,
5064) -> Result<(), SubcError> {
5065 let body =
5066 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5067 let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5068 .map_err(SubcError::FrameBuild)?;
5069 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5070}
5071
5072async fn send_route_bind_error(
5073 tx: &WriterSender,
5074 frame: &Frame,
5075 code: &str,
5076 message: &str,
5077 metrics: &DispatchPathMetrics,
5078) -> Result<(), SubcError> {
5079 send_route_bind_error_parts(
5080 tx,
5081 frame.header.ver,
5082 frame.header.corr,
5083 frame.header.flags,
5084 code,
5085 message,
5086 metrics,
5087 )
5088 .await
5089}
5090
5091async fn send_route_bind_error_parts(
5092 tx: &WriterSender,
5093 ver: u8,
5094 corr: u64,
5095 flags: Flags,
5096 code: &str,
5097 message: &str,
5098 metrics: &DispatchPathMetrics,
5099) -> Result<(), SubcError> {
5100 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5101 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5102 log_route_bind_rejection(code, message);
5103 Ok(())
5104}
5105
5106fn log_route_bind_rejection(code: &str, message: &str) {
5114 const WINDOW: Duration = Duration::from_secs(60);
5115 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5116 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5117 let mut map = match map.try_lock() {
5118 Ok(map) => map,
5119 Err(_) => {
5121 log::warn!("subc attach: route bind rejected ({code}): {message}");
5122 return;
5123 }
5124 };
5125 let now = Instant::now();
5126 if map.len() > 512 {
5130 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5131 }
5132 match map.get_mut(message) {
5133 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5134 *suppressed += 1;
5135 }
5136 Some((window_start, suppressed)) => {
5137 if *suppressed > 0 {
5138 log::warn!(
5139 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5140 *suppressed
5141 );
5142 } else {
5143 log::warn!("subc attach: route bind rejected ({code}): {message}");
5144 }
5145 *window_start = now;
5146 *suppressed = 0;
5147 }
5148 None => {
5149 log::warn!("subc attach: route bind rejected ({code}): {message}");
5150 map.insert(message.to_string(), (now, 0));
5151 }
5152 }
5153}
5154
5155async fn handle_tool_call(
5160 tx: &WriterSender,
5161 frame: &Frame,
5162 mut phase_trace: PhaseTrace,
5163 routes: &HashMap<RouteChannel, RouteIdentity>,
5164 pending_binds: &HashMap<RouteChannel, PendingBind>,
5165 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5166 executor: &Arc<Executor>,
5167 active_tool_calls: &ActiveToolCalls,
5168 pending_deferred_setups: &Arc<AtomicUsize>,
5169 shutdown: &Arc<Notify>,
5170 connection_cancel: &PersistentCancelSignal,
5171 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5172 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5173 metrics: &Arc<DispatchPathMetrics>,
5174 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5175 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5176 next_bash_ask_corr: &mut u64,
5177 bg_subs: &mut HashMap<RouteChannel, BgSub>,
5178 bg_sub_by_session: &mut BgSubsBySession,
5179 bg_wake_pending: &mut BgWakePending,
5180 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5181 dispatch: DispatchFn,
5182 deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5183 allow_native_passthrough: bool,
5184 tool_response_body_limit: usize,
5185) -> Result<(), SubcError> {
5186 let route_id = route_key(frame.header.channel, frame.header.epoch);
5187 if pending_binds.contains_key(&route_id) {
5188 let error = build_error_frame(
5189 frame.header.ver,
5190 frame.header.channel,
5191 frame.header.epoch,
5192 frame.header.corr,
5193 frame.header.flags,
5194 "route_not_bound",
5195 "route is not bound before tool call",
5196 )?;
5197 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5198 }
5199
5200 let Some(identity) = routes.get(&route_id).cloned() else {
5201 let error = build_error_frame(
5202 frame.header.ver,
5203 frame.header.channel,
5204 frame.header.epoch,
5205 frame.header.corr,
5206 frame.header.flags,
5207 "route_not_bound",
5208 "route is not bound before tool call",
5209 )?;
5210 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5211 };
5212 let restore_watcher = live_roots
5213 .get(&identity.root)
5214 .is_some_and(|meta| meta.idle_artifacts_evicted);
5215 if let Some(meta) = live_roots.get_mut(&identity.root) {
5216 meta.reactivate_bound();
5217 }
5218 if restore_watcher {
5219 if let Some(ctx) = executor.actor_context(&identity.root) {
5220 crate::commands::configure::ensure_project_watcher(&ctx);
5221 }
5222 }
5223
5224 let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5225 Ok(request) => request,
5226 Err(error) => {
5227 let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5228 let Some(operation) = management_envelope
5229 .as_ref()
5230 .and_then(|value| value.get("op"))
5231 .and_then(Value::as_str)
5232 else {
5233 return Err(SubcError::Json(error));
5234 };
5235 RouteRequest::ToolCall(ToolCallRequest {
5236 name: operation.to_string(),
5237 arguments: management_envelope
5238 .and_then(|value| value.get("params").cloned())
5239 .unwrap_or_else(|| json!({})),
5240 edit_slot_survives: None,
5241 preview: false,
5242 })
5243 }
5244 };
5245 if matches!(
5246 route_request,
5247 RouteRequest::BgEvents(BgEventsRequest {
5248 op: BgEventsOp::BgEvents
5249 })
5250 ) {
5251 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5252 metrics.record_bg_subscription_ended(
5253 &old_sub.root,
5254 &old_sub.session,
5255 route_id,
5256 "resubscribe",
5257 );
5258 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5259 }
5260 if !identity.trust.allows_bash_observation() {
5261 bg_subs.remove(&route_id);
5262 bg_wake_pending.remove(&route_id);
5263 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5264 let denied_sub = BgSub {
5265 corr: frame.header.corr,
5266 ver: frame.header.ver,
5267 flags: frame.header.flags,
5268 root: identity.root.clone(),
5269 session: identity.session.clone(),
5270 };
5271 metrics.record_bg_subscription_ended(
5272 &identity.root,
5273 &identity.session,
5274 route_id,
5275 "subscribe-denied",
5276 );
5277 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5278 return Ok(());
5279 }
5280 bg_subs.insert(
5281 route_id,
5282 BgSub {
5283 corr: frame.header.corr,
5284 ver: frame.header.ver,
5285 flags: frame.header.flags,
5286 root: identity.root.clone(),
5287 session: identity.session.clone(),
5288 },
5289 );
5290 insert_bg_subscription_index(
5291 bg_sub_by_session,
5292 identity.root.clone(),
5293 identity.session.clone(),
5294 route_id,
5295 );
5296 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5297 push::arm_bg_wake(
5298 identity.root.clone(),
5299 identity.session.clone(),
5300 route_id,
5301 bg_wake_pending,
5302 bg_wake_epoch,
5303 metrics,
5304 );
5305 return Ok(());
5306 }
5307
5308 let RouteRequest::ToolCall(call) = route_request else {
5309 unreachable!("background event subscription returned above")
5310 };
5311 let bare_name = call.name;
5312 let arguments = strip_agent_preview_arg_owned(call.arguments);
5313 let format_context = crate::subc_format::FormatContext::from_tool_call(
5314 &bare_name,
5315 &arguments,
5316 identity.project_root.as_path(),
5317 );
5318
5319 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5320 let bind_trust = identity.trust;
5321 let diagnostics_on_edit = live_roots
5322 .get(&identity.root)
5323 .map(|meta| meta.diagnostics_on_edit)
5324 .unwrap_or(false);
5325
5326 let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5327 && arguments
5328 .get("sandbox")
5329 .or_else(|| {
5330 arguments
5331 .get("params")
5332 .and_then(|params| params.get("sandbox"))
5333 })
5334 .and_then(Value::as_str)
5335 == Some("host");
5336 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5337 let response = Response::error(
5338 request_id.clone(),
5339 "sandbox_escalation_denied",
5340 "sandbox host escalation is unavailable to untrusted principals",
5341 );
5342 let text = crate::subc_format::format_response_with_context(
5343 &bare_name,
5344 &response,
5345 &format_context,
5346 );
5347 let result = ToolCallResult { text, response };
5348 let response_frame = build_tool_response_frame_with_limit(
5349 frame.header.ver,
5350 route_id,
5351 frame.header.corr,
5352 frame.header.flags,
5353 &result,
5354 bind_trust,
5355 tool_response_body_limit,
5356 )?;
5357 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5358 }
5359
5360 if matches!(bind_trust, BindTrust::Untrusted)
5361 && is_bash_family_tool(&bare_name)
5362 && (!matches!(bare_name.as_str(), "bash" | "powershell")
5363 || !identity.consumer_elicitation_capable)
5364 {
5365 let response = bash::bash_denied_untrusted_response(request_id.clone());
5366 let text = crate::subc_format::format_response_with_context(
5367 &bare_name,
5368 &response,
5369 &format_context,
5370 );
5371 let result = ToolCallResult { text, response };
5372 let response_frame = build_tool_response_frame_with_limit(
5373 frame.header.ver,
5374 route_id,
5375 frame.header.corr,
5376 frame.header.flags,
5377 &result,
5378 bind_trust,
5379 tool_response_body_limit,
5380 )?;
5381 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5382 }
5383
5384 if !is_subc_agent_core_tool(&bare_name)
5392 && !is_subc_native_plumbing_tool(&bare_name)
5393 && !allow_native_passthrough
5394 {
5395 log::warn!(
5396 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5397 bare_name,
5398 frame.header.channel
5399 );
5400 let response = Response::error(
5401 request_id.clone(),
5402 "unknown_tool",
5403 format!("tool {:?} is not in the AFT tool manifest", bare_name),
5404 );
5405 let text = crate::subc_format::format_response_with_context(
5406 &bare_name,
5407 &response,
5408 &format_context,
5409 );
5410 let result = ToolCallResult { text, response };
5411 let response_frame = build_tool_response_frame_with_limit(
5412 frame.header.ver,
5413 route_id,
5414 frame.header.corr,
5415 frame.header.flags,
5416 &result,
5417 bind_trust,
5418 tool_response_body_limit,
5419 )?;
5420 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5421 }
5422
5423 if matches!(bare_name.as_str(), "bash" | "powershell") {
5424 if matches!(bind_trust, BindTrust::Untrusted) {
5425 let plan = match bash::prepare_bash_elicitation_plan(
5426 &arguments,
5427 identity.project_root.as_path(),
5428 ) {
5429 Ok(plan) => plan,
5430 Err(error) => {
5431 let response = Response::error(request_id.clone(), error.code, error.message);
5432 let text = crate::subc_format::format_response_with_context(
5433 &bare_name,
5434 &response,
5435 &format_context,
5436 );
5437 let result = ToolCallResult { text, response };
5438 let response_frame = build_tool_response_frame_with_limit(
5439 frame.header.ver,
5440 route_id,
5441 frame.header.corr,
5442 frame.header.flags,
5443 &result,
5444 bind_trust,
5445 tool_response_body_limit,
5446 )?;
5447 return send_reliable_writer_frame(
5448 tx,
5449 metrics,
5450 response_frame,
5451 "tool response",
5452 )
5453 .await;
5454 }
5455 };
5456
5457 let reverse_corr =
5458 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5459 let ask_frame = build_bash_elicitation_request_frame(
5460 frame.header.ver,
5461 route_id,
5462 reverse_corr,
5463 frame.header.flags,
5464 &plan.command,
5465 &plan.asks,
5466 )?;
5467
5468 let meta = live_roots
5469 .entry(identity.root.clone())
5470 .or_insert_with(|| RootMeta::new(Instant::now()));
5471 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5472 meta.reactivate_bound();
5473
5474 let route_cancel =
5475 route_bash_cancels
5476 .entry(route_id)
5477 .or_insert_with(|| bash::RouteBashCancel {
5478 token: PersistentCancelSignal::new(),
5479 active_waits: 0,
5480 });
5481 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5482 let cancel = bash::BashWaitCancel {
5483 connection: connection_cancel.clone(),
5484 route: route_cancel.token.clone(),
5485 };
5486 pending_bash_asks.insert(
5487 ReverseCorrKey {
5488 route: route_id,
5489 corr: reverse_corr,
5490 },
5491 PendingBashAsk {
5492 route: route_id,
5493 tool_corr: frame.header.corr,
5494 tool_flags: frame.header.flags,
5495 tool_ver: frame.header.ver,
5496 root: identity.root.clone(),
5497 project_root: identity.project_root.clone(),
5498 session_id: identity.session.clone(),
5499 spawn_principal: identity.spawn_principal.clone(),
5500 edit_slot_survives: call.edit_slot_survives,
5501 request_id,
5502 arguments,
5503 format_context,
5504 cancel,
5505 grants: plan.grants,
5506 expires_at: Instant::now() + bash_elicitation_timeout(),
5507 },
5508 );
5509 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5510 .await;
5511 }
5512
5513 let meta = live_roots
5514 .entry(identity.root.clone())
5515 .or_insert_with(|| RootMeta::new(Instant::now()));
5516 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5517 meta.reactivate_bound();
5518
5519 let route_cancel =
5520 route_bash_cancels
5521 .entry(route_id)
5522 .or_insert_with(|| bash::RouteBashCancel {
5523 token: PersistentCancelSignal::new(),
5524 active_waits: 0,
5525 });
5526 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5527 let cancel = bash::BashWaitCancel {
5528 connection: connection_cancel.clone(),
5529 route: route_cancel.token.clone(),
5530 };
5531
5532 bash::submit_deferred_bash(
5533 executor,
5534 bash_deferred_tx,
5535 bash_poll_touch_tx,
5536 metrics,
5537 dispatch,
5538 identity.root.clone(),
5539 identity.project_root.clone(),
5540 identity.session.clone(),
5541 request_id,
5542 route_id,
5543 frame.header.corr,
5544 frame.header.flags,
5545 frame.header.ver,
5546 arguments,
5547 format_context,
5548 cancel,
5549 bind_trust,
5550 identity.spawn_principal.clone(),
5551 call.edit_slot_survives,
5552 None,
5553 );
5554 return Ok(());
5555 }
5556
5557 let lane = command_lane(&bare_name);
5558 let tool_call_context = ToolCallContext {
5559 project_root: identity.project_root.clone(),
5560 session_id: Some(identity.session.clone()),
5561 request_id: request_id.clone(),
5562 diagnostics_on_edit,
5563 preview: call.preview,
5564 edit_slot_survives: call.edit_slot_survives,
5565 report_registration_downgrade: true,
5566 };
5567
5568 let uses_deferred_response_seam = bare_name == "inspect"
5569 || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5570 if uses_deferred_response_seam {
5571 let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5572 let response = Response::error(
5573 &request_id,
5574 "actor_not_registered",
5575 "executor actor is not registered",
5576 );
5577 let text = crate::subc_format::format_response_with_context(
5578 &bare_name,
5579 &response,
5580 &format_context,
5581 );
5582 let result = ToolCallResult { text, response };
5583 let response_frame = build_tool_response_frame_with_limit(
5584 frame.header.ver,
5585 route_id,
5586 frame.header.corr,
5587 frame.header.flags,
5588 &result,
5589 bind_trust,
5590 tool_response_body_limit,
5591 )?;
5592 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5593 };
5594 let identity_for_run = identity.clone();
5595 let request_id_for_force = request_id.clone();
5596 let format_context_for_run = format_context.clone();
5597 let bare_name_for_run = bare_name.clone();
5598 let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5599 phase_trace.mark_executor_submitted();
5600 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5601 phase_trace.mark_job_admitted();
5602 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5603 let run = || match prepare_tool_call(
5604 &bare_name_for_run,
5605 arguments,
5606 &format_context_for_run,
5607 &tool_call_context,
5608 ctx,
5609 Some(&mut phase_trace),
5610 ) {
5611 Err(result) => {
5612 let response = result.response;
5613 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5614 text: result.text,
5615 phase_trace,
5616 });
5617 response
5618 }
5619 Ok(prepared) => {
5620 let outcome = if bare_name_for_run == "inspect" {
5621 crate::commands::inspect::handle_inspect_deferred_with_restriction(
5622 &prepared.request,
5623 Arc::clone(&deferred_ctx),
5624 matches!(bind_trust, BindTrust::Untrusted),
5625 )
5626 } else {
5627 crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5628 &prepared.request,
5629 Arc::clone(&deferred_ctx),
5630 matches!(bind_trust, BindTrust::Untrusted),
5631 )
5632 };
5633 match outcome {
5634 DispatchOutcome::Deferred(pending) => {
5635 let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5636 pending,
5637 surface_downgraded: prepared.surface_downgraded,
5638 phase_trace,
5639 });
5640 Response::success(
5641 request_id_for_force.clone(),
5642 json!({ "response_deferred": true }),
5643 )
5644 }
5645 DispatchOutcome::Immediate(response) => {
5646 phase_trace.mark_execute_done();
5647 let finalizer = |response: &mut Response| {
5648 crate::response_finalize::finalize_response_with_bg_completions(
5649 response,
5650 ctx,
5651 &identity_for_run.session,
5652 &bare_name_for_run,
5653 bind_trust.allows_bash_observation(),
5654 );
5655 };
5656 let result = finish_tool_call_response(
5657 &bare_name_for_run,
5658 &format_context_for_run,
5659 response,
5660 prepared.surface_downgraded,
5661 Some(&finalizer),
5662 Some(&mut phase_trace),
5663 );
5664 let response = result.response;
5665 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5666 text: result.text,
5667 phase_trace,
5668 });
5669 response
5670 }
5671 }
5672 }
5673 };
5674 if matches!(bind_trust, BindTrust::Untrusted) {
5675 ctx.with_force_restrict(&request_id_for_force, run)
5676 } else {
5677 run()
5678 }
5679 })
5680 });
5681 let deferred_setup_guard =
5682 PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5683 let rx = submit_active_tool_call(
5684 executor.as_ref(),
5685 active_tool_calls,
5686 route_id,
5687 frame.header.corr,
5688 identity.root.clone(),
5689 lane,
5690 request_id.clone(),
5691 RouteDetachPolicy::CancelOnDetach,
5692 job,
5693 );
5694
5695 let completion_tx = tx.clone();
5696 let completion_shutdown = Arc::clone(shutdown);
5697 let completion_metrics = Arc::clone(metrics);
5698 let active_tool_calls = Arc::clone(active_tool_calls);
5699 let deferred_response_tx = deferred_response_tx.clone();
5700 let route = route_id;
5701 let corr = frame.header.corr;
5702 let flags = frame.header.flags;
5703 let ver = frame.header.ver;
5704 let root = identity.root.clone();
5705 let session_id = identity.session.clone();
5706 tokio::spawn(async move {
5707 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5708 let _deferred_setup = deferred_setup_guard;
5709 let response = await_executor_response(rx, request_id.clone()).await;
5710 match setup_rx.await {
5711 Ok(DeferredSetupOutcome::Deferred {
5712 pending,
5713 surface_downgraded,
5714 phase_trace,
5715 }) => {
5716 let pending = PendingSubcResponse {
5717 route,
5718 corr,
5719 flags,
5720 ver,
5721 root,
5722 session_id,
5723 bare_name,
5724 format_context,
5725 bind_trust,
5726 pending,
5727 surface_downgraded,
5728 phase_trace,
5729 };
5730 if let Err(error) = deferred_response_tx.send(pending) {
5731 if let Some(cancellation) = &error.0.pending.cancellation {
5732 cancellation.request_cancel();
5733 }
5734 finish_active_tool_call(&active_tool_calls, route, corr);
5735 }
5736 }
5737 Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5738 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5739 return;
5740 }
5741 let result = ToolCallResult { text, response };
5742 let fatal = note_fatal_panic_response(&result.response);
5743 match build_tool_response_frame_with_limit(
5744 ver,
5745 route,
5746 corr,
5747 flags,
5748 &result,
5749 bind_trust,
5750 tool_response_body_limit,
5751 ) {
5752 Ok(response_frame) => {
5753 let trace = ToolResponseWriteTrace::new(
5754 phase_trace,
5755 bare_name.clone(),
5756 identity.project_root.clone(),
5757 identity.session.clone(),
5758 route.channel,
5759 corr,
5760 );
5761 if let Err(error) = send_traced_tool_response_frame(
5762 &completion_tx,
5763 &completion_metrics,
5764 response_frame,
5765 trace,
5766 )
5767 .await
5768 {
5769 log::warn!(
5770 "subc attach: failed to queue deferred-seam setup response: {error}"
5771 );
5772 }
5773 }
5774 Err(error) => {
5775 log::error!(
5776 "subc attach: failed to build deferred-seam setup response: {error}"
5777 );
5778 }
5779 }
5780 if fatal {
5781 signal_fatal_teardown(
5782 &completion_tx,
5783 Some(route),
5784 ver,
5785 corr,
5786 &completion_shutdown,
5787 &completion_metrics,
5788 )
5789 .await;
5790 }
5791 }
5792 Err(_) => {
5793 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5794 return;
5795 }
5796 let text = crate::subc_format::format_response_with_context(
5797 &bare_name,
5798 &response,
5799 &format_context,
5800 );
5801 let result = ToolCallResult { text, response };
5802 if let Ok(response_frame) = build_tool_response_frame_with_limit(
5803 ver,
5804 route,
5805 corr,
5806 flags,
5807 &result,
5808 bind_trust,
5809 tool_response_body_limit,
5810 ) {
5811 let _ = send_reliable_writer_frame(
5812 &completion_tx,
5813 &completion_metrics,
5814 response_frame,
5815 "deferred setup failure",
5816 )
5817 .await;
5818 }
5819 }
5820 }
5821 });
5822 return Ok(());
5823 }
5824
5825 let bare_name_for_frame = bare_name.clone();
5826 let identity_for_run = identity.clone();
5827 let completion_session = identity.session.clone();
5828 let completion_root = identity.project_root.clone();
5829 let request_id_for_force = request_id.clone();
5830 let format_context_for_frame = format_context.clone();
5831 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5832 phase_trace.mark_executor_submitted();
5833 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5834 phase_trace.mark_job_admitted();
5835 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5836 let run = || {
5837 let finalizer = |response: &mut Response| {
5838 crate::response_finalize::finalize_response_with_bg_completions(
5839 response,
5840 ctx,
5841 &identity_for_run.session,
5842 &bare_name,
5843 bind_trust.allows_bash_observation(),
5844 );
5845 };
5846 match run_tool_call(
5847 &bare_name,
5848 arguments,
5849 &format_context,
5850 &tool_call_context,
5851 ctx,
5852 &dispatch,
5853 Some(&finalizer),
5854 Some(&mut phase_trace),
5855 ) {
5856 ToolCallOutcome::Unary(result) => {
5857 let response = result.response;
5858 let _ = tool_call_tx.send(ToolCallCompletion {
5859 text: result.text,
5860 phase_trace,
5861 });
5862 response
5863 }
5864 }
5865 };
5866 if matches!(bind_trust, BindTrust::Untrusted) {
5867 ctx.with_force_restrict(&request_id_for_force, run)
5868 } else {
5869 run()
5870 }
5871 })
5872 });
5873 let rx = submit_active_tool_call(
5874 executor.as_ref(),
5875 active_tool_calls,
5876 route_id,
5877 frame.header.corr,
5878 identity.root.clone(),
5879 lane,
5880 request_id.clone(),
5881 RouteDetachPolicy::RetainForReplay,
5882 job,
5883 );
5884 let completion_tx = tx.clone();
5885 let completion_shutdown = Arc::clone(shutdown);
5886 let route = route_id;
5887 let corr = frame.header.corr;
5888 let flags = frame.header.flags;
5889 let ver = frame.header.ver;
5890 let completion_metrics = Arc::clone(metrics);
5891 let active_tool_calls = Arc::clone(active_tool_calls);
5892 tokio::spawn(async move {
5893 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5894 let response = await_executor_response(rx, request_id.clone()).await;
5895 let (text, phase_trace) = match tool_call_rx.await {
5896 Ok(completion) => (completion.text, Some(completion.phase_trace)),
5897 Err(_) => (
5898 crate::subc_format::format_response_with_context(
5899 &bare_name_for_frame,
5900 &response,
5901 &format_context_for_frame,
5902 ),
5903 None,
5904 ),
5905 };
5906 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5907 return;
5908 }
5909 let result = ToolCallResult { text, response };
5910 let fatal = note_fatal_panic_response(&result.response);
5911 match build_tool_response_frame_with_limit(
5912 ver,
5913 route,
5914 corr,
5915 flags,
5916 &result,
5917 bind_trust,
5918 tool_response_body_limit,
5919 ) {
5920 Ok(response_frame) => {
5921 let send_result = if let Some(phase_trace) = phase_trace {
5922 let trace = ToolResponseWriteTrace::new(
5923 phase_trace,
5924 bare_name_for_frame,
5925 completion_root,
5926 completion_session,
5927 route.channel,
5928 corr,
5929 );
5930 send_traced_tool_response_frame(
5931 &completion_tx,
5932 &completion_metrics,
5933 response_frame,
5934 trace,
5935 )
5936 .await
5937 } else {
5938 send_reliable_writer_frame(
5939 &completion_tx,
5940 &completion_metrics,
5941 response_frame,
5942 "tool response",
5943 )
5944 .await
5945 };
5946 if let Err(error) = send_result {
5947 log::warn!("subc attach: failed to queue tool response frame: {error}");
5948 }
5949 }
5950 Err(error) => {
5951 log::error!("subc attach: failed to build tool response frame: {error}");
5952 }
5953 }
5954 if fatal {
5955 signal_fatal_teardown(
5956 &completion_tx,
5957 Some(route),
5958 ver,
5959 corr,
5960 &completion_shutdown,
5961 &completion_metrics,
5962 )
5963 .await;
5964 }
5965 });
5966 Ok(())
5967}
5968
5969fn submit_maintenance_job(
5970 executor: &Arc<Executor>,
5971 root_id: ProjectRootId,
5972 kind: MaintenanceDrainKind,
5973 bg_sessions_to_check: Vec<(String, u64)>,
5974 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5975 metrics: &Arc<DispatchPathMetrics>,
5976) {
5977 let request_id = format!(
5978 "subc-maintenance-drain-{}-{}",
5979 kind.label(),
5980 root_id.as_path().to_string_lossy()
5981 );
5982 let response_id = request_id.clone();
5983 let completion_root_id = root_id.clone();
5984 let maintenance_generation = executor
5985 .actor_context(&root_id)
5986 .map(|ctx| ctx.configure_generation())
5987 .unwrap_or(0);
5988 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5989 let lane = Lane::MaintenanceCommit;
5993 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5994 let outcome = match kind {
5995 MaintenanceDrainKind::Watcher => {
5996 let drained = runtime_drain::drain_watcher_events_bounded(
5997 ctx,
5998 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5999 );
6000 MaintenanceJobOutcome {
6001 empty_bg_sessions: Vec::new(),
6002 unacked_bg_keys: None,
6003 requeue_kind: drained.has_more.then_some(kind),
6004 }
6005 }
6006 MaintenanceDrainKind::Lsp => {
6007 let drained = runtime_drain::drain_lsp_events_bounded(
6008 ctx,
6009 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6010 );
6011 MaintenanceJobOutcome {
6012 empty_bg_sessions: Vec::new(),
6013 unacked_bg_keys: None,
6014 requeue_kind: drained.has_more.then_some(kind),
6015 }
6016 }
6017 MaintenanceDrainKind::ConfigureTail => {
6018 runtime_drain::drain_deferred_configure_maintenance(ctx);
6019 runtime_drain::drain_configure_warning_events(ctx);
6020 MaintenanceJobOutcome::default()
6021 }
6022 MaintenanceDrainKind::CompletionDrains => {
6023 runtime_drain::drain_search_index_events(ctx);
6024 runtime_drain::drain_callgraph_store_events(ctx);
6025 runtime_drain::drain_semantic_index_events(ctx);
6026 runtime_drain::drain_semantic_refresh_events(ctx);
6027 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6028 let empty_bg_sessions = bg_sessions_to_check
6029 .into_iter()
6030 .filter(|(session, _)| {
6031 !ctx.bash_background().has_unacked_wakes_for_session(session)
6032 })
6033 .collect();
6034 MaintenanceJobOutcome {
6035 empty_bg_sessions,
6036 unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6037 requeue_kind: None,
6038 }
6039 }
6040 };
6041 let requeued = outcome.requeue_kind.is_some();
6042 let _ = outcome_tx.send(outcome);
6043 Response::success(
6044 response_id,
6045 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6046 )
6047 });
6048 let rx = match kind {
6049 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6050 root_id,
6051 lane,
6052 request_id.clone(),
6053 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6054 job,
6055 ),
6056 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6057 root_id,
6058 lane,
6059 request_id.clone(),
6060 crate::executor::MaintenanceCoalesceKey::LspDrain,
6061 job,
6062 ),
6063 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6064 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6065 }
6066 };
6067 let completion_tx = completion_tx.clone();
6068 let completion_metrics = Arc::clone(metrics);
6069 tokio::spawn(async move {
6070 let _response_task = ResponseTaskGuard::new(&completion_metrics);
6071 let response = await_executor_response(rx, request_id).await;
6072 let outcome = outcome_rx.await.unwrap_or_default();
6073 let _ = send_counted_channel(
6074 &completion_tx,
6075 &completion_metrics.maintenance_queued,
6076 MaintenanceCompletion {
6077 root_id: completion_root_id,
6078 kind,
6079 response,
6080 empty_bg_sessions: outcome.empty_bg_sessions,
6081 unacked_bg_keys: outcome.unacked_bg_keys,
6082 requeue_kind: outcome.requeue_kind,
6083 },
6084 )
6085 .await;
6086 });
6087}
6088
6089async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6090 rx.await
6091 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6092}
6093
6094async fn deliver_resolved_subc_response(
6095 tx: &WriterSender,
6096 mut resolved: ResolvedSubcResponse,
6097 routes: &HashMap<RouteChannel, RouteIdentity>,
6098 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6099 executor: &Executor,
6100 active_tool_calls: &ActiveToolCalls,
6101 shutdown: &Arc<Notify>,
6102 metrics: &DispatchPathMetrics,
6103 tool_response_body_limit: usize,
6104) -> Result<(), SubcError> {
6105 let entry = &mut resolved.entry;
6106 finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6107 if let Some(meta) = live_roots.get_mut(&entry.root) {
6108 meta.note_activity();
6109 }
6110
6111 let Some(identity) = routes.get(&entry.route) else {
6112 log::debug!(
6113 "subc attach: dropping deferred {} response {} for unbound route {}",
6114 entry.bare_name,
6115 entry.pending.request_id,
6116 entry.route
6117 );
6118 return Ok(());
6119 };
6120 let Some(ctx) = executor.actor_context(&entry.root) else {
6121 return Ok(());
6122 };
6123 entry.phase_trace.mark_execute_done();
6124 let finalizer = |response: &mut Response| {
6125 crate::response_finalize::finalize_response_with_bg_completions(
6126 response,
6127 &ctx,
6128 &entry.session_id,
6129 &entry.bare_name,
6130 entry.bind_trust.allows_bash_observation(),
6131 );
6132 };
6133 let result = finish_tool_call_response(
6134 &entry.bare_name,
6135 &entry.format_context,
6136 resolved.response,
6137 entry.surface_downgraded,
6138 Some(&finalizer),
6139 Some(&mut entry.phase_trace),
6140 );
6141 let fatal = note_fatal_panic_response(&result.response);
6142 let response_frame = build_tool_response_frame_with_limit(
6143 entry.ver,
6144 entry.route,
6145 entry.corr,
6146 entry.flags,
6147 &result,
6148 identity.trust,
6149 tool_response_body_limit,
6150 )?;
6151 let trace = ToolResponseWriteTrace::new(
6152 std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6153 entry.bare_name.clone(),
6154 identity.project_root.clone(),
6155 entry.session_id.clone(),
6156 entry.route.channel,
6157 entry.corr,
6158 );
6159 send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6160 if fatal {
6161 signal_fatal_teardown(
6162 tx,
6163 Some(entry.route),
6164 entry.ver,
6165 entry.corr,
6166 shutdown,
6167 metrics,
6168 )
6169 .await;
6170 }
6171 Ok(())
6172}
6173
6174async fn signal_fatal_teardown(
6175 tx: &WriterSender,
6176 route: Option<RouteChannel>,
6177 ver: u8,
6178 corr: u64,
6179 shutdown: &Arc<Notify>,
6180 metrics: &DispatchPathMetrics,
6181) {
6182 if let Some(route) = route {
6183 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6184 if let Err(error) = send_frame(tx, metrics, frame).await {
6185 log::warn!(
6186 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6187 );
6188 }
6189 }
6190 }
6191 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6192 if let Err(error) = send_frame(tx, metrics, frame).await {
6193 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6194 }
6195 }
6196 shutdown.notify_one();
6197}
6198#[derive(Debug, Deserialize)]
6199#[serde(untagged)]
6200enum RouteRequest {
6201 BgEvents(BgEventsRequest),
6202 ToolCall(ToolCallRequest),
6203}
6204
6205#[derive(Debug, Deserialize)]
6206struct BgEventsRequest {
6207 op: BgEventsOp,
6208}
6209
6210#[derive(Debug, Deserialize)]
6211#[serde(rename_all = "snake_case")]
6212enum BgEventsOp {
6213 BgEvents,
6214}
6215
6216#[derive(Debug, Deserialize)]
6217struct ToolCallRequest {
6218 name: String,
6219 #[serde(default)]
6220 arguments: Value,
6221 #[serde(default)]
6223 edit_slot_survives: Option<bool>,
6224 #[serde(default)]
6229 preview: bool,
6230}
6231
6232#[cfg(test)]
6233pub(crate) mod test_support {
6234 use super::*;
6235 use crate::bash_background::BgTaskStatus;
6236 use crate::protocol::{
6237 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6238 ProgressFrame, StatusChangedFrame,
6239 };
6240 use serde_json::json;
6241
6242 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6243 let dir = tempfile::Builder::new()
6244 .prefix(name)
6245 .tempdir()
6246 .expect("temp root");
6247 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6248 (dir, root)
6249 }
6250
6251 pub(super) fn test_ctx() -> Arc<AppContext> {
6252 Arc::new(AppContext::new(
6253 Box::new(crate::parser::TreeSitterProvider::new()),
6254 crate::config::Config::default(),
6255 ))
6256 }
6257
6258 fn inspect_context(root: &Path) -> Arc<AppContext> {
6259 inspect_context_with_timeout(root, None)
6260 }
6261
6262 fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6263 let mut config = crate::config::Config::default();
6264 config.project_root = Some(root.to_path_buf());
6265 if let Some(timeout_ms) = timeout_ms {
6266 config.inspect.diagnostics_timeout_ms = timeout_ms;
6267 }
6268 let ctx = Arc::new(AppContext::new(
6269 Box::new(crate::parser::TreeSitterProvider::new()),
6270 config,
6271 ));
6272 ctx.set_harness(crate::harness::Harness::Opencode);
6273 ctx
6274 }
6275
6276 fn inspect_request(id: &str) -> RawRequest {
6277 serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6278 }
6279
6280 fn submit_deferred_inspect_setup(
6281 executor: &Arc<Executor>,
6282 root: &ProjectRootId,
6283 ctx: &Arc<AppContext>,
6284 request_id: &str,
6285 ) -> (PendingResponse, JobCancellation) {
6286 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6287 let request = inspect_request(request_id);
6288 let inspect_ctx = Arc::clone(ctx);
6289 let (_rx, cancellation) = executor.submit_cancellable_async(
6290 root.clone(),
6291 Lane::SerialLspStatus,
6292 request_id.to_string(),
6293 Box::new(move |_| {
6294 let DispatchOutcome::Deferred(pending) =
6295 crate::commands::inspect::handle_inspect_deferred_with_restriction(
6296 &request,
6297 inspect_ctx,
6298 true,
6299 )
6300 else {
6301 panic!("inspect setup must defer")
6302 };
6303 pending_tx.send(pending).expect("send pending inspect");
6304 Response::success("inspect-setup", json!({}))
6305 }),
6306 );
6307 let pending = pending_rx
6308 .recv_timeout(Duration::from_secs(1))
6309 .expect("inspect setup leaves the executor lane");
6310 let deadline = Instant::now() + Duration::from_secs(1);
6311 while !executor.actor_is_idle(root) {
6312 assert!(
6313 Instant::now() < deadline,
6314 "inspect setup kept lane counters live"
6315 );
6316 std::thread::sleep(Duration::from_millis(5));
6317 }
6318 (pending, cancellation)
6319 }
6320
6321 fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6322 let deadline = Instant::now() + Duration::from_secs(60);
6323 loop {
6324 if let Some(response) = (pending.poll)(ctx) {
6325 return response;
6326 }
6327 assert!(Instant::now() < deadline, "inspect terminal timed out");
6328 std::thread::sleep(Duration::from_millis(5));
6329 }
6330 }
6331
6332 fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6333 let source_dir = root.join("src");
6334 std::fs::create_dir_all(&source_dir).expect("create source dir");
6335 std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6336 .expect("write Cargo manifest");
6337 let source = source_dir.join("main.rs");
6338 std::fs::write(&source, "fn main() {}\n").expect("write source");
6339 let binary = root.join("cold-navigation-server");
6340 std::fs::write(&binary, b"fixture").expect("write server placeholder");
6341
6342 let ctx = inspect_context(root);
6343 ctx.lsp()
6344 .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6345 (ctx, source)
6346 }
6347
6348 fn navigation_request(id: &str, source: &Path) -> RawRequest {
6349 serde_json::from_value(json!({
6350 "id": id,
6351 "command": "lsp_hover",
6352 "file": source.display().to_string(),
6353 "line": 1,
6354 "character": 1,
6355 }))
6356 .expect("navigation request")
6357 }
6358
6359 fn submit_deferred_navigation_setup(
6360 executor: &Arc<Executor>,
6361 root: &ProjectRootId,
6362 ctx: &Arc<AppContext>,
6363 source: &Path,
6364 request_id: &str,
6365 ) -> (PendingResponse, JobCancellation) {
6366 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6367 let request = navigation_request(request_id, source);
6368 let navigation_ctx = Arc::clone(ctx);
6369 let (_rx, cancellation) = executor.submit_cancellable_async(
6370 root.clone(),
6371 Lane::SerialLspStatus,
6372 request_id.to_string(),
6373 Box::new(move |_| {
6374 let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6375 handle_lsp_navigation_deferred_with_restriction(
6376 &request,
6377 navigation_ctx,
6378 true,
6379 )
6380 else {
6381 panic!("cold navigation setup must defer")
6382 };
6383 pending_tx.send(pending).expect("send pending navigation");
6384 Response::success("navigation-setup", json!({}))
6385 }),
6386 );
6387 let pending = pending_rx
6388 .recv_timeout(Duration::from_secs(1))
6389 .expect("navigation setup leaves the executor lane");
6390 let deadline = Instant::now() + Duration::from_secs(1);
6391 while !executor.actor_is_idle(root) {
6392 assert!(
6393 Instant::now() < deadline,
6394 "navigation setup kept lane counters live"
6395 );
6396 std::thread::sleep(Duration::from_millis(5));
6397 }
6398 (pending, cancellation)
6399 }
6400
6401 fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6402 let deadline = Instant::now() + Duration::from_secs(1);
6403 loop {
6404 if let Some(response) = (pending.poll)(ctx) {
6405 return response;
6406 }
6407 assert!(Instant::now() < deadline, "navigation terminal timed out");
6408 std::thread::sleep(Duration::from_millis(5));
6409 }
6410 }
6411
6412 #[test]
6413 fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6414 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6415 let executor = Arc::new(Executor::new());
6416 let (dir, root) = test_root("deferred-inspect-storm");
6417 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6418 let ctx = inspect_context(dir.path());
6419 executor.register_actor(root.clone(), Arc::clone(&ctx));
6420 let (started_rx, release_tx) =
6421 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6422 let (mut pending, _cancellation) =
6423 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6424 started_rx
6425 .recv_timeout(Duration::from_secs(1))
6426 .expect("deferred inspect body starts");
6427
6428 for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6429 let response = executor.submit(
6430 root.clone(),
6431 Lane::Mutating,
6432 request_id.to_string(),
6433 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6434 );
6435 assert!(
6436 response
6437 .recv_timeout(Duration::from_secs(1))
6438 .expect("writer admits while inspect remains deferred")
6439 .success
6440 );
6441 }
6442 assert_eq!(
6443 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6444 1,
6445 "writer admissions must not finish the detached inspect"
6446 );
6447
6448 release_tx.send(()).expect("release inspect body");
6449 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6450 assert!(
6451 terminal.data.get("inspect_terminal").is_some(),
6452 "inspect must still produce its terminal: {:?}",
6453 terminal.data
6454 );
6455 }
6456
6457 #[test]
6458 fn deferred_inspect_honors_the_shared_request_deadline() {
6459 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6460 let executor = Arc::new(Executor::new());
6461 let (dir, root) = test_root("deferred-inspect-deadline");
6462 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6463 let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6464 executor.register_actor(root.clone(), Arc::clone(&ctx));
6465 let (started_rx, _release_tx) =
6466 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6467 let started = Instant::now();
6468 let (mut pending, _cancellation) =
6469 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6470 started_rx
6471 .recv_timeout(Duration::from_secs(1))
6472 .expect("deferred inspect body starts");
6473
6474 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6475 assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6476 assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6477 assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6478 assert!(
6479 started.elapsed() < Duration::from_secs(8),
6480 "deferred inspect missed its terminal reserve: {:?}",
6481 started.elapsed()
6482 );
6483 }
6484
6485 #[test]
6486 fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6487 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6488 let executor = Arc::new(Executor::new());
6489 let (dir, root) = test_root("deferred-cold-navigation");
6490 let (ctx, source) = cold_navigation_context(dir.path());
6491 executor.register_actor(root.clone(), Arc::clone(&ctx));
6492 let (started_rx, _release_tx) =
6493 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6494 let (mut pending, cancellation) = submit_deferred_navigation_setup(
6495 &executor,
6496 &root,
6497 &ctx,
6498 &source,
6499 "subc-cold-navigation",
6500 );
6501 started_rx
6502 .recv_timeout(Duration::from_secs(1))
6503 .expect("detached navigation reaches its cold-start gate");
6504
6505 for (lane, request_id) in [
6506 (Lane::PureRead, "subc-navigation-read"),
6507 (Lane::Mutating, "subc-navigation-write"),
6508 ] {
6509 let response = executor.submit(
6510 root.clone(),
6511 lane,
6512 request_id.to_string(),
6513 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6514 );
6515 assert!(
6516 response
6517 .recv_timeout(Duration::from_secs(1))
6518 .expect("unrelated work admits while navigation remains deferred")
6519 .success
6520 );
6521 }
6522 assert_eq!(
6523 crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6524 1,
6525 "lane admissions must not finish the detached navigation"
6526 );
6527
6528 cancellation.request_cancel();
6529 let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6530 assert!(!terminal.success);
6531 let deadline = Instant::now() + Duration::from_secs(1);
6532 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6533 assert!(
6534 Instant::now() < deadline,
6535 "cancelled navigation worker did not settle"
6536 );
6537 std::thread::sleep(Duration::from_millis(5));
6538 }
6539 assert!(executor.actor_is_idle(&root));
6540 }
6541
6542 #[test]
6543 fn cancelling_pending_navigation_removes_entry_without_reply() {
6544 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6545 let executor = Arc::new(Executor::new());
6546 let (dir, root) = test_root("cancelled-pending-navigation");
6547 let (ctx, source) = cold_navigation_context(dir.path());
6548 executor.register_actor(root.clone(), Arc::clone(&ctx));
6549 let (started_rx, _release_tx) =
6550 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6551 let (pending, _cancellation) = submit_deferred_navigation_setup(
6552 &executor,
6553 &root,
6554 &ctx,
6555 &source,
6556 "subc-cancel-navigation",
6557 );
6558 started_rx
6559 .recv_timeout(Duration::from_secs(1))
6560 .expect("detached navigation reaches its cancellation gate");
6561
6562 let route = RouteChannel {
6563 channel: 17,
6564 epoch: 1,
6565 };
6566 let mut registry = PendingSubcResponses::default();
6567 registry.register(PendingSubcResponse {
6568 route,
6569 corr: 71,
6570 flags: Flags::new(false, Priority::Passive, false),
6571 ver: PROTOCOL_VERSION,
6572 root: root.clone(),
6573 session_id: "navigation-cancel-session".to_string(),
6574 bare_name: "lsp_hover".to_string(),
6575 format_context: crate::subc_format::FormatContext::from_tool_call(
6576 "lsp_hover",
6577 &json!({}),
6578 dir.path(),
6579 ),
6580 bind_trust: BindTrust::FirstParty,
6581 pending,
6582 surface_downgraded: false,
6583 phase_trace: PhaseTrace::new(Instant::now()),
6584 });
6585
6586 assert!(registry.cancel_request(route, 71));
6587 assert!(registry.is_empty());
6588 let deadline = Instant::now() + Duration::from_secs(1);
6589 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6590 assert!(
6591 Instant::now() < deadline,
6592 "pending cancellation did not settle the detached worker"
6593 );
6594 std::thread::sleep(Duration::from_millis(5));
6595 }
6596 assert!(
6597 registry.poll_ready(executor.as_ref()).is_empty(),
6598 "a cancelled navigation must not leak a reply"
6599 );
6600 }
6601
6602 #[test]
6603 fn same_root_deferred_inspects_are_single_flight() {
6604 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6605 let executor = Arc::new(Executor::new());
6606 let (dir, root) = test_root("single-flight-deferred-inspect");
6607 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6608 let ctx = inspect_context(dir.path());
6609 executor.register_actor(root.clone(), Arc::clone(&ctx));
6610 let (started_rx, release_tx) =
6611 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6612 let (mut first, _first_cancellation) =
6613 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6614 started_rx
6615 .recv_timeout(Duration::from_secs(1))
6616 .expect("first inspect owns the root flight");
6617 let (mut second, second_cancellation) =
6618 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6619
6620 assert_eq!(
6621 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6622 1,
6623 "only one detached body may run for a root"
6624 );
6625 assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6626 second_cancellation.request_cancel();
6627 let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6628 assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6629 assert_eq!(
6630 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6631 1,
6632 "cancelling the queued request must not release the active flight"
6633 );
6634
6635 release_tx.send(()).expect("release first inspect");
6636 let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6637 assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6638 assert_eq!(
6639 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6640 0
6641 );
6642 }
6643
6644 #[test]
6645 fn route_abandonment_cancels_detached_inspect_thread() {
6646 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6647 let executor = Arc::new(Executor::new());
6648 let (dir, root) = test_root("cancelled-deferred-inspect");
6649 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6650 let ctx = inspect_context(dir.path());
6651 executor.register_actor(root.clone(), Arc::clone(&ctx));
6652 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6653 let route = RouteChannel {
6654 channel: 7,
6655 epoch: 1,
6656 };
6657 let (started_rx, _release_tx) =
6658 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6659 let (mut pending, cancellation) =
6660 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6661 active.lock().expect("active tool call map").insert(
6662 (route, 41),
6663 ActiveToolCall {
6664 root_id: root.clone(),
6665 cancellation,
6666 detach_policy: RouteDetachPolicy::CancelOnDetach,
6667 },
6668 );
6669 started_rx
6670 .recv_timeout(Duration::from_secs(1))
6671 .expect("detached inspect reaches cancellation gate");
6672 assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6673
6674 assert!(cancel_active_tool_call(
6675 &active,
6676 executor.as_ref(),
6677 route,
6678 41,
6679 "test route abandonment"
6680 ));
6681 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6682 assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6683 assert_eq!(
6684 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6685 0
6686 );
6687 assert!(executor.actor_is_idle(&root));
6688 assert!(active.lock().expect("active tool call map").is_empty());
6689 let restriction_deadline = Instant::now() + Duration::from_secs(1);
6690 while ctx.request_force_restrict("subc-inspect-abandoned") {
6691 assert!(
6692 Instant::now() < restriction_deadline,
6693 "detached force-restrict guard leaked"
6694 );
6695 std::thread::sleep(Duration::from_millis(5));
6696 }
6697 }
6698
6699 #[test]
6700 fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6701 let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6706 pool_size: 3,
6707 read_cap: 2,
6708 actor_cap: 2,
6709 heavy_permits: 1,
6710 drr_quantum: 1,
6711 }));
6712 let (_dir, root) = test_root("cancelled-interactive-search");
6713 executor.register_actor(root.clone(), test_ctx());
6714 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6715 let route = RouteChannel {
6716 channel: 9,
6717 epoch: 1,
6718 };
6719
6720 let disabled_iterations = Arc::new(AtomicUsize::new(0));
6721 let disabled_probe = Arc::clone(&disabled_iterations);
6722 let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6723 let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6724 root.clone(),
6725 Lane::PureRead,
6726 "untracked-search".to_string(),
6727 Box::new(move |_| {
6728 disabled_started_tx
6729 .send(())
6730 .expect("signal untracked search");
6731 let deadline = Instant::now() + Duration::from_secs(5);
6732 while !crate::commands::semantic_search::search_cancellation_requested() {
6733 if Instant::now() >= deadline {
6734 return Response::error(
6735 "untracked-search",
6736 "test_timeout",
6737 "untracked search did not receive cancellation",
6738 );
6739 }
6740 disabled_probe.fetch_add(1, Ordering::Relaxed);
6741 std::thread::yield_now();
6742 }
6743 Response::error(
6744 "untracked-search",
6745 "request_cancelled",
6746 "cancelled at search checkpoint",
6747 )
6748 }),
6749 );
6750 disabled_started_rx
6751 .recv_timeout(Duration::from_secs(1))
6752 .expect("untracked search starts");
6753 assert_eq!(
6754 apply_route_work_disposition(
6755 &active,
6756 executor.as_ref(),
6757 route,
6758 RouteWorkDisposition::Abandon,
6759 "disabled cancellation wiring",
6760 ),
6761 0
6762 );
6763 let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6764 std::thread::sleep(Duration::from_millis(10));
6765 assert!(
6766 disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6767 "without route registration the search keeps computing"
6768 );
6769 disabled_cancellation.request_cancel();
6770 let disabled_response = disabled_rx
6771 .blocking_recv()
6772 .expect("untracked search settles after explicit cleanup");
6773 assert_eq!(disabled_response.data["code"], "request_cancelled");
6774
6775 let tracked_iterations = Arc::new(AtomicUsize::new(0));
6776 let tracked_probe = Arc::clone(&tracked_iterations);
6777 let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6778 let tracked_rx = submit_active_tool_call(
6779 executor.as_ref(),
6780 &active,
6781 route,
6782 42,
6783 root.clone(),
6784 Lane::PureRead,
6785 "tracked-search".to_string(),
6786 RouteDetachPolicy::RetainForReplay,
6787 Box::new(move |_| {
6788 tracked_started_tx.send(()).expect("signal tracked search");
6789 let deadline = Instant::now() + Duration::from_secs(5);
6790 while !crate::commands::semantic_search::search_cancellation_requested() {
6791 if Instant::now() >= deadline {
6792 return Response::error(
6793 "tracked-search",
6794 "test_timeout",
6795 "tracked search did not receive cancellation",
6796 );
6797 }
6798 tracked_probe.fetch_add(1, Ordering::Relaxed);
6799 std::thread::yield_now();
6800 }
6801 Response::error(
6802 "tracked-search",
6803 "request_cancelled",
6804 "cancelled at search checkpoint",
6805 )
6806 }),
6807 );
6808 tracked_started_rx
6809 .recv_timeout(Duration::from_secs(1))
6810 .expect("tracked search starts");
6811
6812 let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6813 let terminal_rx = submit_active_tool_call(
6814 executor.as_ref(),
6815 &active,
6816 route,
6817 43,
6818 root.clone(),
6819 Lane::PureRead,
6820 "teardown-terminal".to_string(),
6821 RouteDetachPolicy::CancelOnDetach,
6822 Box::new(move |_| {
6823 terminal_started_tx
6824 .send(())
6825 .expect("signal teardown-terminal call");
6826 let deadline = Instant::now() + Duration::from_secs(5);
6827 while !crate::executor::current_job_cancelled() {
6828 if Instant::now() >= deadline {
6829 return Response::error(
6830 "teardown-terminal",
6831 "test_timeout",
6832 "terminal call did not receive cancellation",
6833 );
6834 }
6835 std::thread::yield_now();
6836 }
6837 Response::error(
6838 "teardown-terminal",
6839 "request_cancelled",
6840 "cancelled for terminal-emitting teardown",
6841 )
6842 }),
6843 );
6844 terminal_started_rx
6845 .recv_timeout(Duration::from_secs(1))
6846 .expect("teardown-terminal call starts");
6847 assert_eq!(
6848 apply_route_work_disposition(
6849 &active,
6850 executor.as_ref(),
6851 route,
6852 RouteWorkDisposition::RetainForReplay,
6853 "test route detach",
6854 ),
6855 1,
6856 "only the replayable search remains active after route detach"
6857 );
6858 let terminal_response = terminal_rx
6859 .blocking_recv()
6860 .expect("teardown-terminal call stops at cancellation checkpoint");
6861 assert_eq!(terminal_response.data["code"], "request_cancelled");
6862 let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6863 std::thread::sleep(Duration::from_millis(10));
6864 assert!(
6865 tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6866 "route detach must retain work whose response can replay after rebind"
6867 );
6868 assert_eq!(
6869 apply_route_work_disposition(
6870 &active,
6871 executor.as_ref(),
6872 route,
6873 RouteWorkDisposition::Abandon,
6874 "test session purge",
6875 ),
6876 1
6877 );
6878 let tracked_response = tracked_rx
6879 .blocking_recv()
6880 .expect("tracked search stops at cancellation checkpoint");
6881 assert_eq!(tracked_response.data["code"], "request_cancelled");
6882 assert!(active.lock().expect("active tool calls").is_empty());
6883
6884 let deadline = Instant::now() + Duration::from_secs(1);
6885 while !executor.actor_is_idle(&root) {
6886 assert!(
6887 Instant::now() < deadline,
6888 "cancelled search must release the PureRead lane"
6889 );
6890 std::thread::sleep(Duration::from_millis(2));
6891 }
6892 }
6893
6894 #[test]
6895 fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6896 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6897 let executor = Arc::new(Executor::new());
6898 let (dir, root) = test_root("shutdown-deferred-inspect");
6899 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6900 let ctx = inspect_context(dir.path());
6901 executor.register_actor(root.clone(), Arc::clone(&ctx));
6902 let (started_rx, _release_tx) =
6903 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6904 let (pending, cancellation) =
6905 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6906 let route = RouteChannel {
6907 channel: 8,
6908 epoch: 1,
6909 };
6910 started_rx
6911 .recv_timeout(Duration::from_secs(1))
6912 .expect("detached inspect reaches shutdown gate");
6913 let mut registry = PendingSubcResponses::default();
6914 registry.register(PendingSubcResponse {
6915 route,
6916 corr: 42,
6917 flags: Flags::new(false, Priority::Passive, false),
6918 ver: PROTOCOL_VERSION,
6919 root: root.clone(),
6920 session_id: "shutdown-session".to_string(),
6921 bare_name: "inspect".to_string(),
6922 format_context: crate::subc_format::FormatContext::from_tool_call(
6923 "inspect",
6924 &json!({}),
6925 dir.path(),
6926 ),
6927 bind_trust: BindTrust::FirstParty,
6928 pending,
6929 surface_downgraded: false,
6930 phase_trace: PhaseTrace::new(Instant::now()),
6931 });
6932 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6933 (route, 42),
6934 ActiveToolCall {
6935 root_id: root.clone(),
6936 cancellation,
6937 detach_policy: RouteDetachPolicy::CancelOnDetach,
6938 },
6939 )])));
6940
6941 let resolved = registry.drain_on_shutdown(executor.as_ref());
6942 assert!(registry.is_empty());
6943 assert_eq!(resolved.len(), 1);
6944 assert_eq!(
6945 resolved[0].response.data["failure_reason"],
6946 "daemon_shutdown"
6947 );
6948 finish_active_tool_call(&active, route, 42);
6949 let deadline = Instant::now() + Duration::from_secs(1);
6950 while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6951 assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6952 std::thread::sleep(Duration::from_millis(5));
6953 }
6954 assert!(active.lock().expect("active calls").is_empty());
6955 assert!(executor.actor_is_idle(&root));
6956 }
6957
6958 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6959 let deadline = Instant::now() + Duration::from_secs(30);
6960 loop {
6961 let observed = ctx.watcher_registry_count();
6962 if observed == expected {
6963 return;
6964 }
6965 assert!(
6966 Instant::now() < deadline,
6967 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6968 );
6969 std::thread::sleep(Duration::from_millis(50));
6970 }
6971 }
6972
6973 pub(super) fn reap_until_forgotten(
6983 root: &ProjectRootId,
6984 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6985 pending_binds: &HashMap<RouteChannel, PendingBind>,
6986 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6987 executor: &Arc<Executor>,
6988 metrics: &DispatchPathMetrics,
6989 ) -> IdleReapOutcome {
6990 let deadline = Instant::now() + Duration::from_secs(30);
6991 loop {
6992 let outcome = reap_idle_roots(
6993 Instant::now(),
6994 live_roots,
6995 pending_binds,
6996 root_channels,
6997 executor,
6998 metrics,
6999 );
7000 if outcome.forgotten_deleted_roots.contains(root) {
7001 return outcome;
7002 }
7003 assert!(
7004 Instant::now() < deadline,
7005 "deleted root was never forgotten: {root:?}"
7006 );
7007 std::thread::sleep(Duration::from_millis(10));
7008 }
7009 }
7010
7011 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7012 let deadline = Instant::now() + Duration::from_secs(30);
7013 loop {
7014 let observed = app.actor_root_count();
7015 if observed == expected {
7016 return;
7017 }
7018 assert!(
7019 Instant::now() < deadline,
7020 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7021 );
7022 std::thread::sleep(Duration::from_millis(50));
7023 }
7024 }
7025
7026 pub(super) fn status_frame(seq: u64) -> PushFrame {
7027 status_frame_with_session(seq, None)
7028 }
7029
7030 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7031 PushFrame::StatusChanged(StatusChangedFrame {
7032 frame_type: "status_changed",
7033 session_id: session_id.map(str::to_string),
7034 snapshot: json!({ "seq": seq }),
7035 })
7036 }
7037
7038 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7039 completion_frame_with_session(task_id, "session-1")
7040 }
7041
7042 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7043 PushFrame::BashCompleted(BashCompletedFrame {
7044 frame_type: "bash_completed",
7045 task_id: task_id.to_string(),
7046 session_id: session_id.to_string(),
7047 status: BgTaskStatus::Completed,
7048 exit_code: Some(0),
7049 command: format!("echo {task_id}"),
7050 output_preview: String::new(),
7051 bash_output_list_envelope: None,
7052 output_truncated: false,
7053 original_tokens: None,
7054 compressed_tokens: None,
7055 tokens_skipped: false,
7056 status_reason: None,
7057 live_descendants: Some(Vec::new()),
7058 live_descendants_omitted: 0,
7059 live_descendants_summary: None,
7060 })
7061 }
7062
7063 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7064 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7065 }
7066
7067 pub(super) fn long_running_frame_with_session(
7068 task_id: &str,
7069 session_id: &str,
7070 elapsed_ms: u64,
7071 ) -> PushFrame {
7072 PushFrame::BashLongRunning(BashLongRunningFrame {
7073 frame_type: "bash_long_running",
7074 task_id: task_id.to_string(),
7075 session_id: session_id.to_string(),
7076 command: format!("sleep {elapsed_ms}"),
7077 elapsed_ms,
7078 })
7079 }
7080
7081 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7082 PushFrame::BashPatternMatch(BashPatternMatchFrame {
7083 frame_type: "bash_pattern_match",
7084 task_id: "task-pattern".to_string(),
7085 session_id: session_id.to_string(),
7086 watch_id: "watch-1".to_string(),
7087 match_text: "needle".to_string(),
7088 match_offset: 7,
7089 context: "haystack needle".to_string(),
7090 once: true,
7091 reason: "pattern_match",
7092 })
7093 }
7094
7095 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7096 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7097 frame_type: "configure_warnings",
7098 session_id: session_id.map(str::to_string),
7099 project_root: "/tmp/subc-test".to_string(),
7100 warnings: Vec::new(),
7101 })
7102 }
7103
7104 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7105 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7106 }
7107
7108 pub(super) fn route_identity_with_trust(
7109 root: &ProjectRootId,
7110 session_id: &str,
7111 trust: BindTrust,
7112 ) -> RouteIdentity {
7113 RouteIdentity(Arc::new(RouteIdentityData {
7114 root: root.clone(),
7115 project_root: root.as_path().to_path_buf(),
7116 harness: "opencode".to_string(),
7117 session: session_id.to_string(),
7118 trust,
7119 spawn_principal: AuthenticatedPrincipal::RouteBind {
7120 trust: trust.sandbox_trust(),
7121 route_channel: 0,
7122 route_epoch: 0,
7123 project_root: root.as_path().to_path_buf(),
7124 harness: "opencode".to_string(),
7125 session_id: session_id.to_string(),
7126 principal_id: Some(match trust {
7127 BindTrust::FirstParty => "direct".to_string(),
7128 BindTrust::Untrusted => "unverified".to_string(),
7129 }),
7130 },
7131 consumer_elicitation_capable: false,
7132 }))
7133 }
7134
7135 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7136 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7137 }
7138
7139 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7140 match frame {
7141 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7142 _ => None,
7143 }
7144 }
7145
7146 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7147 match frame {
7148 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7149 _ => None,
7150 }
7151 }
7152
7153 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7154 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7155 body.get("task_id")
7156 .and_then(serde_json::Value::as_str)
7157 .map(str::to_string)
7158 }
7159}
7160
7161#[cfg(test)]
7162mod tests {
7163 use super::test_support::{
7164 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7165 wait_for_actor_root_count, wait_for_watcher_count,
7166 };
7167 use super::*;
7168 use crate::bash_background::BgTaskStatus;
7169
7170 #[test]
7174 fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7175 assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7176 assert!(matches!(
7177 module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7178 Err(SubcError::ConnectionLost)
7179 ));
7180 assert!(matches!(
7181 module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7182 Err(SubcError::ActorFatal)
7183 ));
7184 }
7185
7186 #[test]
7187 fn fatal_panic_responses_are_detected_and_noted() {
7188 let panic = Response::error(
7189 "req-fatal",
7190 "actor_fatal",
7191 "start byte index 7 is not a char boundary",
7192 );
7193 assert!(note_fatal_panic_response(&panic));
7194 let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7195 assert!(!note_fatal_panic_response(&ordinary));
7196 }
7197
7198 fn attach_error(kind: io::ErrorKind) -> SubcError {
7199 SubcError::Connect {
7200 endpoint: "127.0.0.1:1".to_string(),
7201 source: io::Error::new(kind, "constructed attach failure"),
7202 }
7203 }
7204
7205 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7206 SubcError::Auth {
7207 endpoint: "127.0.0.1:1".to_string(),
7208 source: subc_transport::AuthError::Io {
7209 stage: subc_transport::AuthStage::ServerProof,
7210 source: io::Error::new(kind, "constructed auth failure"),
7211 },
7212 }
7213 }
7214
7215 fn cpu_hunt_process_cpu_us() -> u64 {
7216 #[cfg(unix)]
7217 {
7218 let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7219 if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7221 let usage = unsafe { usage.assume_init() };
7222 return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7223 + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7224 }
7225 }
7226 0
7227 }
7228
7229 #[test]
7230 fn route_bind_does_not_recompute_fleet_health() {
7231 let runtime = tokio::runtime::Builder::new_current_thread()
7232 .enable_all()
7233 .build()
7234 .expect("bind runtime");
7235 runtime.block_on(async {
7236 let (dir, root) = test_root("bind-health-work-count");
7237 let app = App::default_shared();
7238 let executor = Arc::new(Executor::new());
7239 let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7240 let mut fixture_dirs = Vec::new();
7241 if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7244 let copy = std::fs::canonicalize(copy).expect("copied store exists");
7245 let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7246 .parent()
7247 .expect("crates directory")
7248 .parent()
7249 .expect("checkout directory")
7250 .canonicalize()
7251 .expect("canonical checkout");
7252 assert!(copy.starts_with(project.join("target")));
7253 for index in 0..36 {
7254 let actor = if index == 0 {
7255 Arc::clone(&ctx)
7256 } else {
7257 Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7258 };
7259 actor.update_config(|config| config.project_root = Some(project.clone()));
7260 *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7261 crate::callgraph_store::CallGraphStore::open_readonly(
7262 copy.clone(),
7263 project.clone(),
7264 )
7265 .expect("open copied graph")
7266 .expect("copied graph ready"),
7267 ));
7268 if index > 0 {
7269 let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7270 assert!(executor.register_actor(id, actor));
7271 fixture_dirs.push(fixture);
7272 }
7273 }
7274 }
7275 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7276 let cache = HealthRollupCache::new();
7277 if !fixture_dirs.is_empty() {
7278 let started = Instant::now();
7279 let cpu = cpu_hunt_process_cpu_us();
7280 cache.refresh(&executor, &app);
7281 eprintln!(
7282 "cpu_hunt health_rollup wall_us={} cpu_us={}",
7283 started.elapsed().as_micros(),
7284 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7285 );
7286 let started = Instant::now();
7287 let cpu = cpu_hunt_process_cpu_us();
7288 std::hint::black_box(ctx.build_status_snapshot());
7289 eprintln!(
7290 "cpu_hunt status wall_us={} cpu_us={}",
7291 started.elapsed().as_micros(),
7292 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7293 );
7294 }
7295 let refreshes_before = cache.refresh_count_for_test();
7296 let metrics = Arc::new(DispatchPathMetrics::new());
7297 let (writer_tx, _writer_rx) = mpsc::channel(8);
7298 let (completion_tx, mut completion_rx) = mpsc::channel(8);
7299 let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7300 let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7301 let senders = PushSenders {
7302 lossy_tx,
7303 reliable_tx,
7304 lossy_overflow: Arc::new(push::LossyOverflow::default()),
7305 lossy_seq: Arc::new(AtomicU64::new(0)),
7306 fleet_status_client: FleetStatusClient::channel(1).0,
7307 };
7308 let request = ModuleControlRequest::RouteBind {
7309 route_channel: 1,
7310 epoch: 1,
7311 target: RouteTarget::ToolProvider {
7312 module_id: "aft".to_string(),
7313 },
7314 identity: subc_protocol::BindIdentity {
7315 project_root: root.as_path().to_path_buf(),
7316 harness: "opencode".to_string(),
7317 session: "bind-health-work-count".to_string(),
7318 },
7319 principal: Some(subc_protocol::Principal::Direct),
7320 consumer_capabilities: None,
7321 admission_facts: Default::default(),
7322 };
7323 let frame = Frame::build_with_version(
7324 PROTOCOL_VERSION,
7325 FrameType::Request,
7326 control_flags(),
7327 0,
7328 0,
7329 1,
7330 serde_json::to_vec(&request).expect("bind body"),
7331 )
7332 .expect("bind frame");
7333 let mut pending_binds = HashMap::new();
7334 let started = Instant::now();
7335 let cpu = cpu_hunt_process_cpu_us();
7336 handle_control_request(
7337 &writer_tx,
7338 &frame,
7339 &app,
7340 &executor,
7341 &mut HashMap::new(),
7342 &mut pending_binds,
7343 &mut HashMap::new(),
7344 &mut HashMap::new(),
7345 &mut HashSet::new(),
7346 &mut HashMap::new(),
7347 &mut HashMap::new(),
7348 &mut HashMap::new(),
7349 &mut HashMap::new(),
7350 &mut HashMap::new(),
7351 &mut HashMap::new(),
7352 &Arc::new(StdMutex::new(HashMap::new())),
7353 &mut PendingSubcResponses::default(),
7354 &mut RetryBuffer::new(),
7355 &mut HashMap::new(),
7356 &Arc::new(Notify::new()),
7357 &completion_tx,
7358 &metrics,
7359 None,
7360 &cache,
7361 &senders,
7362 |request, _| Response::success(request.id, json!({})),
7363 Some(&dir.path().join("absent-user-config.json")),
7364 usize::MAX,
7365 )
7366 .await
7367 .expect("admit route bind");
7368 eprintln!(
7369 "route_bind_health admission_us={} cpu_us={} refreshes={}",
7370 started.elapsed().as_micros(),
7371 cpu_hunt_process_cpu_us().saturating_sub(cpu),
7372 cache.refresh_count_for_test() - refreshes_before
7373 );
7374 let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7375 .await
7376 .expect("configure completes")
7377 .expect("completion delivered");
7378 assert!(completion.configure_response.success);
7379 assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7380 assert_eq!(
7381 cache.refresh_count_for_test() - refreshes_before,
7382 0,
7383 "route admission must not perform a fleet-wide health census"
7384 );
7385 });
7386 }
7387
7388 #[test]
7389 fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7390 let (dir, root) = test_root("health-does-not-lock-bash-db");
7391 let executor = Arc::new(Executor::new());
7392 let ctx = test_ctx();
7393 ctx.set_harness(crate::harness::Harness::Opencode);
7394 let db = Arc::new(StdMutex::new(
7395 crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7396 ));
7397 ctx.bash_background().set_db_pool(Arc::clone(&db));
7398 executor.register_actor(root, ctx);
7399
7400 let guard = db.lock().expect("hold bash-background DB mutex");
7401 let app = App::default_shared();
7402 let metrics = Arc::new(DispatchPathMetrics::new());
7403 let health_rollup_cache = Arc::new(HealthRollupCache::new());
7404 let frame = Frame::build_with_version(
7405 PROTOCOL_VERSION,
7406 FrameType::Request,
7407 control_flags(),
7408 0,
7409 0,
7410 77,
7411 Vec::new(),
7412 )
7413 .expect("health request frame");
7414 let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7415 let (done_tx, done_rx) = std::sync::mpsc::channel();
7416 let join = std::thread::spawn(move || {
7417 let runtime = tokio::runtime::Builder::new_current_thread()
7418 .enable_all()
7419 .build()
7420 .expect("health test runtime");
7421 let result = runtime.block_on(send_cached_health_response(
7422 &writer_tx,
7423 &frame,
7424 &app,
7425 &executor,
7426 &HashMap::new(),
7427 &metrics,
7428 &health_rollup_cache,
7429 ));
7430 done_tx.send(result).expect("report health result");
7431 });
7432
7433 let result = done_rx
7434 .recv_timeout(Duration::from_millis(500))
7435 .expect("channel-0 health blocked on bash-background DB mutex");
7436 result.expect("send cached health response");
7437 drop(guard);
7438 join.join().expect("health thread");
7439 }
7440
7441 #[test]
7442 fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7443 let (_dir_a, root_a) = test_root("health-metric-root-a");
7444 let (_dir_b, root_b) = test_root("health-metric-root-b");
7445 let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7446 let snapshots = HashMap::from([
7447 (root_a, HashSet::from([duplicate_key.to_string()])),
7448 (root_b, HashSet::from([duplicate_key.to_string()])),
7449 ]);
7450 let metrics = DispatchPathMetrics::new();
7451
7452 record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7453
7454 assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7455 }
7456
7457 #[test]
7458 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7459 let transient_errors = vec![
7460 attach_error(io::ErrorKind::ConnectionRefused),
7461 attach_error(io::ErrorKind::TimedOut),
7462 attach_error(io::ErrorKind::ConnectionReset),
7463 auth_io_error(io::ErrorKind::ConnectionAborted),
7464 auth_io_error(io::ErrorKind::BrokenPipe),
7465 SubcError::Auth {
7466 endpoint: "127.0.0.1:1".to_string(),
7467 source: subc_transport::AuthError::UnexpectedEof {
7468 stage: subc_transport::AuthStage::ServerProof,
7469 expected: 4,
7470 actual: 0,
7471 },
7472 },
7473 SubcError::Auth {
7474 endpoint: "127.0.0.1:1".to_string(),
7475 source: subc_transport::AuthError::Timeout {
7476 stage: subc_transport::AuthStage::ServerProof,
7477 deadline: AUTH_DEADLINE,
7478 },
7479 },
7480 ];
7481 for error in &transient_errors {
7482 assert_eq!(
7483 classify_attach_error(error),
7484 AttachErrorClass::Transient,
7485 "expected transient: {error}"
7486 );
7487 }
7488
7489 let permanent_errors = vec![
7490 attach_error(io::ErrorKind::PermissionDenied),
7491 auth_io_error(io::ErrorKind::InvalidData),
7492 SubcError::Auth {
7493 endpoint: "127.0.0.1:1".to_string(),
7494 source: subc_transport::AuthError::InvalidServerProof,
7495 },
7496 SubcError::Auth {
7497 endpoint: "127.0.0.1:1".to_string(),
7498 source: subc_transport::AuthError::DaemonIdMismatch,
7499 },
7500 SubcError::ConnectionFile {
7501 path: PathBuf::from("subc-connection.json"),
7502 source: subc_transport::ConnectionFileError::Invalid {
7503 reason: "constructed invalid file".to_string(),
7504 },
7505 },
7506 SubcError::NoEndpoint {
7507 path: PathBuf::from("subc-connection.json"),
7508 },
7509 SubcError::InvalidEndpoint {
7510 path: PathBuf::from("subc-connection.json"),
7511 endpoint: "not-an-ip:1234".to_string(),
7512 },
7513 ];
7514 for error in &permanent_errors {
7515 assert_eq!(
7516 classify_attach_error(error),
7517 AttachErrorClass::Permanent,
7518 "expected permanent: {error}"
7519 );
7520 }
7521 }
7522
7523 #[test]
7524 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7525 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7526 let conn_path = conn_dir.path().join("subc-connection.json");
7527 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7528 listener
7529 .set_nonblocking(true)
7530 .expect("set listener nonblocking");
7531 let port = listener.local_addr().expect("listener addr").port();
7532 connection_file::write_atomic(
7533 &conn_path,
7534 &connection_file::ConnectionInfo {
7535 schema: connection_file::SCHEMA_VERSION,
7536 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7537 endpoints: vec![connection_file::Endpoint {
7538 host: "127.0.0.1".to_string(),
7539 port,
7540 }],
7541 key: vec![0x42; subc_transport::KEY_LEN],
7542 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7543 pid: std::process::id(),
7544 daemon_ver: "subc-test".to_string(),
7545 },
7546 )
7547 .expect("write connection file");
7548
7549 let runtime = tokio::runtime::Builder::new_current_thread()
7550 .enable_all()
7551 .build()
7552 .expect("test runtime");
7553 let result = runtime.block_on(connect_and_authenticate_with_policy(
7554 &conn_path,
7555 AttachRetryPolicy {
7556 budget: Duration::from_secs(1),
7557 initial_backoff: Duration::from_millis(5),
7558 max_backoff: Duration::from_millis(10),
7559 jitter_percent: 0,
7560 },
7561 None,
7562 ));
7563 assert!(matches!(
7564 result,
7565 Err(SubcError::ConnectionFile {
7566 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7567 ..
7568 })
7569 ));
7570 assert!(matches!(
7571 listener.accept(),
7572 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7573 ));
7574 }
7575
7576 #[test]
7577 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7578 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7579 let conn_path = conn_dir.path().join("subc-connection.json");
7580 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7581 let port = listener.local_addr().expect("reserved addr").port();
7582 drop(listener);
7583 connection_file::write_atomic(
7584 &conn_path,
7585 &connection_file::ConnectionInfo {
7586 schema: connection_file::SCHEMA_VERSION,
7587 wire_version: Some(PROTOCOL_VERSION),
7588 endpoints: vec![connection_file::Endpoint {
7589 host: "127.0.0.1".to_string(),
7590 port,
7591 }],
7592 key: vec![0x42; subc_transport::KEY_LEN],
7593 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7594 pid: std::process::id(),
7595 daemon_ver: "subc-test".to_string(),
7596 },
7597 )
7598 .expect("write connection file");
7599
7600 let policy = AttachRetryPolicy {
7601 budget: Duration::from_millis(40),
7602 initial_backoff: Duration::from_millis(5),
7603 max_backoff: Duration::from_millis(10),
7604 jitter_percent: 0,
7605 };
7606 let runtime = tokio::runtime::Builder::new_current_thread()
7607 .enable_all()
7608 .build()
7609 .expect("test runtime");
7610 let started_at = Instant::now();
7611 let result = runtime.block_on(connect_and_authenticate_with_policy(
7612 &conn_path, policy, None,
7613 ));
7614 let elapsed = started_at.elapsed();
7615 let error = match result {
7616 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7617 Err(error) => error,
7618 };
7619
7620 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7621 assert!(
7622 elapsed >= Duration::from_millis(35),
7623 "retry budget ended too early: {elapsed:?}"
7624 );
7625 assert!(
7626 elapsed < Duration::from_secs(1),
7627 "retry budget was not bounded: {elapsed:?}"
7628 );
7629 }
7630
7631 fn due_maintenance_jobs_without_actor_context(
7632 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7633 budget: usize,
7634 pending_bind_roots: &HashSet<ProjectRootId>,
7635 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7636 due_maintenance_jobs(
7637 live_roots,
7638 None,
7639 &HashMap::new(),
7640 &BgWakePending::new(),
7641 budget,
7642 pending_bind_roots,
7643 )
7644 }
7645
7646 fn actor_ctx_with_dirty_search_index(
7647 root: &Path,
7648 storage: &Path,
7649 file_name: &str,
7650 old_contents: &str,
7651 new_contents: &str,
7652 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7653 let file = root.join(file_name);
7654 std::fs::write(&file, old_contents).expect("write source");
7655 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7656 let ctx = Arc::new(AppContext::new(
7657 Box::new(crate::parser::TreeSitterProvider::new()),
7658 Config {
7659 project_root: Some(root.to_path_buf()),
7660 storage_dir: Some(storage.to_path_buf()),
7661 ..Config::default()
7662 },
7663 ));
7664 ctx.set_canonical_cache_root(canonical_root.clone());
7665
7666 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7667 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7668 let git_head = index.stored_git_head().map(str::to_owned);
7669 index.write_to_disk(&cache_dir, git_head.as_deref());
7670
7671 std::fs::write(&file, new_contents).expect("edit source");
7672 index.update_file(&file);
7673 *ctx.search_index()
7674 .write()
7675 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7676 (ctx, canonical_root, cache_dir)
7677 }
7678
7679 #[test]
7680 fn graceful_shutdown_flushes_every_actor_search_index() {
7681 let storage = tempfile::tempdir().expect("storage tempdir");
7682 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7683 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7684 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7685 root1_dir.path(),
7686 storage.path(),
7687 "alpha.txt",
7688 "old actor one token\n",
7689 "new actor one token\n",
7690 );
7691 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7692 root2_dir.path(),
7693 storage.path(),
7694 "beta.txt",
7695 "old actor two token\n",
7696 "new actor two token\n",
7697 );
7698
7699 let executor = Executor::new();
7700 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7701 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7702
7703 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7704
7705 let mut restored1 =
7706 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7707 .expect("load flushed root one index");
7708 restored1.ready = true;
7709 assert_eq!(
7710 restored1
7711 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7712 .matches
7713 .len(),
7714 1,
7715 "graceful subc shutdown should flush the first root's trigram delta"
7716 );
7717
7718 let mut restored2 =
7719 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7720 .expect("load flushed root two index");
7721 restored2.ready = true;
7722 assert_eq!(
7723 restored2
7724 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7725 .matches
7726 .len(),
7727 1,
7728 "graceful subc shutdown should flush every registered root"
7729 );
7730 }
7731
7732 #[test]
7733 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7734 let _ = env_logger::builder().is_test(true).try_init();
7735 let (root_dir, root) = test_root("idle-root-reaper");
7736 let storage = tempfile::tempdir().expect("storage tempdir");
7737 std::fs::write(
7738 root_dir.path().join("main.rs"),
7739 "fn entry() { leaf(); }\nfn leaf() {}\n",
7740 )
7741 .expect("source file");
7742 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7743 let app = App::default_shared();
7744 let ctx = Arc::new(AppContext::from_app(
7745 Arc::clone(&app),
7746 Config {
7747 project_root: Some(canonical_root.clone()),
7748 storage_dir: Some(storage.path().to_path_buf()),
7749 callgraph_store: true,
7750 search_index: true,
7751 ..Config::default()
7752 },
7753 ));
7754 ctx.set_canonical_cache_root(canonical_root.clone());
7755 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7756 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7757 assert!(ctx
7758 .ensure_callgraph_store()
7759 .expect("build callgraph store")
7760 .is_some());
7761
7762 let cache_dir =
7763 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7764 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7765 let git_head = index.stored_git_head().map(str::to_owned);
7766 index.write_to_disk(&cache_dir, git_head.as_deref());
7767 *ctx.search_index()
7768 .write()
7769 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7770 let seeded_generation =
7773 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7774 .expect("seeded artifact generation");
7775 crate::cache_freshness::record_verify_completed(
7776 &canonical_root,
7777 crate::cache_freshness::VerifyArtifact::Search,
7778 Some(seeded_generation),
7779 );
7780 assert!(
7781 matches!(
7782 crate::cache_freshness::warm_verify_plan(
7783 canonical_root.as_path(),
7784 crate::cache_freshness::VerifyArtifact::Search,
7785 Some(seeded_generation),
7786 ),
7787 crate::cache_freshness::WarmVerifyPlan::Skip
7788 ),
7789 "memo must be warm before eviction for the downgrade assertion to bite"
7790 );
7791
7792 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7793 let _dispatch_tx = dispatch_tx;
7794 let shutdown = Arc::new(AtomicBool::new(false));
7795 let thread_shutdown = Arc::clone(&shutdown);
7796 let join = std::thread::spawn(move || {
7797 while !thread_shutdown.load(Ordering::SeqCst) {
7798 std::thread::yield_now();
7799 }
7800 });
7801 ctx.install_watcher_runtime(
7802 dispatch_rx,
7803 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7804 );
7805 wait_for_watcher_count(&ctx, 1);
7806
7807 let executor = Arc::new(Executor::new());
7808 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7809 ctx.mark_subc_unbound();
7810 let mut live_roots = HashMap::new();
7811 let mut meta = RootMeta::new(Instant::now());
7812 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7813 meta.unbound_quiesced = true;
7814 live_roots.insert(root.clone(), meta);
7815
7816 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7817 assert!(message.contains("evicted idle root"));
7818 assert!(message.contains("freed ~"));
7819 assert!(message.contains("semantic"));
7820 assert!(!message.contains("semantic not estimated retained"));
7821 assert!(message.contains("trigram"));
7822 assert!(message.contains("retained: bash"));
7823 assert!(message.contains("parser_pool"));
7824
7825 assert_eq!(
7826 reap_idle_roots(
7827 Instant::now(),
7828 &mut live_roots,
7829 &HashMap::new(),
7830 &HashMap::new(),
7831 &executor,
7832 &DispatchPathMetrics::new(),
7833 )
7834 .evicted,
7835 1
7836 );
7837 assert!(ctx.search_index().read().unwrap().is_none());
7838 wait_for_watcher_count(&ctx, 0);
7839 assert!(
7844 matches!(
7845 crate::cache_freshness::warm_verify_plan(
7846 canonical_root.as_path(),
7847 crate::cache_freshness::VerifyArtifact::Search,
7848 Some(seeded_generation),
7849 ),
7850 crate::cache_freshness::WarmVerifyPlan::Strict
7851 ),
7852 "idle eviction must force strict re-verification"
7853 );
7854 assert!(
7855 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7856 );
7857 ctx.mark_subc_bound();
7858 assert!(ctx
7859 .ensure_callgraph_store()
7860 .expect("reopen callgraph store")
7861 .is_some());
7862 assert!(live_roots[&root].idle_artifacts_evicted);
7863 }
7864
7865 #[test]
7866 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7867 let (_root_dir, root) = test_root("idle-root-ttl-gate");
7868 let ctx = test_ctx();
7869 let executor = Arc::new(Executor::new());
7870 assert!(executor.register_actor(root.clone(), ctx));
7871 let ctx = executor.actor_context(&root).expect("actor context");
7872 ctx.mark_subc_unbound();
7873 let now = Instant::now();
7874 let mut meta = RootMeta::new(now);
7875 meta.unbound_quiesced = true;
7876 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7877
7878 assert_eq!(
7882 reap_idle_roots(
7883 now,
7884 &mut live_roots,
7885 &HashMap::new(),
7886 &HashMap::new(),
7887 &executor,
7888 &DispatchPathMetrics::new(),
7889 )
7890 .evicted,
7891 0
7892 );
7893 assert!(!live_roots[&root].idle_artifacts_evicted);
7894
7895 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7901 assert_eq!(
7902 reap_idle_roots(
7903 now + IDLE_ROOT_TTL,
7904 &mut live_roots,
7905 &HashMap::new(),
7906 &HashMap::new(),
7907 &executor,
7908 &DispatchPathMetrics::new(),
7909 )
7910 .evicted,
7911 1
7912 );
7913 assert!(live_roots[&root].idle_artifacts_evicted);
7914 assert!(
7915 ctx.take_pending_search_index_paths().is_empty(),
7916 "TTL eviction must dispose retained pending reconciliation paths"
7917 );
7918 }
7919
7920 #[test]
7921 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7922 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7923 let ctx = test_ctx();
7924 let executor = Arc::new(Executor::new());
7925 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7926 ctx.mark_subc_unbound();
7927
7928 let pending = root.as_path().join("edited-while-unbound.rs");
7934 ctx.add_pending_search_index_paths([pending.clone()]);
7935 let dirty_source = root.as_path().join("dirty.rs");
7936 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7937 let mut dirty = crate::search_index::SearchIndex::new();
7938 dirty.ready = true;
7939 dirty.update_file(&dirty_source);
7940 *ctx.search_index()
7941 .write()
7942 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7943 assert!(ctx.artifact_eviction_blocked());
7944
7945 let mut live_roots = HashMap::new();
7946 let mut meta = RootMeta::new(Instant::now());
7947 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7948 meta.unbound_quiesced = true;
7949 live_roots.insert(root.clone(), meta);
7950
7951 assert_eq!(
7952 reap_idle_roots(
7953 Instant::now(),
7954 &mut live_roots,
7955 &HashMap::new(),
7956 &HashMap::new(),
7957 &executor,
7958 &DispatchPathMetrics::new(),
7959 )
7960 .evicted,
7961 0,
7962 "the dirty index must still block this eviction"
7963 );
7964 assert_eq!(
7965 ctx.take_pending_search_index_paths(),
7966 vec![pending],
7967 "a blocked eviction must restore the taken pending paths"
7968 );
7969 }
7970
7971 #[test]
7972 fn idle_reap_with_bound_route_keeps_watcher_running() {
7973 let (_root_dir, root) = test_root("bound-root-reap-gate");
7974 let ctx = test_ctx();
7975 let executor = Arc::new(Executor::new());
7976 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7977
7978 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7979 let _dispatch_tx = dispatch_tx;
7980 let shutdown = Arc::new(AtomicBool::new(false));
7981 let thread_shutdown = Arc::clone(&shutdown);
7982 let join = std::thread::spawn(move || {
7983 while !thread_shutdown.load(Ordering::SeqCst) {
7984 std::thread::yield_now();
7985 }
7986 });
7987 ctx.install_watcher_runtime(
7988 dispatch_rx,
7989 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7990 );
7991
7992 let mut meta = RootMeta::new(Instant::now());
7993 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7994 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7995 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7996 assert_eq!(
7997 reap_idle_roots(
7998 Instant::now(),
7999 &mut live_roots,
8000 &HashMap::new(),
8001 &bound,
8002 &executor,
8003 &DispatchPathMetrics::new(),
8004 )
8005 .evicted,
8006 0
8007 );
8008 wait_for_watcher_count(&ctx, 1);
8009 ctx.stop_watcher_runtime_in_background();
8010 wait_for_watcher_count(&ctx, 0);
8011 }
8012
8013 #[test]
8014 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8015 let (root_dir, root) = test_root("deleted-bound-root-reap");
8016 let executor = Arc::new(Executor::new());
8017 assert!(executor.register_actor(root.clone(), test_ctx()));
8018 root_dir.close().expect("delete project root");
8019
8020 let route = route_key(19, 3);
8021 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8022 let cancel_signal = PersistentCancelSignal::new();
8023 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8024 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8025 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8026 let mut route_bash_cancels = HashMap::from([(
8027 route,
8028 bash::RouteBashCancel {
8029 token: cancel_signal.clone(),
8030 active_waits: 0,
8031 },
8032 )]);
8033 let metrics = DispatchPathMetrics::new();
8034
8035 let first = reap_idle_roots(
8036 Instant::now(),
8037 &mut live_roots,
8038 &HashMap::new(),
8039 &root_channels,
8040 &executor,
8041 &metrics,
8042 );
8043 assert!(first.forgotten_deleted_roots.is_empty());
8044 assert!(executor.actor_registered(&root));
8045
8046 let mut forgotten = Vec::new();
8047 for _ in 0..100 {
8048 let outcome = reap_idle_roots(
8049 Instant::now(),
8050 &mut live_roots,
8051 &HashMap::new(),
8052 &root_channels,
8053 &executor,
8054 &metrics,
8055 );
8056 if !outcome.forgotten_deleted_roots.is_empty() {
8057 forgotten = outcome.forgotten_deleted_roots;
8058 break;
8059 }
8060 std::thread::sleep(Duration::from_millis(10));
8061 }
8062 assert_eq!(forgotten, vec![root.clone()]);
8063 assert!(!executor.actor_registered(&root));
8064
8065 let mut retry_buffer = HashMap::new();
8066 let mut reclaimed_routes = ReclaimedRoutes::default();
8067 let mut session_identity = HashMap::new();
8068 let mut push_buffer = HashMap::new();
8069 let mut bg_subs = HashMap::from([(
8070 route,
8071 BgSub {
8072 corr: 77,
8073 ver: PROTOCOL_VERSION,
8074 flags: control_flags(),
8075 root: root.clone(),
8076 session: "deleted-route".to_string(),
8077 },
8078 )]);
8079 let mut bg_sub_by_session = HashMap::from([(
8080 (root.clone(), "deleted-route".to_string()),
8081 HashSet::from([route]),
8082 )]);
8083 let mut bg_wake_pending =
8084 BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8085 let mut bg_wake_epoch = HashMap::new();
8086 let mut pending_bash_asks = HashMap::new();
8087 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8088 health::take_bg_observability_logs_for_test();
8089 purge_deleted_root_residents(
8090 &root,
8091 &mut routes,
8092 &mut root_channels,
8093 &mut installed_route_epochs,
8094 &mut route_bash_cancels,
8095 &active_tool_calls,
8096 executor.as_ref(),
8097 &mut retry_buffer,
8098 &mut reclaimed_routes,
8099 &mut session_identity,
8100 &mut push_buffer,
8101 &mut bg_subs,
8102 &mut bg_sub_by_session,
8103 &mut bg_wake_pending,
8104 &mut bg_wake_epoch,
8105 &mut pending_bash_asks,
8106 &metrics,
8107 );
8108
8109 assert!(routes.is_empty());
8110 assert!(root_channels.is_empty());
8111 assert!(installed_route_epochs.is_empty());
8112 assert!(route_bash_cancels.is_empty());
8113 assert!(reclaimed_routes.contains(route));
8114 assert!(cancel_signal.is_cancelled());
8115 assert_eq!(
8116 health::take_bg_observability_logs_for_test(),
8117 vec![format!(
8118 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8119 root.as_path().display()
8120 )]
8121 );
8122 }
8123
8124 #[test]
8131 fn live_root_with_bound_route_is_never_reclaimed() {
8132 let (_root_dir, root) = test_root("live-bound-root-retained");
8133 let ctx = test_ctx();
8134 ctx.mark_subc_unbound();
8135 let executor = Arc::new(Executor::new());
8136 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8137
8138 let route = RouteChannel {
8139 channel: 7,
8140 epoch: 1,
8141 };
8142 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8143 meta.unbound_quiesced = true;
8144 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8145 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8146
8147 for _ in 0..3 {
8152 let outcome = reap_idle_roots(
8153 Instant::now(),
8154 &mut live_roots,
8155 &HashMap::new(),
8156 &root_channels,
8157 &executor,
8158 &DispatchPathMetrics::new(),
8159 );
8160 assert!(
8161 outcome.forgotten_deleted_roots.is_empty(),
8162 "a root whose directory exists must never be forgotten"
8163 );
8164 }
8165
8166 assert!(live_roots.contains_key(&root), "live root must be retained");
8167 assert!(
8168 executor.actor_registered(&root),
8169 "live root's actor must survive"
8170 );
8171 assert!(
8172 root.as_path().exists(),
8173 "test vehicle must keep the directory alive; otherwise this control proves nothing"
8174 );
8175 }
8176
8177 #[test]
8178 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8179 let (root_dir, root) = test_root("deleted-root-first-observation");
8180 let ctx = test_ctx();
8181 ctx.mark_subc_unbound();
8182 let executor = Arc::new(Executor::new());
8183 assert!(executor.register_actor(root.clone(), ctx));
8184 root_dir.close().expect("delete project root");
8185
8186 let mut meta = RootMeta::new(Instant::now());
8187 meta.unbound_quiesced = true;
8188 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8189 let outcome = reap_idle_roots(
8190 Instant::now(),
8191 &mut live_roots,
8192 &HashMap::new(),
8193 &HashMap::new(),
8194 &executor,
8195 &DispatchPathMetrics::new(),
8196 );
8197
8198 assert!(outcome.forgotten_deleted_roots.is_empty());
8199 assert!(live_roots.contains_key(&root));
8200 assert!(executor.actor_registered(&root));
8201 }
8202
8203 fn spawn_background_for_root(
8204 ctx: &AppContext,
8205 root: &ProjectRootId,
8206 storage: &tempfile::TempDir,
8207 session_id: &str,
8208 ) -> (String, u32) {
8209 let command = if cfg!(windows) {
8215 "ping -n 31 127.0.0.1 > nul"
8217 } else {
8218 "sleep 30"
8219 };
8220 let task_id = ctx
8221 .bash_background()
8222 .spawn(
8223 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8224 command,
8225 session_id.to_string(),
8226 storage.path().to_path_buf(),
8227 HashMap::new(),
8228 Some(Duration::from_secs(60)),
8229 storage.path().to_path_buf(),
8230 8,
8231 true,
8232 false,
8233 Some(root.as_path().to_path_buf()),
8234 )
8235 .expect("spawn background task");
8236 let snapshot = ctx
8237 .bash_background()
8238 .status(
8239 &task_id,
8240 session_id,
8241 Some(root.as_path()),
8242 Some(storage.path()),
8243 0,
8244 )
8245 .expect("background task status");
8246 (task_id, snapshot.child_pid.expect("background child pid"))
8247 }
8248
8249 fn wait_for_background_exit(pid: u32) {
8250 let deadline = Instant::now() + Duration::from_secs(5);
8251 while crate::bash_background::process::is_process_alive(pid) {
8252 assert!(
8253 Instant::now() < deadline,
8254 "background task process survived kill"
8255 );
8256 std::thread::sleep(Duration::from_millis(20));
8257 }
8258 }
8259
8260 #[test]
8261 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8262 let (root_dir, root) = test_root("deleted-root-background-task");
8263 let storage = tempfile::tempdir().expect("task storage");
8264 let ctx = test_ctx();
8265 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8266 assert!(crate::bash_background::process::is_process_alive(pid));
8267
8268 let executor = Arc::new(Executor::new());
8269 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8270 assert!(executor.actor_is_idle(&root));
8271 root_dir.close().expect("delete project root");
8272 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8273 let pending_binds = HashMap::new();
8274 let root_channels = HashMap::new();
8275 let metrics = DispatchPathMetrics::new();
8276
8277 let first = reap_idle_roots(
8278 Instant::now(),
8279 &mut live_roots,
8280 &pending_binds,
8281 &root_channels,
8282 &executor,
8283 &metrics,
8284 );
8285 assert!(first.forgotten_deleted_roots.is_empty());
8286 assert!(crate::bash_background::process::is_process_alive(pid));
8287
8288 let outcome = reap_until_forgotten(
8289 &root,
8290 &mut live_roots,
8291 &pending_binds,
8292 &root_channels,
8293 &executor,
8294 &metrics,
8295 );
8296 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8297 wait_for_background_exit(pid);
8298
8299 let snapshot = ctx
8300 .bash_background()
8301 .status(
8302 &task_id,
8303 "reclaim-session",
8304 Some(root.as_path()),
8305 Some(storage.path()),
8306 0,
8307 )
8308 .expect("reclaimed task status");
8309 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8310 assert_eq!(
8311 snapshot.info.status_reason.as_deref(),
8312 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8313 );
8314 assert_eq!(
8315 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8316 crate::bash_background::registry::ROOT_RECLAIMED_REASON
8317 );
8318 let completion = ctx
8319 .bash_background()
8320 .drain_completions_for_session(Some("reclaim-session"))
8321 .pop()
8322 .expect("reclaimed task completion");
8323 assert_eq!(
8324 completion.status_reason.as_deref(),
8325 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8326 );
8327 }
8328
8329 #[test]
8330 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8331 let (root_dir, root) = test_root("existing-root-background-task");
8332 let storage = tempfile::tempdir().expect("task storage");
8333 let ctx = test_ctx();
8334 ctx.mark_subc_unbound();
8335 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8336
8337 let executor = Arc::new(Executor::new());
8338 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8339 let mut meta = RootMeta::new(
8340 Instant::now()
8341 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8342 .expect("old root timestamp"),
8343 );
8344 meta.unbound_quiesced = true;
8345 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8346 let pending_binds = HashMap::new();
8347 let root_channels = HashMap::new();
8348 let metrics = DispatchPathMetrics::new();
8349
8350 for _ in 0..8 {
8351 reap_idle_roots(
8352 Instant::now(),
8353 &mut live_roots,
8354 &pending_binds,
8355 &root_channels,
8356 &executor,
8357 &metrics,
8358 );
8359 std::thread::sleep(Duration::from_millis(10));
8360 }
8361 assert!(root_dir.path().exists());
8362 assert!(crate::bash_background::process::is_process_alive(pid));
8363 let snapshot = ctx
8364 .bash_background()
8365 .status(
8366 &task_id,
8367 "existing-session",
8368 Some(root.as_path()),
8369 Some(storage.path()),
8370 0,
8371 )
8372 .expect("existing task status");
8373 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8374 let _ = ctx.bash_background().kill(&task_id, "existing-session");
8375 wait_for_background_exit(pid);
8376 }
8377
8378 #[test]
8379 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8380 let (root_dir, root) = test_root("restored-root-background-task");
8381 let storage = tempfile::tempdir().expect("task storage");
8382 let ctx = test_ctx();
8383 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8384
8385 let executor = Arc::new(Executor::new());
8386 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8387 root_dir.close().expect("delete project root");
8388 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8389 let pending_binds = HashMap::new();
8390 let root_channels = HashMap::new();
8391 let metrics = DispatchPathMetrics::new();
8392
8393 let first = reap_idle_roots(
8394 Instant::now(),
8395 &mut live_roots,
8396 &pending_binds,
8397 &root_channels,
8398 &executor,
8399 &metrics,
8400 );
8401 assert!(first.forgotten_deleted_roots.is_empty());
8402 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8403 let second = reap_idle_roots(
8404 Instant::now(),
8405 &mut live_roots,
8406 &pending_binds,
8407 &root_channels,
8408 &executor,
8409 &metrics,
8410 );
8411 assert!(second.forgotten_deleted_roots.is_empty());
8412 assert!(crate::bash_background::process::is_process_alive(pid));
8413 let _ = ctx.bash_background().kill(&task_id, "restored-session");
8414 wait_for_background_exit(pid);
8415 }
8416
8417 #[test]
8418 fn observing_root_again_resets_deleted_sweep_confirmation() {
8419 let (root_dir, root) = test_root("deleted-root-observation-reset");
8420 let ctx = test_ctx();
8421 ctx.mark_subc_unbound();
8422 let executor = Arc::new(Executor::new());
8423 assert!(executor.register_actor(root.clone(), ctx));
8424 root_dir.close().expect("delete project root");
8425
8426 let mut meta = RootMeta::new(Instant::now());
8427 meta.unbound_quiesced = true;
8428 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8429 let pending_binds = HashMap::new();
8430 let root_channels = HashMap::new();
8431 let metrics = DispatchPathMetrics::new();
8432
8433 let first = reap_idle_roots(
8434 Instant::now(),
8435 &mut live_roots,
8436 &pending_binds,
8437 &root_channels,
8438 &executor,
8439 &metrics,
8440 );
8441 assert!(first.forgotten_deleted_roots.is_empty());
8442
8443 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8444 reap_idle_roots(
8445 Instant::now(),
8446 &mut live_roots,
8447 &pending_binds,
8448 &root_channels,
8449 &executor,
8450 &metrics,
8451 );
8452 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8453
8454 let after_reset = reap_idle_roots(
8455 Instant::now(),
8456 &mut live_roots,
8457 &pending_binds,
8458 &root_channels,
8459 &executor,
8460 &metrics,
8461 );
8462 assert!(after_reset.forgotten_deleted_roots.is_empty());
8463 assert!(live_roots.contains_key(&root));
8464 assert!(executor.actor_registered(&root));
8465 }
8466
8467 #[test]
8468 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8469 let (root_dir, root) = test_root("deleted-root-reap");
8470 let app = App::default_shared();
8471 let ctx = Arc::new(AppContext::from_app(
8472 Arc::clone(&app),
8473 Config {
8474 project_root: Some(root.as_path().to_path_buf()),
8475 ..Config::default()
8476 },
8477 ));
8478 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8479 ctx.mark_subc_unbound();
8480 let executor = Arc::new(Executor::new());
8481 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8482 assert_eq!(app.actor_root_count(), 1);
8483 drop(ctx);
8484 root_dir.close().expect("delete project root");
8485
8486 let mut meta = RootMeta::new(Instant::now());
8487 meta.unbound_quiesced = true;
8488 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8489 let outcome = reap_until_forgotten(
8490 &root,
8491 &mut live_roots,
8492 &HashMap::new(),
8493 &HashMap::new(),
8494 &executor,
8495 &DispatchPathMetrics::new(),
8496 );
8497 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8498 assert!(!executor.actor_registered(&root));
8499 assert!(!live_roots.contains_key(&root));
8500 wait_for_actor_root_count(&app, 0);
8501
8502 let status_ctx = AppContext::from_app(app, Config::default());
8503 let status = status_ctx.build_status_snapshot();
8504 assert_eq!(status["runtime"]["live_actor_roots"], 0);
8505 assert_eq!(status["runtime"]["open_routes"], 0);
8506 }
8507
8508 #[test]
8509 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8510 let (root_dir, root) = test_root("deleted-root-reap-census");
8511 let executor = Arc::new(Executor::new());
8512 assert!(executor.register_actor(root.clone(), test_ctx()));
8513 root_dir.close().expect("delete project root");
8514
8515 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8516 let metrics = DispatchPathMetrics::new();
8517 let outcome = reap_idle_roots(
8518 Instant::now(),
8519 &mut live_roots,
8520 &HashMap::new(),
8521 &HashMap::new(),
8522 &executor,
8523 &metrics,
8524 );
8525 assert_eq!(outcome.evicted, 0);
8526
8527 let app = crate::context::App::default_shared();
8528 let health_rollup_cache = HealthRollupCache::new();
8529 health_rollup_cache.refresh(&executor, &app);
8530 let report = build_health_report(
8531 &health_rollup_cache,
8532 &executor,
8533 &HashMap::new(),
8534 &metrics,
8535 &app,
8536 );
8537 let reap = report
8538 .metrics
8539 .as_ref()
8540 .and_then(|metrics| metrics.get("reap"))
8541 .expect("reap health metrics");
8542 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8543 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8544 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8545 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8546 }
8547
8548 #[test]
8549 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8550 let (root_dir, root) = test_root("connection-exit-deleted-root");
8551 let executor = Arc::new(Executor::new());
8552 assert!(executor.register_actor(root.clone(), test_ctx()));
8553
8554 let route = route_key(11, 1);
8555 let mut meta = RootMeta::new(Instant::now());
8556 meta.maintenance_pending = true;
8557 meta.maintenance_queued_kinds
8558 .push_back(MaintenanceDrainKind::CompletionDrains);
8559 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8560 let mut pending_binds = HashMap::new();
8561 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8562 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8563 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8564 let mut route_bash_cancels = HashMap::new();
8565 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8566
8567 quiesce_connection_roots(
8568 &mut live_roots,
8569 &mut pending_binds,
8570 &mut routes,
8571 &mut root_channels,
8572 &mut installed_route_epochs,
8573 &mut route_bash_cancels,
8574 &active_tool_calls,
8575 &executor,
8576 );
8577 assert!(live_roots[&root].unbound_quiesced);
8578 assert!(!live_roots[&root].maintenance_pending);
8579 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8580 assert!(routes.is_empty());
8581 assert!(root_channels.is_empty());
8582
8583 root_dir.close().expect("delete project root");
8584 let metrics = DispatchPathMetrics::new();
8585 let outcome = reap_until_forgotten(
8586 &root,
8587 &mut live_roots,
8588 &pending_binds,
8589 &root_channels,
8590 &executor,
8591 &metrics,
8592 );
8593 let mut session_identity = HashMap::new();
8594 let mut push_buffer = HashMap::new();
8595 let mut bg_subs = HashMap::new();
8596 let mut bg_sub_by_session = HashMap::new();
8597 let mut bg_wake_pending = BgWakePending::new();
8598 let mut bg_wake_epoch = HashMap::new();
8599 let mut pending_bash_asks = HashMap::new();
8600 let mut retry_buffer = HashMap::new();
8601 let mut reclaimed_routes = ReclaimedRoutes::default();
8602 for forgotten in &outcome.forgotten_deleted_roots {
8603 purge_deleted_root_residents(
8604 forgotten,
8605 &mut routes,
8606 &mut root_channels,
8607 &mut installed_route_epochs,
8608 &mut route_bash_cancels,
8609 &active_tool_calls,
8610 executor.as_ref(),
8611 &mut retry_buffer,
8612 &mut reclaimed_routes,
8613 &mut session_identity,
8614 &mut push_buffer,
8615 &mut bg_subs,
8616 &mut bg_sub_by_session,
8617 &mut bg_wake_pending,
8618 &mut bg_wake_epoch,
8619 &mut pending_bash_asks,
8620 &metrics,
8621 );
8622 }
8623
8624 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8625 assert!(!executor.actor_registered(&root));
8626 assert!(!live_roots.contains_key(&root));
8627 }
8628
8629 #[test]
8630 fn unbound_root_quiesces_maintenance_without_removing_actor() {
8631 let (_root_dir, root) = test_root("unbound-root-quiesce");
8632 let ctx = test_ctx();
8633 let executor = Arc::new(Executor::new());
8634 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8635 let mut meta = RootMeta::new(Instant::now());
8636 meta.maintenance_pending = true;
8637 meta.maintenance_jobs_in_flight = 1;
8638 meta.maintenance_queued_kinds
8639 .push_back(MaintenanceDrainKind::ConfigureTail);
8640 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8641 *ctx.search_index()
8643 .write()
8644 .unwrap_or_else(std::sync::PoisonError::into_inner) =
8645 Some(crate::search_index::SearchIndex::new());
8646 ctx.set_cache_writer_capabilities(true, true);
8647 let pending = root.as_path().join("pending.rs");
8648 ctx.add_pending_search_index_paths([pending.clone()]);
8649 let canonical_root = root.as_path().to_path_buf();
8653 let artifact = canonical_root.join("cache.bin");
8654 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8655 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8656 crate::cache_freshness::record_verify_completed(
8657 &canonical_root,
8658 crate::cache_freshness::VerifyArtifact::Search,
8659 seeded_generation,
8660 );
8661 assert!(matches!(
8662 crate::cache_freshness::warm_verify_plan(
8663 &canonical_root,
8664 crate::cache_freshness::VerifyArtifact::Search,
8665 seeded_generation,
8666 ),
8667 crate::cache_freshness::WarmVerifyPlan::Skip
8668 ));
8669 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8672 let _dispatch_tx = dispatch_tx;
8673 let shutdown = Arc::new(AtomicBool::new(false));
8674 let thread_shutdown = Arc::clone(&shutdown);
8675 let join = std::thread::spawn(move || {
8676 while !thread_shutdown.load(Ordering::SeqCst) {
8677 std::thread::yield_now();
8678 }
8679 });
8680 ctx.install_watcher_runtime(
8681 dispatch_rx,
8682 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8683 );
8684 assert!(ctx.watcher_runtime_active());
8685
8686 quiesce_unbound_root(&root, &mut live_roots, &executor);
8687 let meta = &live_roots[&root];
8688 assert!(meta.unbound_quiesced);
8689 assert!(ctx.subc_unbound_quiesced());
8690 assert!(meta.maintenance_pending);
8691 assert!(meta.maintenance_queued_kinds.is_empty());
8692 assert!(executor.actor_registered(&root));
8693 assert!(
8697 ctx.search_index()
8698 .read()
8699 .unwrap_or_else(std::sync::PoisonError::into_inner)
8700 .is_some(),
8701 "quiesce must not evict resident artifacts"
8702 );
8703 assert_eq!(
8704 ctx.pending_callgraph_store_force_token(),
8705 None,
8706 "quiesce must not force a callgraph rebuild"
8707 );
8708 assert_eq!(
8709 ctx.take_pending_search_index_paths(),
8710 vec![pending],
8711 "quiesce must retain pending watcher-derived paths"
8712 );
8713 assert!(
8714 matches!(
8715 crate::cache_freshness::warm_verify_plan(
8716 &canonical_root,
8717 crate::cache_freshness::VerifyArtifact::Search,
8718 seeded_generation,
8719 ),
8720 crate::cache_freshness::WarmVerifyPlan::Skip
8721 ),
8722 "quiesce must not invalidate the warm verify memo"
8723 );
8724 assert!(
8725 ctx.watcher_runtime_active(),
8726 "quiesce must not stop a running watcher"
8727 );
8728 ctx.stop_watcher_runtime();
8729
8730 let meta = live_roots.get_mut(&root).expect("root metadata");
8731 note_maintenance_completion(
8732 meta,
8733 Some(MaintenanceDrainKind::ConfigureTail),
8734 false,
8735 meta.unbound_quiesced,
8736 );
8737 assert!(!meta.maintenance_pending);
8738 assert!(meta.maintenance_queued_kinds.is_empty());
8739 }
8740
8741 #[test]
8742 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8743 let (_dir, root) = test_root("same-root-replacement");
8744 let route = route_key(7, 1);
8745 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8746 let root_channels = HashMap::new();
8747
8748 assert!(!route_removal_will_quiesce_root(
8749 &root,
8750 route,
8751 &installed_channels,
8752 false,
8753 Some(&root),
8754 ));
8755 assert!(route_removal_will_quiesce_root(
8756 &root,
8757 route,
8758 &installed_channels,
8759 false,
8760 None,
8761 ));
8762 assert!(!should_quiesce_removed_root(
8763 &root,
8764 &root_channels,
8765 false,
8766 Some(&root),
8767 ));
8768 assert!(should_quiesce_removed_root(
8769 &root,
8770 &root_channels,
8771 false,
8772 None,
8773 ));
8774 assert!(!should_quiesce_removed_root(
8775 &root,
8776 &root_channels,
8777 true,
8778 None,
8779 ));
8780 }
8781
8782 #[test]
8783 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8784 let (_root_dir, root) = test_root("unbound-root-route-count");
8785 let executor = Arc::new(Executor::new());
8786 assert!(executor.register_actor(root.clone(), test_ctx()));
8787 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8788 let mut root_channels = HashMap::from([(
8789 root.clone(),
8790 HashSet::from([route_key(7, 1), route_key(8, 1)]),
8791 )]);
8792
8793 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8794 if !root_channels.contains_key(&root) {
8795 quiesce_unbound_root(&root, &mut live_roots, &executor);
8796 }
8797 assert!(!live_roots[&root].unbound_quiesced);
8798
8799 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8800 if !root_channels.contains_key(&root) {
8801 quiesce_unbound_root(&root, &mut live_roots, &executor);
8802 }
8803 assert!(live_roots[&root].unbound_quiesced);
8804
8805 live_roots
8806 .get_mut(&root)
8807 .expect("root metadata")
8808 .note_activity();
8809 assert!(
8810 live_roots[&root].unbound_quiesced,
8811 "late asynchronous activity must not reactivate an unbound root"
8812 );
8813
8814 live_roots
8815 .get_mut(&root)
8816 .expect("root metadata")
8817 .reactivate_bound();
8818 assert!(!live_roots[&root].unbound_quiesced);
8819 }
8820
8821 #[test]
8822 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8823 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8824 let (_active_dir, active_root) = test_root("allocator-relief-active");
8825 let now = Instant::now();
8826 let mut live_roots = HashMap::new();
8827 let mut idle = RootMeta::new(now);
8828 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8829 live_roots.insert(idle_root, idle);
8830 let executor = Executor::new();
8831 assert!(process_has_been_idle(now, &live_roots, &executor));
8832
8833 live_roots.insert(active_root.clone(), RootMeta::new(now));
8834 assert!(!process_has_been_idle(now, &live_roots, &executor));
8835
8836 let active = live_roots
8837 .get_mut(&active_root)
8838 .expect("active root metadata");
8839 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8840 active.active_bash_waits = 1;
8841 assert!(!process_has_been_idle(now, &live_roots, &executor));
8842 }
8843
8844 #[test]
8845 fn pressure_relief_log_reports_before_and_after_measurements() {
8846 let allocator = crate::memory::AllocatorMemorySnapshot {
8847 status: "measured",
8848 bytes_in_use: Some(8 * 1024 * 1024),
8849 size_allocated: Some(12 * 1024 * 1024),
8850 retained_slack_bytes: Some(4 * 1024 * 1024),
8851 not_estimated: None,
8852 };
8853 let relief = crate::memory::AllocatorPressureRelief {
8854 bytes_released: 3 * 1024 * 1024,
8855 rss_before_bytes: Some(20 * 1024 * 1024),
8856 rss_after_bytes: Some(17 * 1024 * 1024),
8857 allocator_before: allocator.clone(),
8858 allocator_after: crate::memory::AllocatorMemorySnapshot {
8859 size_allocated: Some(9 * 1024 * 1024),
8860 retained_slack_bytes: Some(1024 * 1024),
8861 ..allocator
8862 },
8863 };
8864 let message = pressure_relief_label(&relief);
8865 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8866 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8867 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8868 assert!(message.contains("reported 3.0 MB released"));
8869 }
8870
8871 #[test]
8872 fn due_maintenance_jobs_skip_poisoned_roots() {
8873 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8874 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8875 let mut live_roots = HashMap::new();
8876 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8877 let mut poisoned_meta = RootMeta::new(Instant::now());
8878 poisoned_meta.maintenance_poisoned = true;
8879 live_roots.insert(poisoned_root.clone(), poisoned_meta);
8880
8881 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8882 &mut live_roots,
8883 MAINTENANCE_SUBMIT_BUDGET,
8884 &HashSet::new(),
8885 );
8886
8887 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8888 assert!(due.iter().all(|(root, _)| root == &healthy_root));
8889 assert!(!deferred);
8890 assert!(live_roots[&healthy_root].maintenance_pending);
8891 assert_eq!(
8892 live_roots[&healthy_root].maintenance_jobs_in_flight,
8893 INITIAL_MAINTENANCE_JOB_COUNT
8894 );
8895 assert!(!live_roots[&poisoned_root].maintenance_pending);
8896 }
8897
8898 #[test]
8899 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8900 let (_dir, root) = test_root("maintenance-unbound");
8901 let mut meta = RootMeta::new(Instant::now());
8902 meta.unbound_quiesced = true;
8903 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8904
8905 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8906 &mut live_roots,
8907 MAINTENANCE_SUBMIT_BUDGET,
8908 &HashSet::new(),
8909 );
8910
8911 assert!(due.is_empty());
8912 assert!(!deferred);
8913 assert!(!live_roots[&root].maintenance_pending);
8914 }
8915
8916 #[test]
8917 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8918 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8919 let ctx = test_ctx();
8920 assert!(!ctx.completion_drains_have_work());
8921
8922 let executor = Executor::new();
8923 assert!(executor.register_actor(root.clone(), ctx));
8924 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8925 let session = "idle-session".to_string();
8926 let channel = route_key(17, 1);
8927 let metrics = DispatchPathMetrics::new();
8928 let bg_sub_by_session =
8929 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8930 let mut bg_wake_pending = BgWakePending::new();
8931
8932 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8933 &mut live_roots,
8934 Some(&executor),
8935 &bg_sub_by_session,
8936 &bg_wake_pending,
8937 MAINTENANCE_SUBMIT_BUDGET,
8938 &HashSet::new(),
8939 );
8940 assert!(idle_tick_jobs.is_empty());
8941 assert!(!deferred);
8942 assert!(!live_roots[&root].maintenance_pending);
8943
8944 let mut bg_wake_epoch = HashMap::new();
8947 push::arm_bg_wake(
8948 root.clone(),
8949 session,
8950 channel,
8951 &mut bg_wake_pending,
8952 &mut bg_wake_epoch,
8953 &metrics,
8954 );
8955 let (next_tick_jobs, deferred) = due_maintenance_jobs(
8956 &mut live_roots,
8957 Some(&executor),
8958 &bg_sub_by_session,
8959 &bg_wake_pending,
8960 MAINTENANCE_SUBMIT_BUDGET,
8961 &HashSet::new(),
8962 );
8963 assert_eq!(
8964 next_tick_jobs,
8965 vec![(root, MaintenanceDrainKind::CompletionDrains)]
8966 );
8967 assert!(!deferred);
8968 }
8969
8970 async fn assert_slow_configure_tail_admission(
8971 config: crate::executor::ExecutorConfig,
8972 shape: &'static str,
8973 ) {
8974 let root_dir = tempfile::tempdir().unwrap();
8975 let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8976 let root = ProjectRootId::from_path(&root_path).unwrap();
8977 let ctx = test_ctx();
8978 ctx.mark_subc_bound();
8979 let storage_root = ctx.storage_dir();
8980 ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8981 generation: ctx.configure_generation(),
8982 root_path: root_path.clone(),
8983 canonical_cache_root: root_path.clone(),
8984 harness: crate::harness::Harness::Opencode,
8985 storage_root: storage_root.clone(),
8986 harness_dir: storage_root.join("opencode"),
8987 session_id: "first-search-admission".to_string(),
8988 home_match: false,
8989 format_tool_cache_clear_needed: false,
8990 run_bash_replay: false,
8991 refresh_project_runtime: false,
8992 sync_bash_compress_flag: false,
8993 reset_filter_registry: false,
8994 clear_failed_spawns: false,
8995 warm_callgraph_store: false,
8996 supersede_search_artifact_persistence: false,
8997 supersede_callgraph_artifact_persistence: false,
8998 supersede_semantic_artifact_persistence: false,
8999 search_artifact_load_start: None,
9000 semantic_artifact_load_start: None,
9001 })
9002 .expect("queue configure maintenance");
9003 let (_gate, maintenance_reached, release_maintenance) =
9004 crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9005 root_path.clone(),
9006 );
9007
9008 let expected_pool_size = config.pool_size;
9009 let expected_actor_cap = config.actor_cap;
9010 let executor = Arc::new(Executor::with_config(config));
9011 assert_eq!(executor.pool_size(), expected_pool_size);
9012 assert_eq!(executor.actor_cap(), expected_actor_cap);
9013 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9014 let metrics = Arc::new(DispatchPathMetrics::new());
9015 let (completion_tx, mut completion_rx) = mpsc::channel(2);
9016 submit_maintenance_job(
9017 &executor,
9018 root.clone(),
9019 MaintenanceDrainKind::ConfigureTail,
9020 Vec::new(),
9021 &completion_tx,
9022 &metrics,
9023 );
9024 maintenance_reached
9025 .recv_timeout(Duration::from_secs(2))
9026 .expect("configure tail reached gate");
9027
9028 let admission_started = std::time::Instant::now();
9029 let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9030 let search = executor.submit_async(
9031 root.clone(),
9032 Lane::HeavyInit,
9033 "first-search".to_string(),
9034 Box::new(move |_ctx| {
9035 search_admitted_tx
9036 .send(())
9037 .expect("signal search admission");
9038 Response::success("first-search", json!({}))
9039 }),
9040 );
9041 let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9042 let mutation = executor.submit_async(
9043 root,
9044 Lane::Mutating,
9045 "queued-mutation".to_string(),
9046 Box::new(move |_ctx| {
9047 mutation_started_tx.send(()).expect("signal mutation start");
9048 Response::success("queued-mutation", json!({}))
9049 }),
9050 );
9051
9052 let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9058 let admission_elapsed = admission_started.elapsed();
9059 let mutation_waited = mutation_started_rx.try_recv().is_err();
9060 release_maintenance
9061 .send(())
9062 .expect("release configure maintenance");
9063 search_admission
9064 .expect("first search must admit while configure maintenance remains gated");
9065 eprintln!(
9066 "first-search admission while configure maintenance is gated ({shape}): {}ms",
9067 admission_elapsed.as_millis()
9068 );
9069 assert!(
9070 mutation_waited,
9071 "mutating work must wait for configure maintenance to release its read epoch"
9072 );
9073 tokio::time::timeout(Duration::from_secs(5), search)
9074 .await
9075 .expect("first search completion timed out")
9076 .expect("first search completion channel closed");
9077 tokio::time::timeout(Duration::from_secs(5), mutation)
9078 .await
9079 .expect("mutation completion timed out")
9080 .expect("mutation completion channel closed");
9081 tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9082 .await
9083 .expect("configure-tail completion timed out")
9084 .expect("configure-tail completion channel closed");
9085 }
9086
9087 #[tokio::test]
9088 async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9089 assert_slow_configure_tail_admission(
9090 crate::executor::ExecutorConfig {
9091 pool_size: 2,
9092 read_cap: 1,
9093 actor_cap: 1,
9094 heavy_permits: 1,
9095 drr_quantum: 1,
9096 },
9097 "pool=2 actor_cap=1",
9098 )
9099 .await;
9100 assert_slow_configure_tail_admission(
9101 crate::executor::ExecutorConfig {
9102 pool_size: 4,
9103 read_cap: 3,
9104 actor_cap: 3,
9105 heavy_permits: 3,
9106 drr_quantum: 1,
9107 },
9108 "pool=4 actor_cap=3",
9109 )
9110 .await;
9111 }
9112
9113 #[tokio::test]
9114 async fn subc_configure_tail_precedes_completed_search_install() {
9115 let root_dir = tempfile::tempdir().unwrap();
9116 let storage = tempfile::tempdir().unwrap();
9117 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9118 let (ctx, ignored_path) =
9119 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9120 let ctx = Arc::new(ctx);
9121 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9122 &ctx,
9123 &ignored_path
9124 ));
9125
9126 let executor = Arc::new(Executor::new());
9127 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9128 let metrics = Arc::new(DispatchPathMetrics::new());
9129 let (completion_tx, mut completion_rx) = mpsc::channel(4);
9130 submit_maintenance_job(
9131 &executor,
9132 root.clone(),
9133 MaintenanceDrainKind::ConfigureTail,
9134 Vec::new(),
9135 &completion_tx,
9136 &metrics,
9137 );
9138 submit_maintenance_job(
9139 &executor,
9140 root,
9141 MaintenanceDrainKind::CompletionDrains,
9142 Vec::new(),
9143 &completion_tx,
9144 &metrics,
9145 );
9146
9147 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9148 .await
9149 .expect("configure-tail completion timed out")
9150 .expect("configure-tail completion channel closed");
9151 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9152 .await
9153 .expect("completion-drains completion timed out")
9154 .expect("completion-drains completion channel closed");
9155 assert!(first.response.id.contains("configure-tail"));
9156 assert!(second.response.id.contains("completion-drains"));
9157 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9158 &ctx,
9159 &ignored_path
9160 ));
9161 assert_eq!(
9162 ctx.search_index()
9163 .read()
9164 .unwrap_or_else(std::sync::PoisonError::into_inner)
9165 .as_ref()
9166 .expect("completed search index installed")
9167 .file_count(),
9168 0,
9169 "configure must install the ignore matcher before pending paths replay"
9170 );
9171 ctx.stop_watcher_runtime();
9172 }
9173
9174 #[test]
9175 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9176 let (_dir, root) = test_root("maintenance-post-bind");
9177 let mut live_roots = HashMap::new();
9178 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9179
9180 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9181 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9182
9183 let meta = live_roots.get(&root).expect("root metadata");
9184 assert!(meta.maintenance_pending);
9185 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9186 assert_eq!(
9187 meta.maintenance_queued_kinds
9188 .iter()
9189 .copied()
9190 .collect::<Vec<_>>(),
9191 vec![
9192 MaintenanceDrainKind::ConfigureTail,
9193 MaintenanceDrainKind::CompletionDrains,
9194 ]
9195 );
9196
9197 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9198 &mut live_roots,
9199 MAINTENANCE_SUBMIT_BUDGET,
9200 &HashSet::new(),
9201 );
9202
9203 assert_eq!(
9204 due,
9205 vec![
9206 (root.clone(), MaintenanceDrainKind::ConfigureTail),
9207 (root.clone(), MaintenanceDrainKind::CompletionDrains),
9208 ]
9209 );
9210 assert!(!deferred);
9211 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9212 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9213 }
9214
9215 #[test]
9216 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9217 let mut live_roots = HashMap::new();
9218 let mut root_ids = Vec::new();
9219 let mut _dirs = Vec::new();
9220 for index in 0..4 {
9221 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9222 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9223 root_ids.push(root_id);
9224 _dirs.push(dir);
9225 }
9226
9227 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9228 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9229 &mut live_roots,
9230 small_budget,
9231 &HashSet::new(),
9232 );
9233
9234 assert_eq!(first_due.len(), small_budget);
9235 assert!(first_deferred);
9236 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9237 assert!(first_due_set
9238 .iter()
9239 .all(|root| live_roots[root].maintenance_pending));
9240 assert!(first_due_set
9241 .iter()
9242 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9243
9244 let all_roots: HashSet<_> = root_ids.into_iter().collect();
9245 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9246 assert!(deferred_roots
9247 .iter()
9248 .all(|root| !live_roots[root].maintenance_pending));
9249 }
9250
9251 #[test]
9252 fn due_maintenance_jobs_defers_pending_bind_roots() {
9253 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9254 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9255 let mut live_roots = HashMap::new();
9256 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9257 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9258 let pending_bind_roots = HashSet::from([bind_root.clone()]);
9259
9260 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9261 &mut live_roots,
9262 usize::MAX,
9263 &pending_bind_roots,
9264 );
9265
9266 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9267 assert!(due.iter().all(|(root, _)| root == &healthy_root));
9268 assert!(!deferred);
9269 assert!(!live_roots[&bind_root].maintenance_pending);
9270 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9271 }
9272
9273 #[test]
9274 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9275 let (_dir, root) = test_root("maintenance-requeue");
9276 let mut live_roots = HashMap::new();
9277 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9278 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9279 &mut live_roots,
9280 usize::MAX,
9281 &HashSet::new(),
9282 );
9283 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9284 assert!(due.iter().all(|(due_root, _)| due_root == &root));
9285 assert!(!deferred);
9286
9287 let meta = live_roots.get_mut(&root).unwrap();
9288 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9289 assert!(meta.maintenance_pending);
9290 assert_eq!(
9291 meta.maintenance_jobs_in_flight,
9292 INITIAL_MAINTENANCE_JOB_COUNT - 1
9293 );
9294 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9295
9296 let (requeued, deferred) =
9297 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9298 assert_eq!(
9299 requeued,
9300 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9301 );
9302 assert!(!deferred);
9303 let meta = live_roots.get_mut(&root).unwrap();
9304 assert_eq!(
9305 meta.maintenance_jobs_in_flight,
9306 INITIAL_MAINTENANCE_JOB_COUNT
9307 );
9308 assert!(meta.maintenance_queued_kinds.is_empty());
9309
9310 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9311 note_maintenance_completion(meta, None, false, false);
9312 }
9313 assert!(!meta.maintenance_pending);
9314 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9315 }
9316
9317 #[test]
9318 fn maintenance_requeue_drops_while_bind_is_pending() {
9319 let (_dir, root) = test_root("maintenance-bind-requeue");
9320 let mut live_roots = HashMap::new();
9321 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9322 let (due, _) = due_maintenance_jobs_without_actor_context(
9323 &mut live_roots,
9324 usize::MAX,
9325 &HashSet::new(),
9326 );
9327 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9328
9329 let meta = live_roots.get_mut(&root).unwrap();
9330 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9331
9332 assert_eq!(
9333 meta.maintenance_jobs_in_flight,
9334 INITIAL_MAINTENANCE_JOB_COUNT - 1
9335 );
9336 assert!(meta.maintenance_queued_kinds.is_empty());
9337 assert!(meta.maintenance_pending);
9338 }
9339
9340 #[test]
9341 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9342 let mut meta = RootMeta::new(Instant::now());
9343 meta.unbound_quiesced = true;
9344
9345 assert!(!should_requiesce_after_maintenance(
9346 &meta,
9347 MaintenanceDrainKind::Lsp,
9348 false,
9349 ));
9350 assert!(!should_requiesce_after_maintenance(
9351 &meta,
9352 MaintenanceDrainKind::ConfigureTail,
9353 true,
9354 ));
9355 assert!(should_requiesce_after_maintenance(
9356 &meta,
9357 MaintenanceDrainKind::ConfigureTail,
9358 false,
9359 ));
9360 }
9361
9362 #[test]
9363 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9364 let (_dir, root) = test_root("maintenance-fatal");
9365 let mut live_roots = HashMap::new();
9366 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9367 let (due, _) = due_maintenance_jobs_without_actor_context(
9368 &mut live_roots,
9369 usize::MAX,
9370 &HashSet::new(),
9371 );
9372 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9373
9374 let meta = live_roots.get_mut(&root).unwrap();
9375 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9376 assert!(meta.maintenance_poisoned);
9377 assert!(meta.maintenance_queued_kinds.is_empty());
9378
9379 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9380 note_maintenance_completion(meta, None, false, false);
9381 }
9382 assert!(!meta.maintenance_pending);
9383 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9384 }
9385
9386 #[test]
9387 fn trust_for_principal_matrix() {
9388 assert_eq!(
9389 trust_for_principal(&Some(Principal::Direct)),
9390 BindTrust::FirstParty
9391 );
9392 for module_id in [
9399 "llm-runner",
9400 "aft",
9401 "broca",
9402 "alfonso-core",
9403 "prefrontal",
9404 "prefrontal-core",
9405 ] {
9406 assert_eq!(
9407 trust_for_principal(&Some(Principal::Reserved {
9408 module_id: module_id.to_string(),
9409 })),
9410 BindTrust::FirstParty,
9411 "reserved module id '{module_id}' must resolve to first-party trust"
9412 );
9413 }
9414 assert_eq!(
9415 trust_for_principal(&Some(Principal::Reserved {
9416 module_id: "subc-mcp".to_string(),
9417 })),
9418 BindTrust::Untrusted
9419 );
9420 assert_eq!(
9421 trust_for_principal(&Some(Principal::Reserved {
9422 module_id: "anything-unknown".to_string(),
9423 })),
9424 BindTrust::Untrusted
9425 );
9426 assert_eq!(
9427 trust_for_principal(&Some(Principal::Unverified)),
9428 BindTrust::Untrusted
9429 );
9430 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9431 }
9432
9433 #[test]
9434 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9435 let principal = Some(Principal::Direct);
9436 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9437 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9438
9439 assert_eq!(
9440 trust_for_bind(fingerprint_a, &principal),
9441 BindTrust::Untrusted
9442 );
9443 assert_eq!(
9444 trust_for_bind(fingerprint_b, &principal),
9445 BindTrust::Untrusted
9446 );
9447 }
9448
9449 #[test]
9456 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9457 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9458 assert_eq!(
9459 trust_for_bind(harness, &Some(Principal::Direct)),
9460 BindTrust::FirstParty,
9461 "a direct principal must stay first-party on {harness}"
9462 );
9463 assert_eq!(
9464 trust_for_bind(harness, &Some(Principal::Unverified)),
9465 BindTrust::Untrusted,
9466 "an unverified principal must stay untrusted on {harness}"
9467 );
9468 assert_eq!(
9469 trust_for_bind(harness, &None),
9470 BindTrust::Untrusted,
9471 "an absent principal must fail closed on {harness}"
9472 );
9473 assert_eq!(
9474 trust_for_bind(
9475 harness,
9476 &Some(Principal::Reserved {
9477 module_id: "subc-mcp".to_string(),
9478 })
9479 ),
9480 BindTrust::Untrusted,
9481 "a non-allowlisted reserved module must stay untrusted on {harness}"
9482 );
9483 }
9484 }
9485
9486 #[tokio::test]
9487 async fn persistent_cancel_resolves_when_fired_before_await() {
9488 let signal = PersistentCancelSignal::new();
9492 signal.cancel();
9493 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9495 .await
9496 .expect("cancelled() must resolve when cancel fired beforehand");
9497
9498 let racing = PersistentCancelSignal::new();
9500 let racing_for_task = racing.clone();
9501 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9502 racing.cancel();
9503 tokio::time::timeout(Duration::from_secs(1), waiter)
9504 .await
9505 .expect("cancelled() must resolve when cancel races the await")
9506 .expect("waiter task panicked");
9507 }
9508
9509 #[test]
9510 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9511 let installed = HashMap::from([(7, 9)]);
9512 let mut reclaimed = ReclaimedRoutes::default();
9513 reclaimed.insert(route_key(8, 1));
9514 for ty in [
9515 FrameType::Request,
9516 FrameType::Response,
9517 FrameType::Error,
9518 FrameType::Push,
9519 FrameType::Cancel,
9520 FrameType::Goodbye,
9521 ] {
9522 let body = if ty.is_pure_header() {
9523 Vec::new()
9524 } else {
9525 br#"{}"#.to_vec()
9526 };
9527 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9528 assert!(
9529 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9530 "{ty:?}"
9531 );
9532 }
9533
9534 let reclaimed_request = Frame::build(
9535 FrameType::Request,
9536 control_flags(),
9537 8,
9538 1,
9539 42,
9540 br#"{}"#.to_vec(),
9541 )
9542 .unwrap();
9543 assert!(ingress_route_should_be_processed(
9544 &installed,
9545 &reclaimed,
9546 &reclaimed_request
9547 ));
9548
9549 let never_installed = Frame::build(
9550 FrameType::Request,
9551 control_flags(),
9552 9,
9553 1,
9554 43,
9555 br#"{}"#.to_vec(),
9556 )
9557 .unwrap();
9558 assert!(!ingress_route_should_be_processed(
9559 &installed,
9560 &reclaimed,
9561 &never_installed
9562 ));
9563
9564 let current = Frame::build(
9565 FrameType::Request,
9566 control_flags(),
9567 7,
9568 9,
9569 43,
9570 br#"{}"#.to_vec(),
9571 )
9572 .unwrap();
9573 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9574 assert!(ingress_route_should_be_processed(
9575 &installed, &reclaimed, ¤t
9576 ));
9577 assert!(ingress_route_should_be_processed(
9578 &installed, &reclaimed, &control
9579 ));
9580 assert_eq!(installed, HashMap::from([(7, 9)]));
9581 }
9582
9583 #[tokio::test]
9584 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9585 let (_dir, root) = test_root("route-bind-b2-ordering");
9586 let route = route_key(7, 3);
9587 let identity = RouteIdentity(Arc::new(RouteIdentityData {
9588 root: root.clone(),
9589 project_root: root.as_path().to_path_buf(),
9590 harness: "opencode".to_string(),
9591 session: "b2-session".to_string(),
9592 trust: BindTrust::FirstParty,
9593 spawn_principal: AuthenticatedPrincipal::FirstParty,
9594 consumer_elicitation_capable: false,
9595 }));
9596 let replay_key = push::ReplayKey::from_identity(&identity);
9597 let completion = RouteBindCompletion {
9598 route,
9599 identity,
9600 bind_root_id: root.clone(),
9601 inserted_new_actor: false,
9602 configure_response: Response::success("subc-bind-7", json!({})),
9603 diagnostics_on_edit: false,
9604 ver: PROTOCOL_VERSION,
9605 corr: 91,
9606 flags: control_flags(),
9607 };
9608 let mut pending_binds = HashMap::from([(
9609 route,
9610 PendingBind {
9611 bind_root_id: root,
9612 inserted_new_actor: false,
9613 cancelled: false,
9614 configure_request_id: "subc-bind-7".to_string(),
9615 started_at: Instant::now(),
9616 warned_half_deadline: false,
9617 deadline_reported: false,
9618 corr: 91,
9619 ver: PROTOCOL_VERSION,
9620 flags: control_flags(),
9621 cancellation: crate::executor::JobCancellation::new(),
9622 },
9623 )]);
9624 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9625 let mut push_buffer =
9626 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9627 let (writer_tx, mut writer_rx) = mpsc::channel(8);
9628 let metrics = Arc::new(DispatchPathMetrics::new());
9629 let executor = Arc::new(Executor::new());
9630 let standing_actor =
9631 standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9632
9633 handle_route_bind_completion(
9634 &writer_tx,
9635 completion,
9636 &mut HashMap::new(),
9637 &mut HashMap::new(),
9638 &mut HashMap::new(),
9639 &mut push_buffer,
9640 &mut HashMap::new(),
9641 &mut pending_binds,
9642 &mut installed_route_epochs,
9643 &executor,
9644 &standing_actor,
9645 &Arc::new(Notify::new()),
9646 &metrics,
9647 None,
9648 )
9649 .await
9650 .unwrap();
9651
9652 let ack = writer_rx.try_recv().expect("RouteBindAck");
9653 assert_eq!(ack.header.ty, FrameType::Response);
9654 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9655 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9656 assert_eq!(route_frame.header.ty, FrameType::Push);
9657 assert_eq!(
9658 (route_frame.header.channel, route_frame.header.epoch),
9659 (route.channel, route.epoch)
9660 );
9661 }
9662}