1pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38 finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39 PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46 ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47 StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51 MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55 MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71const HELLO_CORR: u64 = 1;
73
74const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106 MaintenanceDrainKind::Watcher,
107 MaintenanceDrainKind::Lsp,
108 MaintenanceDrainKind::ConfigureTail,
109 MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135 channel: u16,
136 epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 write!(f, "{}@{}", self.channel, self.epoch)
142 }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156 build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157 DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158 ReapBlockerCensus, ResponseTaskGuard,
159};
160pub(crate) use self::manifest::is_subc_native_plumbing_tool;
161use self::manifest::{
162 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163 is_subc_agent_core_tool,
164};
165pub use self::wire::SubcError;
166
167#[doc(hidden)]
172#[derive(Clone, Debug, PartialEq, Eq)]
173pub enum SubcLifecycleEvent {
174 AttachDecision {
175 attempt: u32,
176 will_retry: bool,
177 },
178 RouteDetached {
179 route_channel: u16,
180 route_epoch: u32,
181 session_id: String,
182 },
183 ReliableCompletionRetained {
184 task_id: String,
185 session_id: String,
186 },
187 ReliableCompletionReplayed {
188 route_channel: u16,
189 route_epoch: u32,
190 task_id: String,
191 session_id: String,
192 },
193}
194
195#[doc(hidden)]
197#[derive(Clone)]
198pub struct SubcTestLifecycleProbe {
199 events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
200}
201
202impl SubcTestLifecycleProbe {
203 #[doc(hidden)]
204 pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
205 Self { events_tx }
206 }
207
208 fn attach_decision(&self, attempt: u32, will_retry: bool) {
209 let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
210 attempt,
211 will_retry,
212 });
213 }
214
215 fn route_detached(&self, route: RouteChannel, session_id: &str) {
216 let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
217 route_channel: route.channel,
218 route_epoch: route.epoch,
219 session_id: session_id.to_string(),
220 });
221 }
222
223 fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
224 let _ = self
225 .events_tx
226 .send(SubcLifecycleEvent::ReliableCompletionRetained {
227 task_id: task_id.to_string(),
228 session_id: session_id.to_string(),
229 });
230 }
231
232 fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
233 let _ = self
234 .events_tx
235 .send(SubcLifecycleEvent::ReliableCompletionReplayed {
236 route_channel: route.channel,
237 route_epoch: route.epoch,
238 task_id: task_id.to_string(),
239 session_id: session_id.to_string(),
240 });
241 }
242}
243
244pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
248 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
249}
250use self::wire::{
251 build_error_frame, build_goodbye_frame, build_tool_response_frame,
252 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
253 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
254 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
255};
256
257struct DecodedFrame {
258 frame: Frame,
259 phase_trace: PhaseTrace,
260}
261
262struct ToolCallCompletion {
263 text: String,
264 phase_trace: PhaseTrace,
265}
266
267#[derive(Clone, Copy, Debug, PartialEq, Eq)]
268enum RouteDetachPolicy {
269 RetainForReplay,
270 CancelOnDetach,
271}
272
273#[derive(Clone)]
274struct ActiveToolCall {
275 root_id: ProjectRootId,
276 cancellation: JobCancellation,
277 detach_policy: RouteDetachPolicy,
278}
279
280type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
281
282struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
283
284impl PendingDeferredSetupGuard {
285 fn new(count: Arc<AtomicUsize>) -> Self {
286 count.fetch_add(1, Ordering::SeqCst);
287 Self(count)
288 }
289}
290
291impl Drop for PendingDeferredSetupGuard {
292 fn drop(&mut self) {
293 self.0.fetch_sub(1, Ordering::SeqCst);
294 }
295}
296
297enum DeferredSetupOutcome {
298 Immediate {
299 text: String,
300 phase_trace: PhaseTrace,
301 },
302 Deferred {
303 pending: PendingResponse,
304 surface_downgraded: bool,
305 phase_trace: PhaseTrace,
306 },
307}
308
309struct PendingSubcResponse {
310 route: RouteChannel,
311 corr: u64,
312 flags: Flags,
313 ver: u8,
314 root: ProjectRootId,
315 session_id: String,
316 bare_name: String,
317 format_context: crate::subc_format::FormatContext,
318 bind_trust: BindTrust,
319 pending: PendingResponse,
320 surface_downgraded: bool,
321 phase_trace: PhaseTrace,
322}
323
324struct ResolvedSubcResponse {
325 entry: PendingSubcResponse,
326 response: Response,
327}
328
329#[derive(Default)]
330struct PendingSubcResponses {
331 entries: Vec<PendingSubcResponse>,
332}
333
334impl PendingSubcResponses {
335 fn register(&mut self, pending: PendingSubcResponse) {
336 self.entries.retain(|entry| {
337 let keep = entry.route != pending.route || entry.corr != pending.corr;
338 if !keep {
339 if let Some(cancellation) = &entry.pending.cancellation {
340 cancellation.request_cancel();
341 }
342 }
343 keep
344 });
345 self.entries.push(pending);
346 }
347
348 fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
349 let mut ready = Vec::new();
350 let mut waiting = Vec::with_capacity(self.entries.len());
351 for mut entry in self.entries.drain(..) {
352 let response = executor
353 .actor_context(&entry.root)
354 .and_then(|ctx| (entry.pending.poll)(&ctx));
355 if let Some(response) = response {
356 ready.push(ResolvedSubcResponse { entry, response });
357 } else {
358 waiting.push(entry);
359 }
360 }
361 self.entries = waiting;
362 ready
363 }
364
365 fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
366 let mut cancelled = false;
367 self.entries.retain(|entry| {
368 let keep = entry.route != route || entry.corr != corr;
369 if !keep {
370 cancelled = true;
371 if let Some(cancellation) = &entry.pending.cancellation {
372 cancellation.request_cancel();
373 }
374 }
375 keep
376 });
377 cancelled
378 }
379
380 fn drain_route(
381 &mut self,
382 route: RouteChannel,
383 executor: &Executor,
384 ) -> Vec<ResolvedSubcResponse> {
385 self.drain_matching(executor, |entry| entry.route == route)
386 }
387
388 fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
389 self.drain_matching(executor, |_| true)
390 }
391
392 fn drain_matching(
393 &mut self,
394 executor: &Executor,
395 matches: impl Fn(&PendingSubcResponse) -> bool,
396 ) -> Vec<ResolvedSubcResponse> {
397 let mut resolved = Vec::new();
398 let mut waiting = Vec::with_capacity(self.entries.len());
399 for mut entry in self.entries.drain(..) {
400 if !matches(&entry) {
401 waiting.push(entry);
402 continue;
403 }
404 if let Some(cancellation) = &entry.pending.cancellation {
405 cancellation.request_cancel();
406 }
407 if let Some(ctx) = executor.actor_context(&entry.root) {
408 if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
409 let response = on_shutdown(&ctx);
410 resolved.push(ResolvedSubcResponse { entry, response });
411 }
412 }
413 }
414 self.entries = waiting;
415 resolved
416 }
417
418 fn is_empty(&self) -> bool {
419 self.entries.is_empty()
420 }
421}
422
423#[derive(Clone)]
424struct PushSenders {
425 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
426 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
427 lossy_overflow: Arc<push::LossyOverflow>,
428 lossy_seq: Arc<AtomicU64>,
429 fleet_status_client: FleetStatusClient,
430}
431
432#[derive(Clone)]
433struct PersistentCancelSignal {
434 inner: Arc<PersistentCancelInner>,
435}
436
437struct PersistentCancelInner {
438 cancelled: AtomicBool,
439 notify: Notify,
440}
441
442impl PersistentCancelSignal {
443 fn new() -> Self {
444 Self {
445 inner: Arc::new(PersistentCancelInner {
446 cancelled: AtomicBool::new(false),
447 notify: Notify::new(),
448 }),
449 }
450 }
451
452 fn cancel(&self) {
453 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
454 self.inner.notify.notify_waiters();
455 }
456 }
457
458 fn is_cancelled(&self) -> bool {
459 self.inner.cancelled.load(Ordering::SeqCst)
460 }
461
462 async fn cancelled(&self) {
463 loop {
471 let notified = self.inner.notify.notified();
472 tokio::pin!(notified);
473 notified.as_mut().enable();
474 if self.is_cancelled() {
475 return;
476 }
477 notified.await;
478 }
479 }
480}
481
482fn submit_active_tool_call(
483 executor: &Executor,
484 active: &ActiveToolCalls,
485 route: RouteChannel,
486 corr: u64,
487 root_id: ProjectRootId,
488 lane: Lane,
489 request_id: String,
490 detach_policy: RouteDetachPolicy,
491 job: crate::executor::ExecutorJob,
492) -> oneshot::Receiver<Response> {
493 let (rx, cancellation) =
494 executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
495 active
496 .lock()
497 .unwrap_or_else(std::sync::PoisonError::into_inner)
498 .insert(
499 (route, corr),
500 ActiveToolCall {
501 root_id,
502 cancellation,
503 detach_policy,
504 },
505 );
506 rx
507}
508
509fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
510 active
511 .lock()
512 .unwrap_or_else(std::sync::PoisonError::into_inner)
513 .remove(&(route, corr))
514 .is_some()
515}
516
517fn active_tool_call_is_registered(
518 active: &ActiveToolCalls,
519 route: RouteChannel,
520 corr: u64,
521) -> bool {
522 active
523 .lock()
524 .unwrap_or_else(std::sync::PoisonError::into_inner)
525 .contains_key(&(route, corr))
526}
527
528fn cancel_active_tool_call(
529 active: &ActiveToolCalls,
530 executor: &Executor,
531 route: RouteChannel,
532 corr: u64,
533 reason: &str,
534) -> bool {
535 let call = active
536 .lock()
537 .unwrap_or_else(std::sync::PoisonError::into_inner)
538 .remove(&(route, corr));
539 let Some(call) = call else {
540 return false;
541 };
542 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
543 log::debug!(
544 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
545 );
546 true
547}
548
549#[derive(Clone, Copy, Debug, PartialEq, Eq)]
550enum RouteWorkDisposition {
551 RetainForReplay,
552 RetainStartedForReplay,
553 Abandon,
554}
555
556fn apply_route_work_disposition(
557 active: &ActiveToolCalls,
558 executor: &Executor,
559 route: RouteChannel,
560 disposition: RouteWorkDisposition,
561 reason: &str,
562) -> usize {
563 if disposition != RouteWorkDisposition::Abandon {
564 let route_calls = active
565 .lock()
566 .unwrap_or_else(std::sync::PoisonError::into_inner)
567 .iter()
568 .filter(|((call_route, _), _)| *call_route == route)
569 .map(|(key, call)| (*key, call.clone()))
570 .collect::<Vec<_>>();
571 let mut retained = 0usize;
572 let mut cancelled_before_execution = 0usize;
573 let mut cancelled_terminal = 0usize;
574
575 for (key, call) in route_calls {
576 let remove = match (call.detach_policy, disposition) {
577 (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
578 retained += 1;
579 false
580 }
581 (
582 RouteDetachPolicy::RetainForReplay,
583 RouteWorkDisposition::RetainStartedForReplay,
584 ) => {
585 match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
586 PreExecutionCancelOutcome::AlreadyStarted => {
587 retained += 1;
588 false
589 }
590 PreExecutionCancelOutcome::QueuedRemoved
591 | PreExecutionCancelOutcome::DispatchedCancelled => {
592 cancelled_before_execution += 1;
593 true
594 }
595 }
596 }
597 (RouteDetachPolicy::CancelOnDetach, _) => {
598 executor.cancel_job(&call.root_id, &call.cancellation);
599 cancelled_terminal += 1;
600 true
601 }
602 (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
603 };
604 if remove {
605 active
606 .lock()
607 .unwrap_or_else(std::sync::PoisonError::into_inner)
608 .remove(&key);
609 }
610 }
611 log::debug!(
612 "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
613 );
614 return retained;
615 }
616
617 let cancelled = {
618 let mut calls = active
619 .lock()
620 .unwrap_or_else(std::sync::PoisonError::into_inner);
621 let mut cancelled = Vec::new();
622 calls.retain(|(call_route, _), call| {
623 if *call_route == route {
624 cancelled.push(call.clone());
625 false
626 } else {
627 true
628 }
629 });
630 cancelled
631 };
632 for call in &cancelled {
633 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
634 log::debug!(
635 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
636 );
637 }
638 cancelled.len()
639}
640
641fn cancel_all_active_tool_calls(
642 active: &ActiveToolCalls,
643 executor: &Executor,
644 reason: &str,
645) -> usize {
646 let cancelled = {
647 let mut calls = active
648 .lock()
649 .unwrap_or_else(std::sync::PoisonError::into_inner);
650 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
651 };
652 for call in &cancelled {
653 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
654 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
655 }
656 cancelled.len()
657}
658
659#[derive(Debug, Clone, Copy, PartialEq, Eq)]
660pub enum BindTrust {
661 FirstParty,
662 Untrusted,
663}
664
665impl BindTrust {
666 fn allows_bash_observation(self) -> bool {
667 matches!(self, Self::FirstParty)
668 }
669
670 fn label(self) -> &'static str {
671 match self {
672 Self::FirstParty => "first_party",
673 Self::Untrusted => "untrusted",
674 }
675 }
676
677 fn sandbox_trust(self) -> PrincipalTrust {
678 match self {
679 Self::FirstParty => PrincipalTrust::FirstParty,
680 Self::Untrusted => PrincipalTrust::Untrusted,
681 }
682 }
683}
684
685pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
686 match principal {
687 Some(Principal::Direct) => BindTrust::FirstParty,
688 Some(Principal::Reserved { module_id })
699 if module_id == "llm-runner"
700 || module_id == "aft"
701 || module_id == "broca"
702 || module_id == "alfonso-core"
703 || module_id == "prefrontal"
704 || module_id == "prefrontal-core" =>
705 {
706 BindTrust::FirstParty
707 }
708 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
709 BindTrust::Untrusted
710 }
711 }
712}
713
714fn harness_forces_untrusted(harness: &str) -> bool {
715 harness.starts_with("fed:")
716}
717
718pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
719 if harness_forces_untrusted(harness) {
720 BindTrust::Untrusted
721 } else {
722 trust_for_principal(principal)
723 }
724}
725
726fn principal_id(principal: &Option<Principal>) -> Option<String> {
727 match principal {
728 Some(Principal::Direct) => Some("direct".to_string()),
729 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
730 Some(Principal::Unverified) => Some("unverified".to_string()),
731 None => None,
732 }
733}
734
735fn principal_label(principal: &Option<Principal>) -> String {
736 principal_id(principal).unwrap_or_else(|| "absent".to_string())
737}
738
739#[derive(Debug)]
740struct RootMeta {
746 maintenance_pending: bool,
747 maintenance_jobs_in_flight: usize,
748 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
749 maintenance_last_submitted: Option<Instant>,
750 maintenance_poisoned: bool,
751 last_touched: Instant,
752 diagnostics_on_edit: bool,
753 active_bash_waits: usize,
754 idle_artifacts_evicted: bool,
755 unbound_quiesced: bool,
756 consecutive_missing_sweeps: u8,
757}
758
759#[derive(Debug)]
760struct PendingBind {
761 bind_root_id: ProjectRootId,
762 inserted_new_actor: bool,
763 cancelled: bool,
764 configure_request_id: String,
765 started_at: Instant,
766 warned_half_deadline: bool,
767 deadline_reported: bool,
768 corr: u64,
769 ver: u8,
770 flags: Flags,
771 cancellation: crate::executor::JobCancellation,
776}
777
778struct RouteBindCompletion {
779 route: RouteChannel,
780 identity: RouteIdentity,
781 bind_root_id: ProjectRootId,
782 inserted_new_actor: bool,
783 configure_response: Response,
784 diagnostics_on_edit: bool,
785 ver: u8,
786 corr: u64,
787 flags: Flags,
788}
789
790#[derive(Debug, Clone)]
791struct RouteIdentity(Arc<RouteIdentityData>);
792
793#[derive(Debug)]
794struct RouteIdentityData {
795 root: ProjectRootId,
796 project_root: PathBuf,
797 harness: String,
798 session: String,
799 trust: BindTrust,
800 spawn_principal: AuthenticatedPrincipal,
801 consumer_elicitation_capable: bool,
802}
803
804impl Deref for RouteIdentity {
805 type Target = RouteIdentityData;
806
807 fn deref(&self) -> &Self::Target {
808 &self.0
809 }
810}
811
812#[derive(Debug, Clone)]
813struct RetainedSessionIdentity {
814 harness: String,
815 trust: BindTrust,
816}
817
818#[derive(Clone)]
819struct BgSub {
820 corr: u64,
821 ver: u8,
822 flags: Flags,
823 root: ProjectRootId,
824 session: String,
825}
826
827#[derive(Clone, Copy, Debug)]
828struct BgWakeState {
829 next_nudge_at: Instant,
830 nudges_sent: u32,
831}
832
833impl BgWakeState {
834 fn armed(now: Instant) -> Self {
835 Self {
836 next_nudge_at: now,
837 nudges_sent: 0,
838 }
839 }
840}
841
842type BgWakePending = HashMap<RouteChannel, BgWakeState>;
843
844type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
847
848struct MaintenanceCompletion {
849 root_id: ProjectRootId,
850 kind: MaintenanceDrainKind,
851 response: Response,
852 empty_bg_sessions: Vec<(String, u64)>,
853 unacked_bg_keys: Option<HashSet<String>>,
854 requeue_kind: Option<MaintenanceDrainKind>,
855}
856
857#[derive(Clone, Copy, Debug, PartialEq, Eq)]
858enum MaintenanceDrainKind {
859 Watcher,
860 Lsp,
861 ConfigureTail,
862 CompletionDrains,
863}
864
865impl MaintenanceDrainKind {
866 fn label(self) -> &'static str {
867 match self {
868 Self::Watcher => "watcher",
869 Self::Lsp => "lsp",
870 Self::ConfigureTail => "configure-tail",
871 Self::CompletionDrains => "completion-drains",
872 }
873 }
874}
875
876#[derive(Debug, Default)]
877struct MaintenanceJobOutcome {
878 empty_bg_sessions: Vec<(String, u64)>,
879 unacked_bg_keys: Option<HashSet<String>>,
880 requeue_kind: Option<MaintenanceDrainKind>,
881}
882
883#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
884struct ReverseCorrKey {
885 route: RouteChannel,
886 corr: u64,
887}
888
889struct PendingBashAsk {
890 route: RouteChannel,
891 tool_corr: u64,
892 tool_flags: Flags,
893 tool_ver: u8,
894 root: ProjectRootId,
895 project_root: PathBuf,
896 session_id: String,
897 spawn_principal: AuthenticatedPrincipal,
898 edit_slot_survives: Option<bool>,
899 request_id: String,
900 arguments: Value,
901 format_context: crate::subc_format::FormatContext,
902 cancel: bash::BashWaitCancel,
903 grants: Vec<String>,
904 expires_at: Instant,
905}
906
907impl RootMeta {
908 fn new(now: Instant) -> Self {
909 Self {
910 maintenance_pending: false,
911 maintenance_jobs_in_flight: 0,
912 maintenance_queued_kinds: VecDeque::new(),
913 maintenance_last_submitted: None,
914 maintenance_poisoned: false,
915 last_touched: now,
916 diagnostics_on_edit: false,
917 active_bash_waits: 0,
918 idle_artifacts_evicted: false,
919 unbound_quiesced: false,
920 consecutive_missing_sweeps: 0,
921 }
922 }
923
924 fn note_activity(&mut self) {
925 self.last_touched = Instant::now();
926 }
927
928 fn reactivate_bound(&mut self) {
929 self.note_activity();
930 self.idle_artifacts_evicted = false;
931 self.unbound_quiesced = false;
932 }
933}
934
935fn due_maintenance_jobs(
936 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
937 executor: Option<&Executor>,
938 bg_sub_by_session: &BgSubsBySession,
939 bg_wake_pending: &BgWakePending,
940 budget: usize,
941 pending_bind_roots: &HashSet<ProjectRootId>,
942) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
943 let mut jobs = Vec::new();
944 let mut deferred = false;
945 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
946 roots.sort_by(|left, right| {
947 let left_last = live_roots
948 .get(left)
949 .and_then(|meta| meta.maintenance_last_submitted);
950 let right_last = live_roots
951 .get(right)
952 .and_then(|meta| meta.maintenance_last_submitted);
953 left_last
954 .cmp(&right_last)
955 .then_with(|| left.as_path().cmp(right.as_path()))
956 });
957
958 for root_id in roots {
959 let Some(meta) = live_roots.get_mut(&root_id) else {
960 continue;
961 };
962 if meta.maintenance_poisoned {
963 continue;
964 }
965
966 if pending_bind_roots.contains(&root_id) {
967 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
968 deferred = true;
969 }
970 continue;
971 }
972
973 if !meta.maintenance_pending {
974 if jobs.len() >= budget {
975 deferred = true;
976 continue;
977 }
978 let executor_actor_context =
982 executor.and_then(|executor| executor.actor_context(&root_id));
983 let root_has_pending_bg_wake =
984 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
985 sub_root == &root_id
986 && channels
987 .iter()
988 .any(|channel| bg_wake_pending.contains_key(channel))
989 });
990 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
991 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
992 .into_iter()
993 .filter(|kind| {
994 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
995 return false;
996 }
997 match kind {
998 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
999 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
1000 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1001 MaintenanceDrainKind::CompletionDrains => {
1007 root_has_pending_bg_wake || ctx.completion_drains_have_work()
1008 }
1009 }
1010 })
1011 .collect(),
1012 None if meta.unbound_quiesced => Vec::new(),
1013 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1015 };
1016 if kinds_with_work.is_empty() {
1017 continue;
1018 }
1019 meta.maintenance_pending = true;
1020 meta.maintenance_queued_kinds.extend(kinds_with_work);
1021 }
1022
1023 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1024 if jobs.len() >= budget {
1025 meta.maintenance_queued_kinds.push_front(kind);
1026 deferred = true;
1027 break;
1028 }
1029 meta.maintenance_jobs_in_flight += 1;
1030 meta.maintenance_last_submitted = Some(Instant::now());
1031 jobs.push((root_id.clone(), kind));
1032 }
1033
1034 meta.maintenance_pending =
1035 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1036 }
1037
1038 (jobs, deferred)
1039}
1040
1041fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1042 match estimate.estimated_bytes {
1043 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1044 None if estimate.status == "busy" => "busy".to_string(),
1045 None => "not estimated".to_string(),
1046 }
1047}
1048
1049fn optional_memory_label(bytes: Option<u64>) -> String {
1050 bytes.map_or_else(
1051 || "not estimated".to_string(),
1052 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1053 )
1054}
1055
1056fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1057 format!(
1058 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1059 optional_memory_label(relief.rss_before_bytes),
1060 optional_memory_label(relief.rss_after_bytes),
1061 optional_memory_label(relief.allocator_before.bytes_in_use),
1062 optional_memory_label(relief.allocator_after.bytes_in_use),
1063 optional_memory_label(relief.allocator_before.size_allocated),
1064 optional_memory_label(relief.allocator_after.size_allocated),
1065 optional_memory_label(relief.allocator_before.retained_slack_bytes),
1066 optional_memory_label(relief.allocator_after.retained_slack_bytes),
1067 relief.bytes_released as f64 / (1024.0 * 1024.0),
1068 )
1069}
1070
1071fn idle_root_eviction_message(
1072 root_id: &ProjectRootId,
1073 memory: &crate::memory::RootMemorySnapshot,
1074 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1075) -> String {
1076 let freed_bytes = [
1079 &memory.semantic,
1080 &memory.trigram,
1081 &memory.symbols,
1082 &memory.callgraph,
1083 &memory.inspect,
1084 ]
1085 .iter()
1086 .filter_map(|estimate| estimate.estimated_bytes)
1087 .fold(0u64, u64::saturating_add);
1088 let mut message = format!(
1089 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1090 root_id.as_path().display(),
1091 freed_bytes as f64 / (1024.0 * 1024.0),
1092 eviction_estimate_label(&memory.semantic),
1093 eviction_estimate_label(&memory.trigram),
1094 eviction_estimate_label(&memory.symbols),
1095 eviction_estimate_label(&memory.callgraph),
1096 eviction_estimate_label(&memory.inspect),
1097 eviction_estimate_label(&memory.bash),
1098 eviction_estimate_label(&memory.lsp),
1099 eviction_estimate_label(&memory.parser_pool),
1100 );
1101 if let Some(pressure_relief) = pressure_relief {
1102 message.push_str(&pressure_relief_label(pressure_relief));
1103 }
1104 message
1105}
1106
1107fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1108 executor
1109 .actor_context(root_id)
1110 .map(|ctx| ctx.config().idle.root_ttl())
1111 .unwrap_or(IDLE_ROOT_TTL)
1112}
1113
1114fn process_has_been_idle(
1115 now: Instant,
1116 live_roots: &HashMap<ProjectRootId, RootMeta>,
1117 executor: &Executor,
1118) -> bool {
1119 !live_roots.is_empty()
1120 && live_roots.iter().all(|(root_id, meta)| {
1121 now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1122 && meta.active_bash_waits == 0
1123 && !meta.maintenance_pending
1124 && meta.maintenance_queued_kinds.is_empty()
1125 })
1126}
1127
1128fn allocator_pressure_relief_after_idle_sweep(
1129 now: Instant,
1130 live_roots: &HashMap<ProjectRootId, RootMeta>,
1131 executor: &Executor,
1132) -> Option<crate::memory::AllocatorPressureRelief> {
1133 if !process_has_been_idle(now, live_roots, executor)
1134 || live_roots.keys().any(|root_id| {
1135 executor
1136 .actor_context(root_id)
1137 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1138 })
1139 {
1140 return None;
1141 }
1142
1143 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1144 {
1145 Some(crate::memory::relieve_allocator_pressure())
1146 }
1147 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1148 {
1149 None
1150 }
1151}
1152
1153fn quiesce_unbound_root(
1154 root_id: &ProjectRootId,
1155 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1156 executor: &Arc<Executor>,
1157) {
1158 let Some(meta) = live_roots.get_mut(root_id) else {
1159 return;
1160 };
1161
1162 let ctx = executor.actor_context(root_id);
1163 if let Some(ctx) = ctx.as_ref() {
1164 ctx.mark_subc_unbound();
1168 ctx.bash_background()
1169 .replace_live_delivery_sessions(HashSet::new());
1170 }
1171 let cancelled = executor.cancel_queued_maintenance(root_id);
1172 let discarded = ctx
1180 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1181 .unwrap_or(0);
1182 meta.unbound_quiesced = true;
1183 meta.maintenance_queued_kinds.clear();
1184 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1185 log::info!(
1186 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1187 root_id.as_path().display(),
1188 cancelled,
1189 discarded
1190 );
1191}
1192
1193#[allow(clippy::too_many_arguments)]
1194fn quiesce_connection_roots(
1195 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1196 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1197 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1198 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1199 installed_route_epochs: &mut HashMap<u16, u32>,
1200 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1201 active_tool_calls: &ActiveToolCalls,
1202 executor: &Arc<Executor>,
1203) {
1204 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1205 for cancel in route_bash_cancels.values() {
1206 cancel.token.cancel();
1207 }
1208 route_bash_cancels.clear();
1209
1210 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1211 for pending in pending_binds.values_mut() {
1212 pending.cancelled = true;
1213 roots.insert(pending.bind_root_id.clone());
1214 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1215 }
1216
1217 for root_id in roots {
1221 if live_roots.contains_key(&root_id) {
1222 quiesce_unbound_root(&root_id, live_roots, executor);
1223 } else if let Some(ctx) = executor.actor_context(&root_id) {
1224 ctx.mark_subc_unbound();
1225 executor.cancel_queued_maintenance(&root_id);
1226 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1227 }
1228 }
1229
1230 routes.clear();
1231 root_channels.clear();
1232 installed_route_epochs.clear();
1233}
1234
1235#[derive(Debug, Default)]
1239struct ReclaimedRoutes {
1240 highest_epoch_by_channel: HashMap<u16, u32>,
1241}
1242
1243impl ReclaimedRoutes {
1244 fn insert(&mut self, route: RouteChannel) {
1245 self.highest_epoch_by_channel
1246 .entry(route.channel)
1247 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1248 .or_insert(route.epoch);
1249 }
1250
1251 fn contains(&self, route: RouteChannel) -> bool {
1252 self.highest_epoch_by_channel
1253 .get(&route.channel)
1254 .is_some_and(|epoch| route.epoch <= *epoch)
1255 }
1256}
1257
1258#[derive(Debug, Default)]
1259struct IdleReapOutcome {
1260 evicted: usize,
1261 forgotten_deleted_roots: Vec<ProjectRootId>,
1262}
1263
1264fn reap_idle_roots(
1265 now: Instant,
1266 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1267 pending_binds: &HashMap<RouteChannel, PendingBind>,
1268 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1269 executor: &Arc<Executor>,
1270 metrics: &DispatchPathMetrics,
1271) -> IdleReapOutcome {
1272 let pending_bind_roots = pending_binds
1273 .values()
1274 .map(|pending| pending.bind_root_id.clone())
1275 .collect::<HashSet<_>>();
1276 let mut census = ReapBlockerCensus::default();
1277 let mut candidates = Vec::new();
1278
1279 for (root_id, meta) in live_roots.iter_mut() {
1280 let deleted = !root_id.as_path().exists();
1281 if deleted {
1282 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1292 } else {
1293 meta.consecutive_missing_sweeps = 0;
1294 }
1295 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1296 let has_bound_route = root_channels
1297 .get(root_id)
1298 .is_some_and(|channels| !channels.is_empty());
1299 let has_pending_bind = pending_bind_roots.contains(root_id);
1300
1301 if deleted {
1302 let mut retained = false;
1303 if !deletion_confirmed {
1304 census.absence_unconfirmed += 1;
1305 retained = true;
1306 }
1307 if meta.active_bash_waits > 0 {
1311 census.bash_waits += 1;
1312 retained = true;
1313 }
1314 if meta.maintenance_pending {
1315 census.maintenance_pending += 1;
1316 retained = true;
1317 }
1318 if !meta.maintenance_queued_kinds.is_empty() {
1319 census.maintenance_queued += 1;
1320 retained = true;
1321 }
1322 if has_pending_bind {
1323 census.pending_binds += 1;
1324 retained = true;
1325 }
1326 match executor.try_actor_is_idle(root_id) {
1327 Some(true) => {}
1328 Some(false) => {
1329 census.actor_busy += 1;
1330 retained = true;
1331 }
1332 None => {
1333 census.actor_state_busy += 1;
1334 retained = true;
1335 }
1336 }
1337 if retained {
1338 census.deleted_retained += 1;
1339 continue;
1340 }
1341 } else {
1342 if has_bound_route
1346 || !meta.unbound_quiesced
1347 || meta.idle_artifacts_evicted
1348 || now.saturating_duration_since(meta.last_touched)
1349 < root_idle_ttl(executor, root_id)
1350 || meta.active_bash_waits > 0
1351 || meta.maintenance_pending
1352 || !meta.maintenance_queued_kinds.is_empty()
1353 || has_pending_bind
1354 || !executor.actor_is_idle(root_id)
1355 {
1356 continue;
1357 }
1358 }
1359 candidates.push((root_id.clone(), deleted));
1360 }
1361
1362 let mut reaped = Vec::new();
1363 let mut forgotten_deleted_roots = Vec::new();
1364 for (root_id, deleted) in candidates {
1365 let Some(ctx) = executor.actor_context(&root_id) else {
1366 if deleted {
1367 census.deleted_retained += 1;
1368 census.actor_busy += 1;
1369 }
1370 continue;
1371 };
1372 if deleted {
1383 ctx.bash_background()
1384 .kill_running_tasks_for_root(root_id.as_path());
1385 }
1386 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1387 if ctx.artifact_eviction_blocked() {
1388 if let Some(pending) = taken_pending {
1389 ctx.restore_pending_reconciliation_state(pending);
1390 }
1391 if deleted {
1392 census.deleted_retained += 1;
1393 census.artifact_eviction_blocked += 1;
1394 }
1395 continue;
1396 }
1397 let memory_before = ctx.memory_root_snapshot();
1398 if !ctx.evict_idle_artifacts() {
1399 if let Some(pending) = taken_pending {
1400 ctx.restore_pending_reconciliation_state(pending);
1401 }
1402 if deleted {
1403 census.deleted_retained += 1;
1404 census.artifact_eviction_failed += 1;
1405 }
1406 continue;
1407 }
1408 drop(taken_pending);
1409 ctx.stop_watcher_runtime_in_background();
1410 ctx.invalidate_artifacts_after_watcher_gap();
1413
1414 if deleted {
1415 if executor.retire_idle_actor_in_background(&root_id) {
1416 live_roots.remove(&root_id);
1417 forgotten_deleted_roots.push(root_id.clone());
1418 } else {
1419 census.deleted_retained += 1;
1420 census.actor_busy += 1;
1421 }
1422 } else {
1423 if let Some(meta) = live_roots.get_mut(&root_id) {
1424 meta.idle_artifacts_evicted = true;
1425 }
1426 ctx.release_idle_reopenable_resources_in_background();
1427 }
1428 reaped.push((root_id, memory_before));
1429 }
1430
1431 let census_changed = metrics.record_reap(census);
1434 if census_changed {
1435 log::info!(
1436 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1437 census.deleted_retained,
1438 census.blocker_histogram()
1439 );
1440 }
1441
1442 let pressure_relief = (!reaped.is_empty())
1443 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1444 .flatten();
1445 for (root_id, memory_before) in &reaped {
1446 log::info!(
1447 "{}",
1448 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1449 );
1450 }
1451 IdleReapOutcome {
1452 evicted: reaped.len(),
1453 forgotten_deleted_roots,
1454 }
1455}
1456
1457fn reap_idle_lsp_servers(
1465 now: Instant,
1466 live_roots: &HashMap<ProjectRootId, RootMeta>,
1467 executor: &Executor,
1468) {
1469 for (root_id, meta) in live_roots {
1470 let Some(ctx) = executor.actor_context(root_id) else {
1471 continue;
1472 };
1473 crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1474 }
1475}
1476
1477#[allow(clippy::too_many_arguments)]
1478fn purge_deleted_root_residents(
1479 root_id: &ProjectRootId,
1480 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1481 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1482 installed_route_epochs: &mut HashMap<u16, u32>,
1483 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1484 active_tool_calls: &ActiveToolCalls,
1485 executor: &Executor,
1486 retry_buffer: &mut RetryBuffer,
1487 reclaimed_routes: &mut ReclaimedRoutes,
1488 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1489 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1490 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1491 bg_sub_by_session: &mut BgSubsBySession,
1492 bg_wake_pending: &mut BgWakePending,
1493 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1494 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1495 metrics: &DispatchPathMetrics,
1496) {
1497 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1498 stale_routes.extend(
1499 routes
1500 .iter()
1501 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1502 );
1503 stale_routes.extend(
1504 bg_sub_by_session
1505 .iter()
1506 .filter(|((root, _), _)| root == root_id)
1507 .flat_map(|(_, routes)| routes.iter().copied()),
1508 );
1509 stale_routes.extend(
1510 pending_bash_asks
1511 .values()
1512 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1513 );
1514
1515 for route in stale_routes {
1516 reclaimed_routes.insert(route);
1517 remove_installed_route(installed_route_epochs, route);
1518 remove_route_channel(routes, root_channels, route);
1519 if let Some(cancel) = route_bash_cancels.remove(&route) {
1520 cancel.token.cancel();
1521 }
1522 apply_route_work_disposition(
1523 active_tool_calls,
1524 executor,
1525 route,
1526 RouteWorkDisposition::Abandon,
1527 "root reclaim",
1528 );
1529 retry_buffer.remove(&route);
1530 if let Some(sub) = bg_subs.remove(&route) {
1531 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1532 }
1533 bg_wake_pending.remove(&route);
1534 }
1535 root_channels.remove(root_id);
1536 session_identity.retain(|(root, _), _| root != root_id);
1537 push_buffer.retain(|key, _| &key.root != root_id);
1538 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1539 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1540 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1541 sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1542
1543 log::info!(
1544 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1545 root_id.as_path().display()
1546 );
1547}
1548
1549#[allow(clippy::too_many_arguments)]
1550fn submit_due_maintenance_jobs(
1551 executor: &Arc<Executor>,
1552 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1553 pending_binds: &HashMap<RouteChannel, PendingBind>,
1554 bg_sub_by_session: &BgSubsBySession,
1555 bg_wake_pending: &BgWakePending,
1556 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1557 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1558 metrics: &Arc<DispatchPathMetrics>,
1559) {
1560 let pending_bind_roots = pending_binds
1561 .values()
1562 .map(|pending| pending.bind_root_id.clone())
1563 .collect::<HashSet<_>>();
1564 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1565 live_roots,
1566 Some(executor),
1567 bg_sub_by_session,
1568 bg_wake_pending,
1569 MAINTENANCE_SUBMIT_BUDGET,
1570 &pending_bind_roots,
1571 );
1572 if deferred_jobs {
1573 metrics
1574 .maintenance_budget_deferrals
1575 .fetch_add(1, Ordering::Relaxed);
1576 }
1577 for (root_id, kind) in due_jobs {
1578 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1579 bg_sub_by_session
1580 .iter()
1581 .filter_map(|((root, session), _)| {
1582 if root == &root_id {
1583 Some((
1584 session.clone(),
1585 bg_wake_epoch
1586 .get(&(root_id.clone(), session.clone()))
1587 .copied()
1588 .unwrap_or(0),
1589 ))
1590 } else {
1591 None
1592 }
1593 })
1594 .collect()
1595 } else {
1596 Vec::new()
1597 };
1598 submit_maintenance_job(
1599 executor,
1600 root_id,
1601 kind,
1602 bg_sessions_to_check,
1603 maintenance_tx,
1604 metrics,
1605 );
1606 }
1607}
1608
1609fn should_requiesce_after_maintenance(
1610 meta: &RootMeta,
1611 completed_kind: MaintenanceDrainKind,
1612 bind_pending: bool,
1613) -> bool {
1614 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1615}
1616
1617fn note_maintenance_completion(
1618 meta: &mut RootMeta,
1619 requeue_kind: Option<MaintenanceDrainKind>,
1620 fatal: bool,
1621 defer_requeue: bool,
1622) {
1623 if fatal {
1624 meta.maintenance_poisoned = true;
1625 }
1626
1627 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1628 meta.maintenance_queued_kinds.push_back(kind);
1629 }
1630
1631 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1632 meta.maintenance_pending =
1633 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1634}
1635
1636fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1637 RouteChannel { channel, epoch }
1638}
1639
1640fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1641 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1642 installed_epochs.remove(&route.channel);
1643 }
1644}
1645
1646fn ingress_route_should_be_processed(
1647 installed_epochs: &HashMap<u16, u32>,
1648 reclaimed_routes: &ReclaimedRoutes,
1649 frame: &Frame,
1650) -> bool {
1651 if frame.header.channel == 0
1652 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1653 {
1654 return true;
1655 }
1656
1657 frame.header.ty == FrameType::Request
1662 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1663}
1664
1665fn bash_elicitation_timeout() -> Duration {
1666 if cfg!(debug_assertions) {
1667 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1668 if let Ok(ms) = raw.parse::<u64>() {
1669 if ms > 0 {
1670 return Duration::from_millis(ms);
1671 }
1672 }
1673 }
1674 }
1675 BASH_ELICITATION_TIMEOUT
1676}
1677
1678fn allocate_reverse_corr(
1679 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1680 route: RouteChannel,
1681 next_corr: &mut u64,
1682) -> u64 {
1683 loop {
1684 let corr = *next_corr;
1685 *next_corr = (*next_corr).wrapping_add(1).max(1);
1686 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1687 return corr;
1688 }
1689 }
1690}
1691
1692fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1693 match kind {
1694 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1695 crate::bash_permissions::PermissionKind::Bash => "bash",
1696 }
1697}
1698
1699fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1700 let mut patterns = Vec::new();
1701 let mut seen = HashSet::new();
1702 for ask in asks {
1703 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1704 if seen.insert(pattern.clone()) {
1705 patterns.push(pattern.clone());
1706 }
1707 }
1708 }
1709 patterns
1710}
1711
1712fn bash_elicitation_message(
1713 command: &str,
1714 asks: &[crate::bash_permissions::PermissionAsk],
1715) -> String {
1716 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1717 let patterns = bash_elicitation_patterns(asks);
1718 let pattern_text = if patterns.is_empty() {
1719 "no matched permission patterns".to_string()
1720 } else {
1721 patterns.join(", ")
1722 };
1723 let ask_kinds = asks
1724 .iter()
1725 .map(|ask| bash_permission_kind_label(&ask.kind))
1726 .collect::<HashSet<_>>()
1727 .into_iter()
1728 .collect::<Vec<_>>()
1729 .join(", ");
1730 if ask_kinds.is_empty() {
1731 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1732 } else {
1733 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1734 }
1735}
1736
1737fn bash_elicitation_request_body(
1738 command: &str,
1739 asks: &[crate::bash_permissions::PermissionAsk],
1740) -> Value {
1741 json!({
1742 "method": BASH_ELICITATION_CREATE_METHOD,
1743 "params": {
1744 "mode": "form",
1745 "message": bash_elicitation_message(command, asks),
1746 "requestedSchema": {
1747 "type": "object",
1748 "properties": {
1749 "decision": {
1750 "type": "string",
1751 "enum": ["allow", "deny"],
1752 "description": "Choose allow to run this bash command once, or deny to block it."
1753 }
1754 },
1755 "required": ["decision"],
1756 "additionalProperties": false
1757 },
1758 "_meta": {
1759 "aft": {
1760 "tool": "bash",
1761 "command": command,
1762 "asks": asks
1763 }
1764 }
1765 }
1766 })
1767}
1768
1769fn build_bash_elicitation_request_frame(
1770 ver: u8,
1771 route: RouteChannel,
1772 corr: u64,
1773 flags: Flags,
1774 command: &str,
1775 asks: &[crate::bash_permissions::PermissionAsk],
1776) -> Result<Frame, SubcError> {
1777 let body = bash_elicitation_request_body(command, asks);
1778 Frame::build_with_version(
1779 ver,
1780 FrameType::Request,
1781 flags,
1782 route.channel,
1783 route.epoch,
1784 corr,
1785 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1786 )
1787 .map_err(SubcError::FrameBuild)
1788}
1789
1790fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1791 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1792 return false;
1793 };
1794 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1795}
1796
1797fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1798 let Some(object) = value.as_object() else {
1799 return false;
1800 };
1801 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1802}
1803
1804fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1805 let Some(object) = value.as_object() else {
1806 return false;
1807 };
1808 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1809 return false;
1810 }
1811 let Some(content) = object.get("content").and_then(Value::as_object) else {
1812 return false;
1813 };
1814 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1815}
1816
1817#[allow(clippy::too_many_arguments)]
1818async fn settle_pending_bash_ask_denied(
1819 tx: &WriterSender,
1820 pending: PendingBashAsk,
1821 routes: &HashMap<RouteChannel, RouteIdentity>,
1822 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1823 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1824 shutdown: &Arc<Notify>,
1825 metrics: &DispatchPathMetrics,
1826) -> Result<(), SubcError> {
1827 let completion = bash::bash_denied_untrusted_completion(
1828 pending.route,
1829 pending.tool_corr,
1830 pending.tool_flags,
1831 pending.tool_ver,
1832 pending.root,
1833 pending.request_id,
1834 pending.format_context,
1835 );
1836 bash::handle_bash_deferred_completion(
1837 tx,
1838 completion,
1839 routes,
1840 live_roots,
1841 route_bash_cancels,
1842 shutdown,
1843 metrics,
1844 )
1845 .await
1846}
1847
1848fn take_pending_bash_asks_for_route(
1849 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1850 route: RouteChannel,
1851) -> Vec<PendingBashAsk> {
1852 let keys = pending_bash_asks
1853 .keys()
1854 .copied()
1855 .filter(|key| key.route == route)
1856 .collect::<Vec<_>>();
1857 keys.into_iter()
1858 .filter_map(|key| pending_bash_asks.remove(&key))
1859 .collect()
1860}
1861
1862#[allow(clippy::too_many_arguments)]
1863async fn settle_pending_bash_asks_for_route(
1864 tx: &WriterSender,
1865 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1866 route: RouteChannel,
1867 routes: &HashMap<RouteChannel, RouteIdentity>,
1868 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1869 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1870 shutdown: &Arc<Notify>,
1871 metrics: &DispatchPathMetrics,
1872) -> Result<(), SubcError> {
1873 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1874 settle_pending_bash_ask_denied(
1875 tx,
1876 pending,
1877 routes,
1878 live_roots,
1879 route_bash_cancels,
1880 shutdown,
1881 metrics,
1882 )
1883 .await?;
1884 }
1885 Ok(())
1886}
1887
1888#[allow(clippy::too_many_arguments)]
1889async fn settle_all_pending_bash_asks(
1890 tx: &WriterSender,
1891 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1892 routes: &HashMap<RouteChannel, RouteIdentity>,
1893 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1894 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1895 shutdown: &Arc<Notify>,
1896 metrics: &DispatchPathMetrics,
1897) -> Result<(), SubcError> {
1898 let pending = pending_bash_asks
1899 .drain()
1900 .map(|(_, pending)| pending)
1901 .collect::<Vec<_>>();
1902 for pending in pending {
1903 settle_pending_bash_ask_denied(
1904 tx,
1905 pending,
1906 routes,
1907 live_roots,
1908 route_bash_cancels,
1909 shutdown,
1910 metrics,
1911 )
1912 .await?;
1913 }
1914 Ok(())
1915}
1916
1917#[allow(clippy::too_many_arguments)]
1918async fn expire_pending_bash_asks(
1919 tx: &WriterSender,
1920 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1921 routes: &HashMap<RouteChannel, RouteIdentity>,
1922 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1923 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1924 shutdown: &Arc<Notify>,
1925 metrics: &DispatchPathMetrics,
1926) -> Result<(), SubcError> {
1927 let now = Instant::now();
1928 let expired = pending_bash_asks
1929 .iter()
1930 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1931 .collect::<Vec<_>>();
1932 for key in expired {
1933 if let Some(pending) = pending_bash_asks.remove(&key) {
1934 log::debug!(
1935 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1936 key.corr,
1937 pending.route
1938 );
1939 settle_pending_bash_ask_denied(
1940 tx,
1941 pending,
1942 routes,
1943 live_roots,
1944 route_bash_cancels,
1945 shutdown,
1946 metrics,
1947 )
1948 .await?;
1949 }
1950 }
1951 Ok(())
1952}
1953
1954#[allow(clippy::too_many_arguments)]
1955async fn handle_bash_elicitation_reply(
1956 tx: &WriterSender,
1957 frame: &Frame,
1958 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1959 routes: &HashMap<RouteChannel, RouteIdentity>,
1960 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1961 executor: &Arc<Executor>,
1962 shutdown: &Arc<Notify>,
1963 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1964 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1965 metrics: &Arc<DispatchPathMetrics>,
1966 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1967 dispatch: DispatchFn,
1968) -> Result<(), SubcError> {
1969 let key = ReverseCorrKey {
1970 route: route_key(frame.header.channel, frame.header.epoch),
1971 corr: frame.header.corr,
1972 };
1973 let Some(pending) = pending_bash_asks.remove(&key) else {
1974 return Ok(());
1975 };
1976
1977 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1978 if routes.contains_key(&key.route) {
1979 bash::submit_deferred_bash(
1980 executor,
1981 bash_deferred_tx,
1982 bash_poll_touch_tx,
1983 metrics,
1984 dispatch,
1985 pending.root,
1986 pending.project_root,
1987 pending.session_id,
1988 pending.request_id,
1989 pending.route,
1990 pending.tool_corr,
1991 pending.tool_flags,
1992 pending.tool_ver,
1993 pending.arguments,
1994 pending.format_context,
1995 pending.cancel,
1996 BindTrust::Untrusted,
1997 pending.spawn_principal,
1998 pending.edit_slot_survives,
1999 Some(pending.grants),
2000 );
2001 return Ok(());
2002 }
2003 log::debug!(
2004 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2005 key.corr,
2006 pending.route
2007 );
2008 }
2009
2010 settle_pending_bash_ask_denied(
2011 tx,
2012 pending,
2013 routes,
2014 live_roots,
2015 route_bash_cancels,
2016 shutdown,
2017 metrics,
2018 )
2019 .await
2020}
2021
2022#[allow(clippy::too_many_arguments)]
2023async fn cancel_pending_bash_ask_for_tool_call(
2024 tx: &WriterSender,
2025 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2026 route: RouteChannel,
2027 tool_corr: u64,
2028 routes: &HashMap<RouteChannel, RouteIdentity>,
2029 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2030 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2031 shutdown: &Arc<Notify>,
2032 metrics: &DispatchPathMetrics,
2033) -> Result<(), SubcError> {
2034 let keys = pending_bash_asks
2035 .iter()
2036 .filter_map(|(key, pending)| {
2037 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2038 })
2039 .collect::<Vec<_>>();
2040 for key in keys {
2041 if let Some(pending) = pending_bash_asks.remove(&key) {
2042 settle_pending_bash_ask_denied(
2043 tx,
2044 pending,
2045 routes,
2046 live_roots,
2047 route_bash_cancels,
2048 shutdown,
2049 metrics,
2050 )
2051 .await?;
2052 }
2053 }
2054 Ok(())
2055}
2056
2057fn remove_root_channel(
2058 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2059 root: &ProjectRootId,
2060 channel: RouteChannel,
2061) {
2062 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2063 channels.remove(&channel);
2064 channels.is_empty()
2065 } else {
2066 false
2067 };
2068 if remove_root {
2069 root_channels.remove(root);
2070 }
2071}
2072
2073fn remove_route_channel(
2074 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2075 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2076 channel: RouteChannel,
2077) -> Option<RouteIdentity> {
2078 let removed = routes.remove(&channel);
2079 if let Some(identity) = &removed {
2080 remove_root_channel(root_channels, &identity.root, channel);
2081 }
2082 removed
2083}
2084
2085fn insert_route_channel(
2086 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2087 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2088 channel: RouteChannel,
2089 identity: RouteIdentity,
2090) {
2091 if let Some(previous) = routes.insert(channel, identity.clone()) {
2092 remove_root_channel(root_channels, &previous.root, channel);
2093 }
2094 root_channels
2095 .entry(identity.root.clone())
2096 .or_default()
2097 .insert(channel);
2098}
2099
2100fn sync_bg_live_delivery_sessions(
2101 executor: &Executor,
2102 routes: &HashMap<RouteChannel, RouteIdentity>,
2103 additional_root: Option<&ProjectRootId>,
2104) {
2105 let sessions = routes
2109 .values()
2110 .filter(|identity| identity.trust.allows_bash_observation())
2111 .map(|identity| identity.session.clone())
2112 .collect::<HashSet<_>>();
2113 let mut roots = routes
2114 .values()
2115 .map(|identity| identity.root.clone())
2116 .collect::<HashSet<_>>();
2117 roots.extend(additional_root.cloned());
2118 for root in roots {
2119 if let Some(ctx) = executor.actor_context(&root) {
2120 ctx.bash_background()
2121 .replace_live_delivery_sessions(sessions.clone());
2122 }
2123 }
2124}
2125
2126fn insert_bg_subscription_index(
2127 bg_sub_by_session: &mut BgSubsBySession,
2128 root: ProjectRootId,
2129 session: String,
2130 channel: RouteChannel,
2131) {
2132 bg_sub_by_session
2133 .entry((root, session))
2134 .or_default()
2135 .insert(channel);
2136}
2137
2138fn remove_bg_subscription_index(
2139 bg_sub_by_session: &mut BgSubsBySession,
2140 channel: RouteChannel,
2141 identity: Option<&RouteIdentity>,
2142) {
2143 if let Some(identity) = identity {
2144 let key = (identity.root.clone(), identity.session.clone());
2145 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2146 channels.remove(&channel);
2147 channels.is_empty()
2148 });
2149 if remove_key {
2150 bg_sub_by_session.remove(&key);
2151 }
2152 } else {
2153 bg_sub_by_session.retain(|_, channels| {
2154 channels.remove(&channel);
2155 !channels.is_empty()
2156 });
2157 }
2158}
2159
2160fn route_removal_will_quiesce_root(
2161 root: &ProjectRootId,
2162 route: RouteChannel,
2163 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2164 has_pending_bind: bool,
2165 replacement_root: Option<&ProjectRootId>,
2166) -> bool {
2167 let removes_last_route = root_channels
2168 .get(root)
2169 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2170 removes_last_route && !has_pending_bind && replacement_root != Some(root)
2171}
2172
2173fn should_quiesce_removed_root(
2174 root: &ProjectRootId,
2175 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2176 has_pending_bind: bool,
2177 replacement_root: Option<&ProjectRootId>,
2178) -> bool {
2179 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2180}
2181
2182async fn end_bg_subscription(
2183 writer_tx: &WriterSender,
2184 metrics: &DispatchPathMetrics,
2185 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2186 bg_sub_by_session: &mut BgSubsBySession,
2187 bg_wake_pending: &mut BgWakePending,
2188 channel: RouteChannel,
2189 identity: Option<&RouteIdentity>,
2190 cause: &str,
2191) -> Result<(), SubcError> {
2192 if let Some(sub) = bg_subs.remove(&channel) {
2193 bg_wake_pending.remove(&channel);
2194 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2195 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2196 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2197 }
2198 Ok(())
2199}
2200
2201#[allow(clippy::too_many_arguments)]
2202async fn teardown_installed_route(
2203 tx: &WriterSender,
2204 metrics: &DispatchPathMetrics,
2205 executor: &Arc<Executor>,
2206 channel: RouteChannel,
2207 cancellation_reason: &str,
2208 replacement_root: Option<&ProjectRootId>,
2209 installed_route_epochs: &mut HashMap<u16, u32>,
2210 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2211 management_routes: &mut HashSet<RouteChannel>,
2212 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2213 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2214 bg_sub_by_session: &mut BgSubsBySession,
2215 bg_wake_pending: &mut BgWakePending,
2216 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2217 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2218 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2219 active_tool_calls: &ActiveToolCalls,
2220 pending_responses: &mut PendingSubcResponses,
2221 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2222 retry_buffer: &mut RetryBuffer,
2223 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2224 shutdown: &Arc<Notify>,
2225 tool_response_body_limit: usize,
2226 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2227) -> Result<(), SubcError> {
2228 remove_installed_route(installed_route_epochs, channel);
2229 management_routes.remove(&channel);
2230 let bg_end_cause = match cancellation_reason {
2231 "Goodbye" => "goodbye",
2232 "higher-epoch RouteBind" => "higher-epoch",
2233 other => other,
2234 };
2235 end_bg_subscription(
2236 tx,
2237 metrics,
2238 bg_subs,
2239 bg_sub_by_session,
2240 bg_wake_pending,
2241 channel,
2242 routes.get(&channel),
2243 bg_end_cause,
2244 )
2245 .await?;
2246 settle_pending_bash_asks_for_route(
2247 tx,
2248 pending_bash_asks,
2249 channel,
2250 routes,
2251 live_roots,
2252 route_bash_cancels,
2253 shutdown,
2254 metrics,
2255 )
2256 .await?;
2257 if let Some(cancel) = route_bash_cancels.remove(&channel) {
2258 cancel.token.cancel();
2259 }
2260 for resolved in pending_responses.drain_route(channel, executor) {
2261 deliver_resolved_subc_response(
2262 tx,
2263 resolved,
2264 routes,
2265 live_roots,
2266 executor.as_ref(),
2267 active_tool_calls,
2268 shutdown,
2269 metrics,
2270 tool_response_body_limit,
2271 )
2272 .await?;
2273 }
2274 let work_disposition = if replacement_root.is_some() {
2279 RouteWorkDisposition::RetainForReplay
2280 } else {
2281 RouteWorkDisposition::RetainStartedForReplay
2282 };
2283 apply_route_work_disposition(
2284 active_tool_calls,
2285 executor,
2286 channel,
2287 work_disposition,
2288 cancellation_reason,
2289 );
2290 if let Some(pending) = pending_binds.get_mut(&channel) {
2291 pending.cancelled = true;
2292 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2293 log::debug!(
2294 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2295 channel.channel
2296 );
2297 }
2298 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2299 if let Some(identity) = routes.get(&channel) {
2300 let has_pending_bind = pending_binds
2301 .values()
2302 .any(|pending| pending.bind_root_id == identity.root);
2303 if route_removal_will_quiesce_root(
2304 &identity.root,
2305 channel,
2306 root_channels,
2307 has_pending_bind,
2308 replacement_root,
2309 ) {
2310 if let Some(ctx) = executor.actor_context(&identity.root) {
2311 ctx.mark_subc_unbound();
2314 }
2315 }
2316 }
2317 delay_route_detach_for_test(lifecycle_probe).await;
2320 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2321 sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2322 if let Some(probe) = lifecycle_probe {
2323 probe.route_detached(channel, &identity.session);
2324 }
2325 let session_still_routed = routes
2326 .values()
2327 .any(|route| route.root == identity.root && route.session == identity.session);
2328 if !session_still_routed {
2329 if let Some(ctx) = executor.actor_context(&identity.root) {
2330 ctx.hashline_bindings()
2331 .teardown(identity.root.as_path(), &identity.session);
2332 }
2333 }
2334 if migrated > 0 {
2335 log::debug!(
2336 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2337 channel.channel
2338 );
2339 }
2340 if let Some(meta) = live_roots.get_mut(&identity.root) {
2341 let idle_for = meta.last_touched.elapsed();
2342 meta.note_activity();
2343 log::debug!(
2344 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2345 channel.channel,
2346 identity.root.as_path().display(),
2347 identity.harness,
2348 identity.session,
2349 idle_for
2350 );
2351 } else {
2352 log::debug!(
2353 "subc attach: route {} torn down for root {} harness {} session {}",
2354 channel.channel,
2355 identity.root.as_path().display(),
2356 identity.harness,
2357 identity.session
2358 );
2359 }
2360 let has_pending_bind = pending_binds
2361 .values()
2362 .any(|pending| pending.bind_root_id == identity.root);
2363 if should_quiesce_removed_root(
2364 &identity.root,
2365 root_channels,
2366 has_pending_bind,
2367 replacement_root,
2368 ) {
2369 quiesce_unbound_root(&identity.root, live_roots, executor);
2370 }
2371 } else {
2372 if migrated > 0 {
2373 log::debug!(
2374 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2375 channel.channel
2376 );
2377 }
2378 log::debug!("subc attach: unbound route {} torn down", channel.channel);
2379 }
2380 Ok(())
2381}
2382
2383async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2384 if lifecycle_probe.is_none() {
2385 return;
2386 }
2387 let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2388 .ok()
2389 .and_then(|raw| raw.parse::<u64>().ok())
2390 else {
2391 return;
2392 };
2393 tokio::time::sleep(Duration::from_millis(delay)).await;
2394}
2395
2396fn remember_session_identity(
2397 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2398 identity: &RouteIdentity,
2399) {
2400 let key = (identity.root.clone(), identity.session.clone());
2401 if matches!(identity.trust, BindTrust::Untrusted)
2402 && session_identity
2403 .get(&key)
2404 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2405 {
2406 return;
2407 }
2408
2409 session_identity.insert(
2414 key,
2415 RetainedSessionIdentity {
2416 harness: identity.harness.clone(),
2417 trust: identity.trust,
2418 },
2419 );
2420}
2421
2422fn replay_key_for_session(
2423 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2424 root: &ProjectRootId,
2425 session: &str,
2426) -> Option<(push::ReplayKey, BindTrust)> {
2427 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2428 Some((
2429 push::ReplayKey {
2430 root: root.clone(),
2431 harness: retained.harness.clone(),
2432 session: session.to_string(),
2433 },
2434 retained.trust,
2435 ))
2436}
2437pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2440
2441#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2442enum ModuleLoopExit {
2443 Graceful,
2446 ConnectionLost,
2449 SkipSearchFlush,
2454}
2455
2456pub fn run_subc_mode(
2461 connection_file_path: &Path,
2462 ctx: Arc<AppContext>,
2463 executor: Arc<Executor>,
2464 dispatch: DispatchFn,
2465 user_config_path: Option<PathBuf>,
2466) -> Result<(), SubcError> {
2467 run_subc_mode_inner(
2471 connection_file_path,
2472 ctx,
2473 executor,
2474 dispatch,
2475 user_config_path,
2476 false,
2477 MAX_FRAME_BODY_LEN as usize,
2478 None,
2479 )
2480}
2481
2482fn run_subc_mode_inner(
2483 connection_file_path: &Path,
2484 ctx: Arc<AppContext>,
2485 executor: Arc<Executor>,
2486 dispatch: DispatchFn,
2487 user_config_path: Option<PathBuf>,
2488 allow_native_passthrough: bool,
2489 tool_response_body_limit: usize,
2490 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2491) -> Result<(), SubcError> {
2492 let runtime = tokio::runtime::Builder::new_current_thread()
2493 .enable_all()
2494 .build()
2495 .map_err(SubcError::Runtime)?;
2496
2497 let executor_for_loop = Arc::clone(&executor);
2498 let loop_result = runtime.block_on(async move {
2499 let shared_app = ctx.app();
2500 drop(ctx);
2501 let stream =
2502 connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2503 log::info!(
2504 "subc attach: authenticated to daemon via {}",
2505 connection_file_path.display()
2506 );
2507 let (read_half, write_half) = tokio::io::split(stream);
2508 run_module_loop(
2509 read_half,
2510 write_half,
2511 connection_file_path,
2512 shared_app,
2513 executor_for_loop,
2514 dispatch,
2515 user_config_path,
2516 allow_native_passthrough,
2517 tool_response_body_limit,
2518 lifecycle_probe,
2519 )
2520 .await
2521 });
2522
2523 let actor_contexts = executor.actor_contexts();
2524 if matches!(
2525 loop_result,
2526 Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2527 ) {
2528 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2531 }
2532 for actor_ctx in &actor_contexts {
2533 actor_ctx.lsp().shutdown_all();
2534 actor_ctx.bash_background().detach();
2535 }
2536
2537 match loop_result {
2538 Ok(exit) => module_loop_exit_result(exit),
2539 Err(error) => Err(error),
2540 }
2541}
2542
2543fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2547 match exit {
2548 ModuleLoopExit::Graceful => Ok(()),
2549 ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2550 ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2551 }
2552}
2553
2554fn note_fatal_panic_response(response: &Response) -> bool {
2558 let fatal = response_is_fatal_panic(response);
2559 if fatal {
2560 log::error!(
2561 "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2562 response.id,
2563 response
2564 .data
2565 .get("message")
2566 .and_then(Value::as_str)
2567 .unwrap_or("(no message)")
2568 );
2569 }
2570 fatal
2571}
2572
2573fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2574 for actor_ctx in actor_contexts {
2575 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2576 }
2577 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2578}
2579
2580#[doc(hidden)]
2585pub fn run_subc_mode_for_test(
2586 connection_file_path: &Path,
2587 ctx: Arc<AppContext>,
2588 executor: Arc<Executor>,
2589 dispatch: DispatchFn,
2590 user_config_path: Option<PathBuf>,
2591) -> Result<(), SubcError> {
2592 run_subc_mode_inner(
2593 connection_file_path,
2594 ctx,
2595 executor,
2596 dispatch,
2597 user_config_path,
2598 true,
2599 MAX_FRAME_BODY_LEN as usize,
2600 None,
2601 )
2602}
2603
2604#[doc(hidden)]
2606pub fn run_subc_mode_for_test_with_lifecycle_probe(
2607 connection_file_path: &Path,
2608 ctx: Arc<AppContext>,
2609 executor: Arc<Executor>,
2610 dispatch: DispatchFn,
2611 user_config_path: Option<PathBuf>,
2612 lifecycle_probe: SubcTestLifecycleProbe,
2613) -> Result<(), SubcError> {
2614 run_subc_mode_inner(
2615 connection_file_path,
2616 ctx,
2617 executor,
2618 dispatch,
2619 user_config_path,
2620 true,
2621 MAX_FRAME_BODY_LEN as usize,
2622 Some(lifecycle_probe),
2623 )
2624}
2625
2626#[doc(hidden)]
2629pub fn run_subc_mode_for_test_with_response_body_limit(
2630 connection_file_path: &Path,
2631 ctx: Arc<AppContext>,
2632 executor: Arc<Executor>,
2633 dispatch: DispatchFn,
2634 user_config_path: Option<PathBuf>,
2635 tool_response_body_limit: usize,
2636) -> Result<(), SubcError> {
2637 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2638 run_subc_mode_inner(
2639 connection_file_path,
2640 ctx,
2641 executor,
2642 dispatch,
2643 user_config_path,
2644 true,
2645 tool_response_body_limit,
2646 None,
2647 )
2648}
2649
2650#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2651enum AttachErrorClass {
2652 Transient,
2653 Permanent,
2654}
2655
2656impl fmt::Display for AttachErrorClass {
2657 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2658 match self {
2659 Self::Transient => f.write_str("transient"),
2660 Self::Permanent => f.write_str("permanent"),
2661 }
2662 }
2663}
2664
2665#[derive(Clone, Copy)]
2666struct AttachRetryPolicy {
2667 budget: Duration,
2668 initial_backoff: Duration,
2669 max_backoff: Duration,
2670 jitter_percent: u64,
2671}
2672
2673const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2674 budget: ATTACH_RETRY_BUDGET,
2675 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2676 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2677 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2678};
2679
2680fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2683 let transient = match error {
2684 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2685 SubcError::Auth { source, .. } => match source {
2686 subc_transport::AuthError::Timeout { .. }
2687 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2688 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2689 _ => false,
2690 },
2691 _ => false,
2692 };
2693 if transient {
2694 AttachErrorClass::Transient
2695 } else {
2696 AttachErrorClass::Permanent
2697 }
2698}
2699
2700fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2701 matches!(
2702 kind,
2703 io::ErrorKind::ConnectionRefused
2704 | io::ErrorKind::TimedOut
2705 | io::ErrorKind::ConnectionReset
2706 | io::ErrorKind::ConnectionAborted
2707 | io::ErrorKind::BrokenPipe
2708 | io::ErrorKind::UnexpectedEof
2709 )
2710}
2711
2712async fn connect_and_authenticate(
2716 connection_file_path: &Path,
2717 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2718) -> Result<TcpStream, SubcError> {
2719 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2720 .await
2721}
2722
2723async fn connect_and_authenticate_with_policy(
2724 connection_file_path: &Path,
2725 policy: AttachRetryPolicy,
2726 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2727) -> Result<TcpStream, SubcError> {
2728 let started_at = Instant::now();
2729 let deadline = started_at + policy.budget;
2730 let mut attempt = 0_u32;
2731 let mut backoff = policy.initial_backoff;
2732 let mut history = Vec::new();
2733
2734 loop {
2735 attempt = attempt.saturating_add(1);
2736 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2737 Ok(stream) => return Ok(stream),
2738 Err(error) => error,
2739 };
2740 let class = classify_attach_error(&error);
2741 let will_retry = class != AttachErrorClass::Permanent;
2742 if let Some(probe) = lifecycle_probe {
2743 probe.attach_decision(attempt, will_retry);
2744 }
2745 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2746 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2747
2748 if !will_retry {
2749 log_attach_final_failure(started_at.elapsed(), &history);
2750 return Err(error);
2751 }
2752
2753 let remaining = deadline.saturating_duration_since(Instant::now());
2754 if remaining.is_zero() {
2755 log_attach_final_failure(started_at.elapsed(), &history);
2756 return Err(error);
2757 }
2758
2759 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2760 log::info!(
2761 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2762 );
2763 tokio::time::sleep(delay).await;
2764
2765 if Instant::now() >= deadline {
2766 log_attach_final_failure(started_at.elapsed(), &history);
2767 return Err(error);
2768 }
2769 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2770 }
2771}
2772
2773fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2774 let jitter_percent = jitter_percent.min(100);
2775 if jitter_percent == 0 {
2776 return base;
2777 }
2778
2779 let mut random_bytes = [0_u8; 8];
2780 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2781 u64::from_le_bytes(random_bytes)
2782 } else {
2783 let timestamp = std::time::SystemTime::now()
2784 .duration_since(std::time::UNIX_EPOCH)
2785 .unwrap_or_default()
2786 .subsec_nanos();
2787 u64::from(timestamp) ^ u64::from(attempt)
2788 };
2789 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2790 let multiplier_percent = 100 - jitter_percent + random % span;
2791 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2792 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2793}
2794
2795fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2796 log::error!(
2797 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2798 history.len(),
2799 history.join(" | ")
2800 );
2801}
2802
2803async fn connect_and_authenticate_once(
2804 connection_file_path: &Path,
2805 deadline: Instant,
2806) -> Result<TcpStream, SubcError> {
2807 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2810 SubcError::ConnectionFile {
2811 path: connection_file_path.to_path_buf(),
2812 source,
2813 }
2814 })?;
2815
2816 let endpoint = conn
2817 .endpoints
2818 .first()
2819 .ok_or_else(|| SubcError::NoEndpoint {
2820 path: connection_file_path.to_path_buf(),
2821 })?;
2822 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2823 let ip = endpoint
2824 .host
2825 .parse::<IpAddr>()
2826 .map_err(|_| SubcError::InvalidEndpoint {
2827 path: connection_file_path.to_path_buf(),
2828 endpoint: endpoint_label.clone(),
2829 })?;
2830 let addr = SocketAddr::new(ip, endpoint.port);
2831
2832 let connect_budget = deadline.saturating_duration_since(Instant::now());
2833 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2834 .await
2835 .map_err(|_| SubcError::Connect {
2836 endpoint: endpoint_label.clone(),
2837 source: io::Error::new(
2838 io::ErrorKind::TimedOut,
2839 "initial subc attach retry budget elapsed during TCP connect",
2840 ),
2841 })?
2842 .map_err(|source| SubcError::Connect {
2843 endpoint: endpoint_label.clone(),
2844 source,
2845 })?;
2846 stream
2847 .set_nodelay(true)
2848 .map_err(|source| SubcError::Connect {
2849 endpoint: endpoint_label.clone(),
2850 source,
2851 })?;
2852
2853 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2854 authenticate_client(&mut stream, &conn, auth_budget)
2855 .await
2856 .map_err(|source| SubcError::Auth {
2857 endpoint: endpoint_label,
2858 source,
2859 })?;
2860
2861 Ok(stream)
2862}
2863
2864#[allow(clippy::too_many_arguments)]
2865async fn process_route_bind_completion(
2866 writer_tx: &WriterSender,
2867 completion: RouteBindCompletion,
2868 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2869 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2870 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2871 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2872 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2873 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2874 installed_route_epochs: &mut HashMap<u16, u32>,
2875 executor: &Arc<Executor>,
2876 standing_actor: &standing::StandingActor,
2877 shutdown: &Arc<Notify>,
2878 metrics: &Arc<DispatchPathMetrics>,
2879 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2880) -> Result<(), SubcError> {
2881 decrement_counted_channel(&metrics.control_completion_queued);
2882 handle_route_bind_completion(
2883 writer_tx,
2884 completion,
2885 routes,
2886 root_channels,
2887 session_identity,
2888 push_buffer,
2889 live_roots,
2890 pending_binds,
2891 installed_route_epochs,
2892 executor,
2893 standing_actor,
2894 shutdown,
2895 metrics,
2896 lifecycle_probe,
2897 )
2898 .await
2899}
2900
2901#[allow(clippy::too_many_arguments)]
2902async fn drain_pending_route_bind_completions(
2903 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2904 writer_tx: &WriterSender,
2905 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2906 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2907 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2908 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2909 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2910 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2911 installed_route_epochs: &mut HashMap<u16, u32>,
2912 executor: &Arc<Executor>,
2913 standing_actor: &standing::StandingActor,
2914 shutdown: &Arc<Notify>,
2915 metrics: &Arc<DispatchPathMetrics>,
2916 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2917) -> Result<usize, SubcError> {
2918 let mut drained = 0;
2919 while let Ok(completion) = control_completion_rx.try_recv() {
2920 process_route_bind_completion(
2921 writer_tx,
2922 completion,
2923 routes,
2924 root_channels,
2925 session_identity,
2926 push_buffer,
2927 live_roots,
2928 pending_binds,
2929 installed_route_epochs,
2930 executor,
2931 standing_actor,
2932 shutdown,
2933 metrics,
2934 lifecycle_probe,
2935 )
2936 .await?;
2937 drained += 1;
2938 }
2939 Ok(drained)
2940}
2941
2942async fn run_module_loop<R, W>(
2946 mut read: R,
2947 mut write: W,
2948 connection_file_path: &Path,
2949 shared_app: Arc<App>,
2950 executor: Arc<Executor>,
2951 dispatch: DispatchFn,
2952 user_config_path: Option<PathBuf>,
2953 allow_native_passthrough: bool,
2954 tool_response_body_limit: usize,
2955 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2956) -> Result<ModuleLoopExit, SubcError>
2957where
2958 R: AsyncRead + Unpin + Send + 'static,
2959 W: AsyncWrite + Unpin + Send + 'static,
2960{
2961 let hello = ModuleHelloBody {
2966 manifest: build_manifest(),
2967 protocol_ver: PROTOCOL_VERSION,
2968 control_ops: control_ops(),
2969 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2970 };
2971 let hello_frame = Frame::build(
2972 FrameType::Hello,
2973 control_flags(),
2974 0,
2975 0,
2976 HELLO_CORR,
2977 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2978 )
2979 .map_err(SubcError::FrameBuild)?;
2980 write_frame(&mut write, &hello_frame)
2981 .await
2982 .map_err(SubcError::FrameIo)?;
2983
2984 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2986 None => return Err(SubcError::ClosedBeforeHelloAck),
2987 Some(frame) => match frame.header.ty {
2988 FrameType::HelloAck => {
2989 log::info!("subc attach: registered (HelloAck received)");
2990 }
2991 FrameType::Error => {
2992 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2993 return Err(SubcError::HelloRejected { body });
2994 }
2995 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2996 },
2997 }
2998
2999 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
3000 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3001 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3002 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3009 let reader_task = spawn_reader_task(read, reader_tx);
3010 let shutdown = Arc::new(Notify::new());
3011 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3020 let mut next_maintenance_at = next_drain_at;
3021 let standing_actor =
3022 standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3023 standing_actor.reconcile_at_startup();
3026 let mut next_standing_pass_at = tokio::time::Instant::now();
3027 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3028 let (bash_deferred_tx, mut bash_deferred_rx) =
3029 mpsc::channel::<bash::BashDeferredCompletion>(256);
3030 let (deferred_response_tx, mut deferred_response_rx) =
3031 mpsc::unbounded_channel::<PendingSubcResponse>();
3032 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3033 let (control_completion_tx, mut control_completion_rx) =
3034 mpsc::channel::<RouteBindCompletion>(256);
3035 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3036 let lossy_overflow = Arc::new(push::LossyOverflow::default());
3037 let lossy_seq = Arc::new(AtomicU64::new(0));
3038 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3039 let (fleet_status_client, fleet_status_task) =
3040 spawn_fleet_status_dial(connection_file_path, 64);
3041 let push_senders = PushSenders {
3042 lossy_tx,
3043 reliable_tx,
3044 lossy_overflow: Arc::clone(&lossy_overflow),
3045 lossy_seq,
3046 fleet_status_client: fleet_status_client.clone(),
3047 };
3048 let connection_cancel = PersistentCancelSignal::new();
3049 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3050 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3051 let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3052 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3053 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3054 let mut bg_wake_pending = BgWakePending::new();
3055 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3056 let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3057 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3058 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3059 HashMap::new();
3060 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3061 let mut retry_buffer: RetryBuffer = HashMap::new();
3062 let mut reclaimed_routes = ReclaimedRoutes::default();
3063 let mut completed_tasks = push::CompletedTaskIds::default();
3064 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3065 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3066 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3067 let mut next_bash_ask_corr: u64 = 1;
3068 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3069 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3070 let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3071 let mut pending_responses = PendingSubcResponses::default();
3072 let health_rollup_cache = Arc::new(HealthRollupCache::new());
3073 let health_rollup_worker = HealthRollupWorker::start(
3074 Arc::clone(&health_rollup_cache),
3075 Arc::clone(&executor),
3076 Arc::clone(&shared_app),
3077 );
3078
3079 let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3080 shared_app.set_open_route_count(routes.len() + management_routes.len());
3081 crate::logging::perf_tick(Some(&executor));
3082 dispatch_path_metrics.mark_frame_loop_tick();
3083 let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3084 for resolved in ready_inspects {
3085 if let Err(error) = deliver_resolved_subc_response(
3086 &writer_tx,
3087 resolved,
3088 &routes,
3089 &mut live_roots,
3090 executor.as_ref(),
3091 &active_tool_calls,
3092 &shutdown,
3093 &dispatch_path_metrics,
3094 tool_response_body_limit,
3095 )
3096 .await
3097 {
3098 break 'module_loop Err(error);
3099 }
3100 }
3101 if let Err(error) = expire_pending_bash_asks(
3102 &writer_tx,
3103 &mut pending_bash_asks,
3104 &routes,
3105 &mut live_roots,
3106 &mut route_bash_cancels,
3107 &shutdown,
3108 &dispatch_path_metrics,
3109 )
3110 .await
3111 {
3112 break Err(error);
3113 }
3114
3115 match drain_pending_route_bind_completions(
3119 &mut control_completion_rx,
3120 &writer_tx,
3121 &mut routes,
3122 &mut root_channels,
3123 &mut session_identity,
3124 &mut push_buffer,
3125 &mut live_roots,
3126 &mut pending_binds,
3127 &mut installed_route_epochs,
3128 &executor,
3129 &standing_actor,
3130 &shutdown,
3131 &dispatch_path_metrics,
3132 lifecycle_probe.as_ref(),
3133 )
3134 .await
3135 {
3136 Ok(drained) => {
3137 if drained > 0 {
3138 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3139 health_rollup_worker.request_refresh();
3140 }
3141 }
3142 Err(error) => break Err(error),
3143 }
3144
3145 if tokio::time::Instant::now() >= next_drain_at {
3146 push::emit_bg_event_wakes(
3147 &writer_tx,
3148 &dispatch_path_metrics,
3149 &bg_subs,
3150 &mut bg_wake_pending,
3151 );
3152 dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3153 warn_slow_pending_binds(&mut pending_binds, &executor);
3154 warn_slow_running_interactive_jobs(&executor);
3155 if let Err(error) = expire_overdue_route_binds(
3156 &writer_tx,
3157 &executor,
3158 &mut pending_binds,
3159 &mut installed_route_epochs,
3160 &dispatch_path_metrics,
3161 )
3162 .await
3163 {
3164 break Err(error);
3165 }
3166
3167 let retried = push::drain_retry_buffers_for_bound_routes(
3168 &writer_tx,
3169 &dispatch_path_metrics,
3170 &routes,
3171 &mut retry_buffer,
3172 );
3173 if retried > 0 {
3174 log::debug!(
3175 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3176 );
3177 }
3178
3179 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3180 }
3181
3182 let overflow_batch = lossy_overflow.drain();
3188 if !overflow_batch.is_empty() {
3189 let (_, deferred) = push::drain_reliable_push_turn(
3190 &writer_tx,
3191 &dispatch_path_metrics,
3192 &routes,
3193 &root_channels,
3194 &session_identity,
3195 &mut retry_buffer,
3196 &mut push_buffer,
3197 &mut completed_tasks,
3198 &bg_sub_by_session,
3199 &mut bg_wake_pending,
3200 &mut bg_wake_epoch,
3201 &mut reliable_rx,
3202 None,
3203 lifecycle_probe.as_ref(),
3204 );
3205 if deferred {
3206 tokio::task::yield_now().await;
3207 }
3208
3209 let mut batch = Vec::new();
3210 while let Ok(item) = lossy_rx.try_recv() {
3211 batch.push(item);
3212 }
3213 batch.extend(overflow_batch);
3214 push::process_lossy_push_envelope_batch(
3215 &writer_tx,
3216 &dispatch_path_metrics,
3217 &routes,
3218 &root_channels,
3219 &completed_tasks,
3220 batch,
3221 );
3222 }
3223
3224 tokio::select! {
3225 biased;
3226 Some(completion) = control_completion_rx.recv() => {
3227 if let Err(error) = process_route_bind_completion(
3228 &writer_tx,
3229 completion,
3230 &mut routes,
3231 &mut root_channels,
3232 &mut session_identity,
3233 &mut push_buffer,
3234 &mut live_roots,
3235 &mut pending_binds,
3236 &mut installed_route_epochs,
3237 &executor,
3238 &standing_actor,
3239 &shutdown,
3240 &dispatch_path_metrics,
3241 lifecycle_probe.as_ref(),
3242 )
3243 .await
3244 {
3245 break Err(error);
3246 }
3247 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3248 health_rollup_worker.request_refresh();
3249 }
3250 _ = shutdown.notified() => {
3251 log::warn!("subc attach: fatal executor response requested teardown");
3252 break Ok(ModuleLoopExit::SkipSearchFlush);
3253 }
3254 maybe_frame = reader_rx.recv() => {
3255 let frame = match maybe_frame {
3256 None => {
3257 log::warn!(
3258 "subc attach: daemon connection ended without Goodbye; exiting for restart"
3259 );
3260 break Ok(ModuleLoopExit::ConnectionLost);
3261 }
3262 Some(Err(error)) => break Err(error),
3263 Some(Ok(frame)) => frame,
3264 };
3265 let phase_trace = frame.phase_trace;
3266 let frame = frame.frame;
3267
3268 if !ingress_route_should_be_processed(
3269 &installed_route_epochs,
3270 &reclaimed_routes,
3271 &frame,
3272 ) {
3273 log::debug!(
3274 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3275 frame.header.ty,
3276 frame.header.channel,
3277 frame.header.epoch
3278 );
3279 continue;
3280 }
3281
3282 match frame.header.ty {
3283 FrameType::Ping if frame.header.channel == 0 => {
3284 let pong = match Frame::build_with_version(
3285 frame.header.ver,
3286 FrameType::Pong,
3287 frame.header.flags,
3288 0,
3289 0,
3290 frame.header.corr,
3291 Vec::new(),
3292 ) {
3293 Ok(pong) => pong,
3294 Err(error) => break Err(SubcError::FrameBuild(error)),
3295 };
3296 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3297 break Err(error);
3298 }
3299 }
3300 FrameType::Goodbye if frame.header.channel == 0 => {
3301 log::info!("subc attach: received channel-0 Goodbye");
3302 break Ok(ModuleLoopExit::Graceful);
3303 }
3304 FrameType::Goodbye => {
3305 let channel = route_key(frame.header.channel, frame.header.epoch);
3306 if let Err(error) = teardown_installed_route(
3307 &writer_tx,
3308 &dispatch_path_metrics,
3309 &executor,
3310 channel,
3311 "Goodbye",
3312 None,
3313 &mut installed_route_epochs,
3314 &mut routes,
3315 &mut management_routes,
3316 &mut root_channels,
3317 &mut bg_subs,
3318 &mut bg_sub_by_session,
3319 &mut bg_wake_pending,
3320 &mut pending_bash_asks,
3321 &mut live_roots,
3322 &mut route_bash_cancels,
3323 &active_tool_calls,
3324 &mut pending_responses,
3325 &mut pending_binds,
3326 &mut retry_buffer,
3327 &mut push_buffer,
3328 &shutdown,
3329 tool_response_body_limit,
3330 lifecycle_probe.as_ref(),
3331 )
3332 .await
3333 {
3334 break Err(error);
3335 }
3336 }
3337 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3338 if let Err(error) = handle_bash_elicitation_reply(
3339 &writer_tx,
3340 &frame,
3341 &mut pending_bash_asks,
3342 &routes,
3343 &mut live_roots,
3344 &executor,
3345 &shutdown,
3346 &bash_deferred_tx,
3347 &bash_poll_touch_tx,
3348 &dispatch_path_metrics,
3349 &mut route_bash_cancels,
3350 dispatch,
3351 )
3352 .await
3353 {
3354 break Err(error);
3355 }
3356 }
3357 FrameType::Request if frame.header.channel == 0 => {
3358 if let Err(error) = handle_control_request(
3359 &writer_tx,
3360 &frame,
3361 &shared_app,
3362 &executor,
3363 &mut live_roots,
3364 &mut pending_binds,
3365 &mut installed_route_epochs,
3366 &mut routes,
3367 &mut management_routes,
3368 &mut root_channels,
3369 &mut bg_subs,
3370 &mut bg_sub_by_session,
3371 &mut bg_wake_pending,
3372 &mut pending_bash_asks,
3373 &mut route_bash_cancels,
3374 &active_tool_calls,
3375 &mut pending_responses,
3376 &mut retry_buffer,
3377 &mut push_buffer,
3378 &shutdown,
3379 &control_completion_tx,
3380 &dispatch_path_metrics,
3381 lifecycle_probe.as_ref(),
3382 &health_rollup_cache,
3383 &push_senders,
3384 dispatch,
3385 user_config_path.as_deref(),
3386 tool_response_body_limit,
3387 )
3388 .await
3389 {
3390 break Err(error);
3391 }
3392 }
3393 FrameType::Request => {
3394 let route = route_key(frame.header.channel, frame.header.epoch);
3395 let result = if management_routes.contains(&route) {
3396 handle_management_request(
3397 &writer_tx,
3398 &frame,
3399 &shared_app,
3400 &executor,
3401 &live_roots,
3402 &root_channels,
3403 &health_rollup_cache,
3404 &dispatch_path_metrics,
3405 )
3406 .await
3407 } else {
3408 handle_tool_call(
3409 &writer_tx,
3410 &frame,
3411 phase_trace,
3412 &routes,
3413 &pending_binds,
3414 &mut live_roots,
3415 &executor,
3416 &active_tool_calls,
3417 &pending_deferred_setups,
3418 &shutdown,
3419 &connection_cancel,
3420 &bash_deferred_tx,
3421 &bash_poll_touch_tx,
3422 &dispatch_path_metrics,
3423 &mut route_bash_cancels,
3424 &mut pending_bash_asks,
3425 &mut next_bash_ask_corr,
3426 &mut bg_subs,
3427 &mut bg_sub_by_session,
3428 &mut bg_wake_pending,
3429 &mut bg_wake_epoch,
3430 dispatch,
3431 &deferred_response_tx,
3432 allow_native_passthrough,
3433 tool_response_body_limit,
3434 )
3435 .await
3436 };
3437 if let Err(error) = result {
3438 break Err(error);
3439 }
3440 }
3441 FrameType::Cancel => {
3442 let channel = route_key(frame.header.channel, frame.header.epoch);
3443 cancel_active_tool_call(
3444 &active_tool_calls,
3445 executor.as_ref(),
3446 channel,
3447 frame.header.corr,
3448 "Cancel frame",
3449 );
3450 pending_responses.cancel_request(channel, frame.header.corr);
3451 if bg_subs.contains_key(&channel) {
3452 if let Err(error) = end_bg_subscription(
3453 &writer_tx,
3454 &dispatch_path_metrics,
3455 &mut bg_subs,
3456 &mut bg_sub_by_session,
3457 &mut bg_wake_pending,
3458 channel,
3459 routes.get(&channel),
3460 "cancel",
3461 )
3462 .await
3463 {
3464 break Err(error);
3465 }
3466 }
3467 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3468 &writer_tx,
3469 &mut pending_bash_asks,
3470 channel,
3471 frame.header.corr,
3472 &routes,
3473 &mut live_roots,
3474 &mut route_bash_cancels,
3475 &shutdown,
3476 &dispatch_path_metrics,
3477 )
3478 .await
3479 {
3480 break Err(error);
3481 }
3482 }
3483 _ => {}
3487 }
3488 }
3489 Some(pending) = deferred_response_rx.recv() => {
3490 if routes.contains_key(&pending.route)
3491 && active_tool_call_is_registered(
3492 &active_tool_calls,
3493 pending.route,
3494 pending.corr,
3495 )
3496 {
3497 pending_responses.register(pending);
3498 } else {
3499 if let Some(cancellation) = &pending.pending.cancellation {
3500 cancellation.request_cancel();
3501 }
3502 finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3503 }
3504 }
3505 Some((root_id, frame)) = reliable_rx.recv() => {
3506 let (_, deferred) = push::drain_reliable_push_turn(
3510 &writer_tx,
3511 &dispatch_path_metrics,
3512 &routes,
3513 &root_channels,
3514 &session_identity,
3515 &mut retry_buffer,
3516 &mut push_buffer,
3517 &mut completed_tasks,
3518 &bg_sub_by_session,
3519 &mut bg_wake_pending,
3520 &mut bg_wake_epoch,
3521 &mut reliable_rx,
3522 Some((root_id, frame)),
3523 lifecycle_probe.as_ref(),
3524 );
3525 if deferred {
3526 tokio::task::yield_now().await;
3527 }
3528 }
3529 Some((order, root_id, frame)) = lossy_rx.recv() => {
3530 let (_, deferred) = push::drain_reliable_push_turn(
3534 &writer_tx,
3535 &dispatch_path_metrics,
3536 &routes,
3537 &root_channels,
3538 &session_identity,
3539 &mut retry_buffer,
3540 &mut push_buffer,
3541 &mut completed_tasks,
3542 &bg_sub_by_session,
3543 &mut bg_wake_pending,
3544 &mut bg_wake_epoch,
3545 &mut reliable_rx,
3546 None,
3547 lifecycle_probe.as_ref(),
3548 );
3549 if deferred {
3550 tokio::task::yield_now().await;
3551 }
3552
3553 let mut batch = vec![(order, root_id, frame)];
3560 while let Ok(item) = lossy_rx.try_recv() {
3561 batch.push(item);
3562 }
3563 batch.extend(lossy_overflow.drain());
3564 push::process_lossy_push_envelope_batch(
3565 &writer_tx,
3566 &dispatch_path_metrics,
3567 &routes,
3568 &root_channels,
3569 &completed_tasks,
3570 batch,
3571 );
3572 }
3573 Some(done) = bash_deferred_rx.recv() => {
3574 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3575 if let Err(error) = bash::handle_bash_deferred_completion(
3576 &writer_tx,
3577 done,
3578 &routes,
3579 &mut live_roots,
3580 &mut route_bash_cancels,
3581 &shutdown,
3582 &dispatch_path_metrics,
3583 )
3584 .await
3585 {
3586 break Err(error);
3587 }
3588 }
3589 Some(root_id) = bash_poll_touch_rx.recv() => {
3590 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3591 if let Some(meta) = live_roots.get_mut(&root_id) {
3592 meta.note_activity();
3593 }
3594 }
3595 Some(completion) = maintenance_rx.recv() => {
3596 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3597 let root_id = completion.root_id.clone();
3598 let response = completion.response;
3599 let response_is_fatal = response_is_fatal_panic(&response);
3600 let bind_pending = pending_binds
3601 .values()
3602 .any(|pending| pending.bind_root_id == root_id);
3603 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3604 let defer_requeue = meta.unbound_quiesced || bind_pending;
3605 note_maintenance_completion(
3606 meta,
3607 completion.requeue_kind,
3608 response_is_fatal,
3609 defer_requeue,
3610 );
3611 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3612 } else {
3613 false
3614 };
3615 if requiesce {
3616 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3617 }
3618 push::clear_stale_bg_wakes_for_empty_sessions(
3619 &root_id,
3620 &completion.empty_bg_sessions,
3621 &bg_sub_by_session,
3622 &mut bg_wake_pending,
3623 &bg_wake_epoch,
3624 );
3625 if let Some(keys) = completion.unacked_bg_keys {
3626 bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3627 }
3628 record_bg_runtime_from_snapshots(
3629 &dispatch_path_metrics,
3630 bg_subs.len(),
3631 bg_wake_pending.len(),
3632 &bg_unacked_keys_by_root,
3633 );
3634 if response_is_fatal {
3635 if let Some(meta) = live_roots.get_mut(&root_id) {
3636 meta.maintenance_poisoned = true;
3637 }
3638 log::warn!(
3639 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3640 );
3641 }
3642 }
3643 _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3644 }
3647 _ = tokio::time::sleep_until(next_drain_at) => {
3648 }
3651 _ = tokio::time::sleep_until(next_maintenance_at) => {
3652 crate::logging::maybe_sweep_logs();
3657 let retention_registries = shared_app.try_memory_contexts().map(|contexts| {
3658 contexts
3659 .into_iter()
3660 .map(|(_, context)| context.bash_background().clone())
3661 .collect()
3662 });
3663 crate::db::compression_events::maybe_spawn_retention(
3664 shared_app.db(),
3665 retention_registries,
3666 );
3667 let reaped_lsp_children = shared_app
3668 .lsp_child_registry()
3669 .reap_children_with_gone_cwd_or_reclaimed_root();
3670 if reaped_lsp_children > 0 {
3671 log::warn!(
3672 "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3673 );
3674 }
3675 let now = Instant::now();
3676 reap_idle_lsp_servers(now, &live_roots, &executor);
3677 let reap = reap_idle_roots(
3678 now,
3679 &mut live_roots,
3680 &pending_binds,
3681 &root_channels,
3682 &executor,
3683 &dispatch_path_metrics,
3684 );
3685 for root_id in &reap.forgotten_deleted_roots {
3686 bg_unacked_keys_by_root.remove(root_id);
3687 purge_deleted_root_residents(
3688 root_id,
3689 &mut routes,
3690 &mut root_channels,
3691 &mut installed_route_epochs,
3692 &mut route_bash_cancels,
3693 &active_tool_calls,
3694 executor.as_ref(),
3695 &mut retry_buffer,
3696 &mut reclaimed_routes,
3697 &mut session_identity,
3698 &mut push_buffer,
3699 &mut bg_subs,
3700 &mut bg_sub_by_session,
3701 &mut bg_wake_pending,
3702 &mut bg_wake_epoch,
3703 &mut pending_bash_asks,
3704 &dispatch_path_metrics,
3705 );
3706 }
3707 if reap.evicted > 0 {
3708 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3709 }
3710 record_bg_runtime_from_snapshots(
3711 &dispatch_path_metrics,
3712 bg_subs.len(),
3713 bg_wake_pending.len(),
3714 &bg_unacked_keys_by_root,
3715 );
3716 submit_due_maintenance_jobs(
3717 &executor,
3718 &mut live_roots,
3719 &pending_binds,
3720 &bg_sub_by_session,
3721 &bg_wake_pending,
3722 &bg_wake_epoch,
3723 &maintenance_tx,
3724 &dispatch_path_metrics,
3725 );
3726 if tokio::time::Instant::now() >= next_standing_pass_at {
3727 standing_actor.tick();
3728 next_standing_pass_at = tokio::time::Instant::now()
3729 + standing::STANDING_MAINTENANCE_INTERVAL;
3730 }
3731 #[cfg(any(target_os = "macos", target_os = "linux"))]
3738 {
3739 let now_std = std::time::Instant::now();
3740 let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3741 }
3742 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3743 }
3744 }
3745 };
3746
3747 shared_app.set_open_route_count(0);
3748 health_rollup_worker.shutdown();
3749
3750 connection_cancel.cancel();
3751 cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3752 let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3753 while pending_deferred_setups.load(Ordering::SeqCst) != 0
3754 && tokio::time::Instant::now() < setup_drain_deadline
3755 {
3756 tokio::select! {
3757 biased;
3758 Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3759 _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3760 }
3761 }
3762 if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3763 log::warn!(
3764 "subc attach: timed out waiting for deferred response setup registration during shutdown"
3765 );
3766 }
3767 while let Ok(pending) = deferred_response_rx.try_recv() {
3768 pending_responses.register(pending);
3769 }
3770 for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3771 if let Err(error) = deliver_resolved_subc_response(
3772 &writer_tx,
3773 resolved,
3774 &routes,
3775 &mut live_roots,
3776 executor.as_ref(),
3777 &active_tool_calls,
3778 &shutdown,
3779 &dispatch_path_metrics,
3780 tool_response_body_limit,
3781 )
3782 .await
3783 {
3784 log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3785 }
3786 }
3787 quiesce_connection_roots(
3790 &mut live_roots,
3791 &mut pending_binds,
3792 &mut routes,
3793 &mut root_channels,
3794 &mut installed_route_epochs,
3795 &mut route_bash_cancels,
3796 &active_tool_calls,
3797 &executor,
3798 );
3799
3800 fleet_status_client.set_route_live(false);
3801 fleet_status_task.abort();
3802 let _ = fleet_status_task.await;
3803
3804 let mut loop_result = loop_result;
3805 if !pending_bash_asks.is_empty() {
3806 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3807 if let Err(error) = settle_all_pending_bash_asks(
3808 &writer_tx,
3809 &mut pending_bash_asks,
3810 &no_routes,
3811 &mut live_roots,
3812 &mut route_bash_cancels,
3813 &shutdown,
3814 &dispatch_path_metrics,
3815 )
3816 .await
3817 {
3818 loop_result = loop_result.and(Err(error));
3819 }
3820 }
3821
3822 reader_task.abort();
3825 drop(writer_tx);
3826 let writer_result = finish_writer_task(writer_task).await;
3827 loop_result.and_then(|exit| writer_result.map(|_| exit))
3828}
3829
3830fn spawn_writer_task<W>(
3831 mut write: W,
3832 mut rx: mpsc::Receiver<WriterFrame>,
3833 metrics: Arc<DispatchPathMetrics>,
3834) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3835where
3836 W: AsyncWrite + Unpin + Send + 'static,
3837{
3838 tokio::spawn(async move {
3839 let mut write_buffer = Vec::new();
3840 while let Some(mut queued) = rx.recv().await {
3841 let measure = queued.tool_response_trace.is_some();
3842 let dequeued = measure.then(Instant::now);
3843 metrics.writer_active.store(true, Ordering::Relaxed);
3844 decrement_counted_channel(&metrics.writer_queued);
3845 let write_timing = write_frame_contiguous(
3846 &mut write,
3847 queued.frame(),
3848 queued.body(),
3849 &mut write_buffer,
3850 measure,
3851 )
3852 .await;
3853 metrics.writer_active.store(false, Ordering::Relaxed);
3854 let write_timing = write_timing?;
3855
3856 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3857 (queued.tool_response_trace.take(), dequeued, write_timing)
3858 {
3859 if let Some(completed) = trace.finish(
3860 dequeued,
3861 write_timing.write_started,
3862 write_timing.write_finished,
3863 write_timing.frame_bytes,
3864 ) {
3865 log_ctx::with_session(Some(completed.session), || {
3866 crate::logging::note_tool_call_trace(
3867 &completed.name,
3868 &completed.root,
3869 completed.channel,
3870 completed.corr,
3871 completed.phases,
3872 );
3873 });
3874 }
3875 }
3876 }
3877 Ok(())
3878 })
3879}
3880
3881struct FrameWriteTiming {
3882 write_started: Instant,
3883 write_finished: Instant,
3884 frame_bytes: usize,
3885}
3886
3887async fn write_frame_contiguous<W>(
3891 writer: &mut W,
3892 frame: &Frame,
3893 body: &[u8],
3894 buffer: &mut Vec<u8>,
3895 measure: bool,
3896) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3897where
3898 W: AsyncWrite + Unpin,
3899{
3900 if frame.header.len as usize != body.len() {
3901 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3902 header_len: frame.header.len,
3903 body_len: body.len(),
3904 });
3905 }
3906
3907 let header = frame.header.encode();
3908 buffer.clear();
3909 buffer.reserve(header.len() + body.len());
3910 buffer.extend_from_slice(&header);
3911 buffer.extend_from_slice(body);
3912 let write_started = measure.then(Instant::now);
3913 writer
3914 .write_all(buffer)
3915 .await
3916 .map_err(subc_transport::FrameIoError::Io)?;
3917 Ok(write_started.map(|write_started| FrameWriteTiming {
3918 write_started,
3919 write_finished: Instant::now(),
3920 frame_bytes: buffer.len(),
3921 }))
3922}
3923
3924fn spawn_reader_task<R>(
3925 mut read: R,
3926 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3927) -> JoinHandle<()>
3928where
3929 R: AsyncRead + Unpin + Send + 'static,
3930{
3931 tokio::spawn(async move {
3932 loop {
3933 match read_frame(&mut read).await {
3934 Ok(Some(frame)) => {
3935 let decoded = DecodedFrame {
3936 frame,
3937 phase_trace: PhaseTrace::new(Instant::now()),
3938 };
3939 if tx.send(Ok(decoded)).await.is_err() {
3940 return;
3941 }
3942 }
3943 Ok(None) => {
3944 return;
3946 }
3947 Err(error) => {
3948 if let subc_transport::FrameIoError::Io(io_error) = &error {
3955 if matches!(
3956 io_error.kind(),
3957 std::io::ErrorKind::ConnectionReset
3958 | std::io::ErrorKind::ConnectionAborted
3959 ) {
3960 log::info!(
3961 "subc attach: connection reset by daemon; treating as close"
3962 );
3963 return;
3964 }
3965 }
3966 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3967 return;
3968 }
3969 }
3970 }
3971 })
3972}
3973
3974async fn finish_writer_task(
3975 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3976) -> Result<(), SubcError> {
3977 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3978 Ok(Ok(Ok(()))) => Ok(()),
3979 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3980 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3981 Err(_) => {
3982 writer_task.abort();
3983 Ok(())
3984 }
3985 }
3986}
3987
3988fn register_actor_for_bind(
3989 shared_app: &Arc<App>,
3990 executor: &Arc<Executor>,
3991 push_senders: &PushSenders,
3992 bind_root_id: &ProjectRootId,
3993 route_channel: u16,
3994 root_was_live: bool,
3995) -> bool {
3996 if executor.actor_registered(bind_root_id) {
3997 log::debug!(
3998 "subc attach: reusing actor for route {} root {}",
3999 route_channel,
4000 bind_root_id.as_path().display()
4001 );
4002 return false;
4003 }
4004
4005 if root_was_live {
4006 log::warn!(
4007 "subc attach: recreating missing actor for live root {} on route {}",
4008 bind_root_id.as_path().display(),
4009 route_channel
4010 );
4011 }
4012
4013 let actor_ctx = Arc::new(AppContext::from_app(
4014 Arc::clone(shared_app),
4015 Config::default(),
4016 ));
4017 install_bash_compressor(&actor_ctx);
4018 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4019 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4020 push_senders.clone(),
4021 bind_root_id.clone(),
4022 )));
4023 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4024 drop(actor_ctx);
4025 if inserted {
4026 log::debug!(
4030 "subc attach: registered actor for route {} root {}",
4031 route_channel,
4032 bind_root_id.as_path().display()
4033 );
4034 } else {
4035 log::debug!(
4036 "subc attach: actor appeared while binding route {} root {}; reusing it",
4037 route_channel,
4038 bind_root_id.as_path().display()
4039 );
4040 }
4041 inserted
4042}
4043
4044fn rollback_pending_bind_actor(
4045 executor: &Arc<Executor>,
4046 live_roots: &HashMap<ProjectRootId, RootMeta>,
4047 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4048 root_id: &ProjectRootId,
4049 inserted_new_actor: bool,
4050) {
4051 if !inserted_new_actor || live_roots.contains_key(root_id) {
4052 return;
4053 }
4054
4055 if let Some((route, pending)) = pending_binds
4056 .iter_mut()
4057 .find(|(_, pending)| &pending.bind_root_id == root_id)
4058 {
4059 pending.inserted_new_actor = true;
4060 log::debug!(
4061 "subc attach: transferred rollback ownership for root {} to pending route {}",
4062 root_id.as_path().display(),
4063 route
4064 );
4065 return;
4066 }
4067
4068 executor.remove_actor(root_id);
4069}
4070
4071fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4072 match response.data.get("code").and_then(|code| code.as_str()) {
4073 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4078 Some("cache_key_probe_failed") => "cache_key_probe_failed",
4082 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4086 _ => "config_divergence",
4087 }
4088}
4089
4090fn queue_post_bind_configure_and_completion_maintenance(
4091 root_id: &ProjectRootId,
4092 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4093) {
4094 let Some(meta) = live_roots.get_mut(root_id) else {
4095 return;
4096 };
4097 if meta.maintenance_poisoned || meta.maintenance_pending {
4098 return;
4099 }
4100
4101 meta.maintenance_pending = true;
4102 meta.maintenance_queued_kinds
4103 .push_back(MaintenanceDrainKind::ConfigureTail);
4104 meta.maintenance_queued_kinds
4105 .push_back(MaintenanceDrainKind::CompletionDrains);
4106}
4107
4108#[allow(clippy::too_many_arguments)]
4109async fn handle_route_bind_completion(
4110 tx: &WriterSender,
4111 completion: RouteBindCompletion,
4112 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4113 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4114 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4115 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4116 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4117 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4118 installed_route_epochs: &mut HashMap<u16, u32>,
4119 executor: &Arc<Executor>,
4120 standing_actor: &standing::StandingActor,
4121 shutdown: &Arc<Notify>,
4122 metrics: &Arc<DispatchPathMetrics>,
4123 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4124) -> Result<(), SubcError> {
4125 let route_id = completion.route;
4126 let Some(pending) = pending_binds.remove(&route_id) else {
4127 log::warn!(
4128 "subc attach: dropping RouteBind completion for non-pending route {}",
4129 completion.route
4130 );
4131 rollback_pending_bind_actor(
4132 executor,
4133 live_roots,
4134 pending_binds,
4135 &completion.bind_root_id,
4136 completion.inserted_new_actor,
4137 );
4138 let has_pending_bind = pending_binds
4139 .values()
4140 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4141 if !root_channels
4142 .get(&completion.bind_root_id)
4143 .is_some_and(|channels| !channels.is_empty())
4144 && !has_pending_bind
4145 {
4146 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4147 }
4148 remove_installed_route(installed_route_epochs, route_id);
4149 return Ok(());
4150 };
4151
4152 if pending.bind_root_id != completion.bind_root_id {
4153 log::warn!(
4154 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4155 completion.route,
4156 pending.bind_root_id.as_path().display(),
4157 completion.bind_root_id.as_path().display()
4158 );
4159 }
4160
4161 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4162 if pending.cancelled {
4163 rollback_pending_bind_actor(
4164 executor,
4165 live_roots,
4166 pending_binds,
4167 &completion.bind_root_id,
4168 inserted_new_actor,
4169 );
4170 let has_pending_bind = pending_binds
4171 .values()
4172 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4173 if !root_channels
4174 .get(&completion.bind_root_id)
4175 .is_some_and(|channels| !channels.is_empty())
4176 && !has_pending_bind
4177 {
4178 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4179 }
4180 log::debug!(
4181 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4182 completion.route,
4183 completion.bind_root_id.as_path().display()
4184 );
4185 remove_installed_route(installed_route_epochs, route_id);
4186 return Ok(());
4187 }
4188
4189 let failure = if !completion.configure_response.success {
4190 Some((
4191 &completion.configure_response,
4192 "configure failed during route bind",
4193 ))
4194 } else {
4195 None
4196 };
4197
4198 if let Some((response, fallback)) = failure {
4199 rollback_pending_bind_actor(
4200 executor,
4201 live_roots,
4202 pending_binds,
4203 &completion.bind_root_id,
4204 inserted_new_actor,
4205 );
4206 let has_pending_bind = pending_binds
4207 .values()
4208 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4209 if !root_channels
4210 .get(&completion.bind_root_id)
4211 .is_some_and(|channels| !channels.is_empty())
4212 && !has_pending_bind
4213 {
4214 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4215 }
4216 let message = response_message(response, fallback);
4217 let fatal = response_is_fatal_panic(response);
4218 let error_code = route_bind_error_code_for_configure_response(response);
4219 send_route_bind_error_parts(
4220 tx,
4221 completion.ver,
4222 completion.corr,
4223 completion.flags,
4224 error_code,
4225 &message,
4226 metrics,
4227 )
4228 .await?;
4229 remove_installed_route(installed_route_epochs, route_id);
4230 if fatal {
4231 signal_fatal_teardown(
4232 tx,
4233 Some(completion.route),
4234 completion.ver,
4235 completion.corr,
4236 shutdown,
4237 metrics,
4238 )
4239 .await;
4240 }
4241 return Ok(());
4242 }
4243
4244 remember_session_identity(session_identity, &completion.identity);
4245 let replay_key = push::ReplayKey::from_identity(&completion.identity);
4246 let bind_trust = completion.identity.trust;
4247 insert_route_channel(routes, root_channels, route_id, completion.identity);
4248 sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4249 let restore_watcher = live_roots
4250 .get(&completion.bind_root_id)
4251 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4252 live_roots
4253 .entry(completion.bind_root_id.clone())
4254 .and_modify(|meta| {
4255 meta.reactivate_bound();
4256 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4257 meta.maintenance_poisoned = false;
4258 })
4259 .or_insert_with(|| RootMeta::new(Instant::now()));
4260 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4261 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4262 meta.maintenance_poisoned = false;
4263 }
4264 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4265 standing_actor.begin_session_bind(&ctx);
4268 ctx.mark_subc_bound();
4269 if restore_watcher {
4270 crate::commands::configure::ensure_project_watcher(&ctx);
4271 }
4272 }
4273
4274 let ack =
4275 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4276 let response = Frame::build_with_version(
4277 completion.ver,
4278 FrameType::Response,
4279 control_flags(),
4280 0,
4281 0,
4282 completion.corr,
4283 ack,
4284 )
4285 .map_err(SubcError::FrameBuild)?;
4286 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4287 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4288 let replayed = push::replay_buffered_push_frames(
4289 tx,
4290 metrics,
4291 route_id,
4292 push_buffer,
4293 &replay_key,
4294 bind_trust,
4295 lifecycle_probe,
4296 );
4297 if replayed > 0 {
4298 log::debug!(
4299 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4300 replayed,
4301 completion.route,
4302 replay_key.root.as_path().display(),
4303 replay_key.harness,
4304 replay_key.session
4305 );
4306 }
4307 log::info!(
4308 "subc attach: route {} bound to root {}",
4309 completion.route,
4310 completion.bind_root_id.as_path().display()
4311 );
4312 Ok(())
4313}
4314
4315async fn expire_overdue_route_binds(
4316 tx: &WriterSender,
4317 executor: &Arc<Executor>,
4318 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4319 installed_route_epochs: &mut HashMap<u16, u32>,
4320 metrics: &DispatchPathMetrics,
4321) -> Result<(), SubcError> {
4322 let now = Instant::now();
4323 let expired: Vec<_> = pending_binds
4324 .iter()
4325 .filter_map(|(route, pending)| {
4326 let age = now.saturating_duration_since(pending.started_at);
4327 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4328 (
4329 *route,
4330 pending.corr,
4331 pending.ver,
4332 pending.flags,
4333 pending.bind_root_id.clone(),
4334 pending.configure_request_id.clone(),
4335 age,
4336 )
4337 })
4338 })
4339 .collect();
4340
4341 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4342 if let Some(pending) = pending_binds.get_mut(&route) {
4343 pending.cancelled = true;
4344 pending.deadline_reported = true;
4345 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4346 log::debug!(
4347 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4348 );
4349 }
4350 remove_installed_route(installed_route_epochs, route);
4351 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4352 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4353 send_route_bind_error_parts(
4354 tx,
4355 ver,
4356 corr,
4357 flags,
4358 "actor_not_ready",
4359 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4360 metrics,
4361 )
4362 .await?;
4363 log::warn!(
4364 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4365 route,
4366 root_id.as_path().display(),
4367 deadline_ms,
4368 configure_request_id
4369 );
4370 }
4371
4372 Ok(())
4373}
4374
4375fn record_bg_runtime_from_snapshots(
4376 metrics: &DispatchPathMetrics,
4377 subscriptions: usize,
4378 wake_pending: usize,
4379 unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4380) {
4381 let unacked_total = unacked_keys_by_root
4382 .values()
4383 .flatten()
4384 .collect::<HashSet<_>>()
4385 .len();
4386 metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4387}
4388
4389async fn send_cached_health_response(
4390 tx: &WriterSender,
4391 frame: &Frame,
4392 shared_app: &App,
4393 executor: &Executor,
4394 pending_binds: &HashMap<RouteChannel, PendingBind>,
4395 metrics: &DispatchPathMetrics,
4396 health_rollup_cache: &HealthRollupCache,
4397) -> Result<(), SubcError> {
4398 let report = build_health_report(
4399 health_rollup_cache,
4400 executor,
4401 pending_binds,
4402 metrics,
4403 shared_app,
4404 );
4405 let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4406 let response = Frame::build_with_version(
4407 frame.header.ver,
4408 FrameType::Response,
4409 frame.header.flags,
4410 0,
4411 0,
4412 frame.header.corr,
4413 body,
4414 )
4415 .map_err(SubcError::FrameBuild)?;
4416 send_frame(tx, metrics, response).await
4417}
4418
4419#[allow(clippy::too_many_arguments)]
4423async fn handle_control_request(
4424 tx: &WriterSender,
4425 frame: &Frame,
4426 shared_app: &Arc<App>,
4427 executor: &Arc<Executor>,
4428 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4429 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4430 installed_route_epochs: &mut HashMap<u16, u32>,
4431 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4432 management_routes: &mut HashSet<RouteChannel>,
4433 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4434 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4435 bg_sub_by_session: &mut BgSubsBySession,
4436 bg_wake_pending: &mut BgWakePending,
4437 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4438 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4439 active_tool_calls: &ActiveToolCalls,
4440 pending_responses: &mut PendingSubcResponses,
4441 retry_buffer: &mut RetryBuffer,
4442 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4443 shutdown: &Arc<Notify>,
4444 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4445 metrics: &Arc<DispatchPathMetrics>,
4446 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4447 health_rollup_cache: &HealthRollupCache,
4448 push_senders: &PushSenders,
4449 dispatch: DispatchFn,
4450 user_config_path: Option<&Path>,
4451 tool_response_body_limit: usize,
4452) -> Result<(), SubcError> {
4453 let request =
4454 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4455 match request {
4456 ModuleControlRequest::RouteBind {
4457 route_channel,
4458 epoch,
4459 target,
4460 identity,
4461 principal,
4462 consumer_capabilities,
4463 admission_facts: _,
4464 } => {
4465 let route_id = route_key(route_channel, epoch);
4466 if epoch == 0 {
4467 return send_route_bind_error(
4468 tx,
4469 frame,
4470 "config_divergence",
4471 "route bind uses an invalid channel generation",
4472 metrics,
4473 )
4474 .await;
4475 }
4476
4477 let bind_trust = trust_for_bind(&identity.harness, &principal);
4478 if let RouteTarget::ManagementSurface { module_id } = &target {
4479 if module_id != "aft" {
4480 return send_route_bind_error(
4481 tx,
4482 frame,
4483 "route_refused",
4484 "management route target is not AFT",
4485 metrics,
4486 )
4487 .await;
4488 }
4489 if !matches!(bind_trust, BindTrust::FirstParty) {
4490 return send_route_bind_error(
4491 tx,
4492 frame,
4493 "route_refused",
4494 "AFT management routes require a first-party principal",
4495 metrics,
4496 )
4497 .await;
4498 }
4499 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4500 if installed_epoch >= epoch {
4501 return send_route_bind_error(
4502 tx,
4503 frame,
4504 "config_divergence",
4505 "route bind generation is not newer than the installed generation",
4506 metrics,
4507 )
4508 .await;
4509 }
4510 teardown_installed_route(
4511 tx,
4512 metrics,
4513 executor,
4514 route_key(route_channel, installed_epoch),
4515 "higher-epoch RouteBind",
4516 None,
4517 installed_route_epochs,
4518 routes,
4519 management_routes,
4520 root_channels,
4521 bg_subs,
4522 bg_sub_by_session,
4523 bg_wake_pending,
4524 pending_bash_asks,
4525 live_roots,
4526 route_bash_cancels,
4527 active_tool_calls,
4528 pending_responses,
4529 pending_binds,
4530 retry_buffer,
4531 push_buffer,
4532 shutdown,
4533 tool_response_body_limit,
4534 lifecycle_probe,
4535 )
4536 .await?;
4537 }
4538 if pending_binds.contains_key(&route_id) {
4539 return send_route_bind_error(
4540 tx,
4541 frame,
4542 "config_divergence",
4543 "route bind is already pending for channel",
4544 metrics,
4545 )
4546 .await;
4547 }
4548
4549 installed_route_epochs.insert(route_channel, epoch);
4550 management_routes.insert(route_id);
4551 return send_route_bind_ack(
4552 tx,
4553 frame.header.ver,
4554 frame.header.corr,
4555 frame.header.flags,
4556 metrics,
4557 )
4558 .await;
4559 }
4560 if matches!(&target, RouteTarget::InternalService { .. }) {
4561 return send_route_bind_error(
4562 tx,
4563 frame,
4564 "route_refused",
4565 "AFT does not provide an internal-service route",
4566 metrics,
4567 )
4568 .await;
4569 }
4570
4571 let mut bind_root_id = None;
4572 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4573 if installed_epoch >= epoch {
4574 return send_route_bind_error(
4575 tx,
4576 frame,
4577 "config_divergence",
4578 "route bind generation is not newer than the installed generation",
4579 metrics,
4580 )
4581 .await;
4582 }
4583
4584 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4585 Ok(root_id) => root_id,
4586 Err(error) => {
4587 return send_route_bind_error(
4588 tx,
4589 frame,
4590 "config_divergence",
4591 &format!("invalid route project root: {error}"),
4592 metrics,
4593 )
4594 .await;
4595 }
4596 };
4597 teardown_installed_route(
4598 tx,
4599 metrics,
4600 executor,
4601 route_key(route_channel, installed_epoch),
4602 "higher-epoch RouteBind",
4603 Some(&replacement_root),
4604 installed_route_epochs,
4605 routes,
4606 management_routes,
4607 root_channels,
4608 bg_subs,
4609 bg_sub_by_session,
4610 bg_wake_pending,
4611 pending_bash_asks,
4612 live_roots,
4613 route_bash_cancels,
4614 active_tool_calls,
4615 pending_responses,
4616 pending_binds,
4617 retry_buffer,
4618 push_buffer,
4619 shutdown,
4620 tool_response_body_limit,
4621 lifecycle_probe,
4622 )
4623 .await?;
4624 bind_root_id = Some(replacement_root);
4625 }
4626 if pending_binds.contains_key(&route_id) {
4627 return send_route_bind_error(
4628 tx,
4629 frame,
4630 "config_divergence",
4631 "route bind is already pending for channel",
4632 metrics,
4633 )
4634 .await;
4635 }
4636 let bind_root_id = match bind_root_id {
4637 Some(root_id) => root_id,
4638 None => match ProjectRootId::from_path(&identity.project_root) {
4639 Ok(root_id) => root_id,
4640 Err(error) => {
4641 return send_route_bind_error(
4642 tx,
4643 frame,
4644 "config_divergence",
4645 &format!("invalid route project root: {error}"),
4646 metrics,
4647 )
4648 .await;
4649 }
4650 },
4651 };
4652
4653 let request_id = format!("subc-bind-{route_channel}");
4656 let bind_project_root = identity.project_root.clone();
4657 let bind_harness = identity.harness.clone();
4658 let bind_session = identity.session.clone();
4659 let bind_principal_id = principal_id(&principal);
4660 let consumer_elicitation_capable = consumer_capabilities
4665 .as_ref()
4666 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4667 log::info!(
4668 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4669 route_channel,
4670 bind_harness,
4671 principal_label(&principal),
4672 bind_trust.label(),
4673 consumer_elicitation_capable
4674 );
4675
4676 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4681 user_config_path,
4682 Path::new(&bind_project_root),
4683 );
4684 let config_tiers: Vec<Value> = local_tiers
4685 .iter()
4686 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4687 .collect();
4688 let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4691 let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4692 &local_tiers,
4693 active_harness.as_ref(),
4694 )
4695 .config
4696 .diagnostics_on_edit;
4697 let configure_json = json!({
4698 "id": request_id,
4699 "command": "configure",
4700 "project_root": bind_project_root,
4701 "harness": bind_harness,
4702 "session_id": bind_session.clone(),
4703 "config": config_tiers,
4704 });
4705 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4706 Ok(req) => req,
4707 Err(error) => {
4708 return send_route_bind_error(
4709 tx,
4710 frame,
4711 "config_divergence",
4712 &format!("failed to build configure request: {error}"),
4713 metrics,
4714 )
4715 .await;
4716 }
4717 };
4718
4719 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4720 root: bind_root_id.clone(),
4721 project_root: PathBuf::from(&bind_project_root),
4722 harness: bind_harness.clone(),
4723 session: bind_session.clone(),
4724 trust: bind_trust,
4725 spawn_principal: AuthenticatedPrincipal::RouteBind {
4726 trust: bind_trust.sandbox_trust(),
4727 route_channel,
4728 route_epoch: epoch,
4729 project_root: PathBuf::from(&bind_project_root),
4730 harness: bind_harness.clone(),
4731 session_id: bind_session.clone(),
4732 principal_id: bind_principal_id,
4733 },
4734 consumer_elicitation_capable,
4735 }));
4736 let configure_session = route_identity.session.clone();
4737 let root_was_live = live_roots.contains_key(&bind_root_id);
4738 let inserted_new_actor = register_actor_for_bind(
4739 shared_app,
4740 executor,
4741 push_senders,
4742 &bind_root_id,
4743 route_channel,
4744 root_was_live,
4745 );
4746
4747 sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4748 let configure_request_id = configure_req.id.clone();
4749 installed_route_epochs.insert(route_channel, epoch);
4750 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4751 meta.maintenance_queued_kinds.clear();
4752 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4753 }
4754 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4755 bind_root_id.clone(),
4756 Lane::Mutating,
4757 configure_request_id.clone(),
4758 Box::new(move |ctx| {
4759 log_ctx::with_session(Some(configure_session.clone()), || {
4760 dispatch(configure_req, ctx)
4761 })
4762 }),
4763 );
4764 pending_binds.insert(
4765 route_id,
4766 PendingBind {
4767 bind_root_id: bind_root_id.clone(),
4768 inserted_new_actor,
4769 cancelled: false,
4770 configure_request_id: configure_request_id.clone(),
4771 started_at: Instant::now(),
4772 warned_half_deadline: false,
4773 deadline_reported: false,
4774 corr: frame.header.corr,
4775 ver: frame.header.ver,
4776 flags: frame.header.flags,
4777 cancellation: configure_cancellation,
4778 },
4779 );
4780
4781 let completion_tx = control_completion_tx.clone();
4782 let completion_identity = route_identity;
4783 let completion_root = bind_root_id.clone();
4784 let completion_route_channel = route_channel;
4785 let completion_ver = frame.header.ver;
4786 let completion_corr = frame.header.corr;
4787 let completion_flags = frame.header.flags;
4788 let completion_metrics = Arc::clone(metrics);
4789 tokio::spawn(async move {
4790 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4791 let configure_response =
4792 await_executor_response(configure_rx, configure_request_id.clone()).await;
4793 let completion = RouteBindCompletion {
4798 route: route_key(completion_route_channel, epoch),
4799 identity: completion_identity,
4800 bind_root_id: completion_root,
4801 inserted_new_actor,
4802 configure_response,
4803 diagnostics_on_edit,
4804 ver: completion_ver,
4805 corr: completion_corr,
4806 flags: completion_flags,
4807 };
4808 if send_counted_channel(
4809 &completion_tx,
4810 &completion_metrics.control_completion_queued,
4811 completion,
4812 )
4813 .await
4814 .is_err()
4815 {
4816 log::debug!(
4817 "subc attach: dropped RouteBind completion for route {} after loop exit",
4818 completion_route_channel
4819 );
4820 }
4821 });
4822
4823 Ok(())
4826 }
4827 ModuleControlRequest::HealthCheck {} => {
4828 send_cached_health_response(
4829 tx,
4830 frame,
4831 shared_app,
4832 executor,
4833 pending_binds,
4834 metrics,
4835 health_rollup_cache,
4836 )
4837 .await
4838 }
4839 }
4840}
4841
4842async fn handle_management_request(
4843 tx: &WriterSender,
4844 frame: &Frame,
4845 shared_app: &App,
4846 executor: &Executor,
4847 live_roots: &HashMap<ProjectRootId, RootMeta>,
4848 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4849 health_rollup_cache: &HealthRollupCache,
4850 metrics: &DispatchPathMetrics,
4851) -> Result<(), SubcError> {
4852 let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4853 let operation = decoded
4854 .as_ref()
4855 .and_then(|value| value.get("op"))
4856 .and_then(Value::as_str);
4857 let Some(operation) = operation else {
4858 let error = build_error_frame(
4859 frame.header.ver,
4860 frame.header.channel,
4861 frame.header.epoch,
4862 frame.header.corr,
4863 frame.header.flags,
4864 "unknown_management_op",
4865 "management routes accept only declared operation envelopes",
4866 )?;
4867 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4868 };
4869
4870 let result = match operation {
4871 crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4872 "management-memory-census",
4873 memory_census_with_lifecycle(
4874 health_rollup_cache,
4875 shared_app,
4876 executor,
4877 live_roots,
4878 root_channels,
4879 ),
4880 ),
4881 crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4882 let params = decoded
4883 .as_ref()
4884 .and_then(|value| value.get("params"))
4885 .cloned()
4886 .unwrap_or_else(|| json!({}));
4887 let Some(params) = params.as_object() else {
4888 return send_management_response(
4889 tx,
4890 frame,
4891 operation,
4892 Response::error(
4893 "management-health-digest",
4894 "invalid_request",
4895 "health.digest params must be an object",
4896 ),
4897 metrics,
4898 )
4899 .await;
4900 };
4901 let root = params
4902 .get("project_root")
4903 .or_else(|| params.get("root"))
4904 .and_then(Value::as_str);
4905 let Some(root) = root else {
4906 return send_management_response(
4907 tx,
4908 frame,
4909 operation,
4910 Response::error(
4911 "management-health-digest",
4912 "invalid_request",
4913 "health.digest requires params.project_root",
4914 ),
4915 metrics,
4916 )
4917 .await;
4918 };
4919
4920 let mut request = params.clone();
4921 request.insert("id".to_string(), json!("management-health-digest"));
4922 request.insert(
4923 "command".to_string(),
4924 json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4925 );
4926 let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4927 .map_err(SubcError::Json)?;
4928 match ProjectRootId::from_path(Path::new(root))
4929 .ok()
4930 .and_then(|root_id| executor.actor_context(&root_id))
4931 {
4932 Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4933 None => crate::commands::health_digest::root_not_bound_response(&request, root),
4934 }
4935 }
4936 _ => {
4937 let error = build_error_frame(
4938 frame.header.ver,
4939 frame.header.channel,
4940 frame.header.epoch,
4941 frame.header.corr,
4942 frame.header.flags,
4943 "unknown_management_op",
4944 &format!("management operation {operation:?} is not declared by AFT"),
4945 )?;
4946 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4947 }
4948 };
4949
4950 send_management_response(tx, frame, operation, result, metrics).await
4951}
4952
4953fn memory_census_with_lifecycle(
4954 health_rollup_cache: &HealthRollupCache,
4955 shared_app: &App,
4956 executor: &Executor,
4957 live_roots: &HashMap<ProjectRootId, RootMeta>,
4958 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4959) -> Value {
4960 let mut census = health_rollup_cache.memory_census();
4961 if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4962 let lifecycle = shared_app.lifecycle_census_snapshot();
4963 for (root_id, meta) in live_roots {
4964 let root = root_id.as_path().display().to_string();
4965 let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4966 let age_ms = Instant::now()
4967 .saturating_duration_since(meta.last_touched)
4968 .as_millis()
4969 .min(u128::from(u64::MAX)) as u64;
4970 let ttl_ms = root_idle_ttl(executor, root_id)
4971 .as_millis()
4972 .min(u128::from(u64::MAX)) as u64;
4973 let lsp = lifecycle
4974 .lsp
4975 .children_by_root
4976 .iter()
4977 .find(|child| child.root == root);
4978 if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4979 let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4980 row.insert("root_id".to_string(), json!(root));
4981 row.insert("bound_routes".to_string(), json!(bound_routes));
4982 row.insert("last_request_age_ms".to_string(), json!(age_ms));
4983 row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4984 row.insert(
4985 "lsp_idle_ttl_ms".to_string(),
4986 json!(executor
4987 .actor_context(root_id)
4988 .map(|ctx| ctx
4989 .config()
4990 .idle
4991 .lsp_ttl()
4992 .as_millis()
4993 .min(u128::from(u64::MAX)) as u64)
4994 .unwrap_or(0)),
4995 );
4996 row.insert(
4997 "evictable_in_ms".to_string(),
4998 crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4999 .map_or(Value::Null, |value| json!(value)),
5000 );
5001 row.insert(
5002 "evictable_bytes".to_string(),
5003 if bound_routes == 0 {
5004 evictable_bytes
5005 } else {
5006 json!(0)
5007 },
5008 );
5009 row.insert(
5010 "lsp_children".to_string(),
5011 json!({
5012 "count": lsp.map_or(0, |child| child.count),
5013 "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5014 }),
5015 );
5016 }
5017 }
5018 }
5019 census
5020}
5021
5022async fn send_management_response(
5023 tx: &WriterSender,
5024 request: &Frame,
5025 operation: &str,
5026 result: Response,
5027 metrics: &DispatchPathMetrics,
5028) -> Result<(), SubcError> {
5029 let status = if result.success { "ok" } else { "error" };
5030 let body = json!({ "op": operation, "status": status, "data": result.data });
5031 let response = Frame::build_with_version(
5032 request.header.ver,
5033 FrameType::Response,
5034 request.header.flags,
5035 request.header.channel,
5036 request.header.epoch,
5037 request.header.corr,
5038 serde_json::to_vec(&body).map_err(SubcError::Json)?,
5039 )
5040 .map_err(SubcError::FrameBuild)?;
5041 send_reliable_writer_frame(tx, metrics, response, "management response").await
5042}
5043
5044fn install_bash_compressor(ctx: &AppContext) {
5045 let filter_registry_handle = ctx.shared_filter_registry();
5047 let compress_flag = ctx.bash_compress_flag();
5048 ctx.bash_background().set_compressor_with_exit_code(
5049 move |command: &str, output: String, exit_code: Option<i32>| {
5050 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5051 return crate::compress::CompressionResult::new(output);
5052 }
5053 let registry_guard = match filter_registry_handle.read() {
5054 Ok(g) => g,
5055 Err(poisoned) => poisoned.into_inner(),
5056 };
5057 crate::compress::compress_with_registry_exit_code(
5058 command,
5059 &output,
5060 exit_code,
5061 ®istry_guard,
5062 )
5063 },
5064 );
5065}
5066
5067async fn send_route_bind_ack(
5068 tx: &WriterSender,
5069 ver: u8,
5070 corr: u64,
5071 flags: Flags,
5072 metrics: &DispatchPathMetrics,
5073) -> Result<(), SubcError> {
5074 let body =
5075 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5076 let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5077 .map_err(SubcError::FrameBuild)?;
5078 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5079}
5080
5081async fn send_route_bind_error(
5082 tx: &WriterSender,
5083 frame: &Frame,
5084 code: &str,
5085 message: &str,
5086 metrics: &DispatchPathMetrics,
5087) -> Result<(), SubcError> {
5088 send_route_bind_error_parts(
5089 tx,
5090 frame.header.ver,
5091 frame.header.corr,
5092 frame.header.flags,
5093 code,
5094 message,
5095 metrics,
5096 )
5097 .await
5098}
5099
5100async fn send_route_bind_error_parts(
5101 tx: &WriterSender,
5102 ver: u8,
5103 corr: u64,
5104 flags: Flags,
5105 code: &str,
5106 message: &str,
5107 metrics: &DispatchPathMetrics,
5108) -> Result<(), SubcError> {
5109 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5110 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5111 log_route_bind_rejection(code, message);
5112 Ok(())
5113}
5114
5115fn log_route_bind_rejection(code: &str, message: &str) {
5123 const WINDOW: Duration = Duration::from_secs(60);
5124 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5125 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5126 let mut map = match map.try_lock() {
5127 Ok(map) => map,
5128 Err(_) => {
5130 log::warn!("subc attach: route bind rejected ({code}): {message}");
5131 return;
5132 }
5133 };
5134 let now = Instant::now();
5135 if map.len() > 512 {
5139 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5140 }
5141 match map.get_mut(message) {
5142 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5143 *suppressed += 1;
5144 }
5145 Some((window_start, suppressed)) => {
5146 if *suppressed > 0 {
5147 log::warn!(
5148 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5149 *suppressed
5150 );
5151 } else {
5152 log::warn!("subc attach: route bind rejected ({code}): {message}");
5153 }
5154 *window_start = now;
5155 *suppressed = 0;
5156 }
5157 None => {
5158 log::warn!("subc attach: route bind rejected ({code}): {message}");
5159 map.insert(message.to_string(), (now, 0));
5160 }
5161 }
5162}
5163
5164async fn handle_tool_call(
5169 tx: &WriterSender,
5170 frame: &Frame,
5171 mut phase_trace: PhaseTrace,
5172 routes: &HashMap<RouteChannel, RouteIdentity>,
5173 pending_binds: &HashMap<RouteChannel, PendingBind>,
5174 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5175 executor: &Arc<Executor>,
5176 active_tool_calls: &ActiveToolCalls,
5177 pending_deferred_setups: &Arc<AtomicUsize>,
5178 shutdown: &Arc<Notify>,
5179 connection_cancel: &PersistentCancelSignal,
5180 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5181 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5182 metrics: &Arc<DispatchPathMetrics>,
5183 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5184 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5185 next_bash_ask_corr: &mut u64,
5186 bg_subs: &mut HashMap<RouteChannel, BgSub>,
5187 bg_sub_by_session: &mut BgSubsBySession,
5188 bg_wake_pending: &mut BgWakePending,
5189 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5190 dispatch: DispatchFn,
5191 deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5192 allow_native_passthrough: bool,
5193 tool_response_body_limit: usize,
5194) -> Result<(), SubcError> {
5195 let route_id = route_key(frame.header.channel, frame.header.epoch);
5196 if pending_binds.contains_key(&route_id) {
5197 let error = build_error_frame(
5198 frame.header.ver,
5199 frame.header.channel,
5200 frame.header.epoch,
5201 frame.header.corr,
5202 frame.header.flags,
5203 "route_not_bound",
5204 "route is not bound before tool call",
5205 )?;
5206 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5207 }
5208
5209 let Some(identity) = routes.get(&route_id).cloned() else {
5210 let error = build_error_frame(
5211 frame.header.ver,
5212 frame.header.channel,
5213 frame.header.epoch,
5214 frame.header.corr,
5215 frame.header.flags,
5216 "route_not_bound",
5217 "route is not bound before tool call",
5218 )?;
5219 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5220 };
5221 let restore_watcher = live_roots
5222 .get(&identity.root)
5223 .is_some_and(|meta| meta.idle_artifacts_evicted);
5224 if let Some(meta) = live_roots.get_mut(&identity.root) {
5225 meta.reactivate_bound();
5226 }
5227 if restore_watcher {
5228 if let Some(ctx) = executor.actor_context(&identity.root) {
5229 crate::commands::configure::ensure_project_watcher(&ctx);
5230 }
5231 }
5232
5233 let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5234 Ok(request) => request,
5235 Err(error) => {
5236 let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5237 let Some(operation) = management_envelope
5238 .as_ref()
5239 .and_then(|value| value.get("op"))
5240 .and_then(Value::as_str)
5241 else {
5242 return Err(SubcError::Json(error));
5243 };
5244 RouteRequest::ToolCall(ToolCallRequest {
5245 name: operation.to_string(),
5246 arguments: management_envelope
5247 .and_then(|value| value.get("params").cloned())
5248 .unwrap_or_else(|| json!({})),
5249 edit_slot_survives: None,
5250 preview: false,
5251 })
5252 }
5253 };
5254 if matches!(
5255 route_request,
5256 RouteRequest::BgEvents(BgEventsRequest {
5257 op: BgEventsOp::BgEvents
5258 })
5259 ) {
5260 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5261 metrics.record_bg_subscription_ended(
5262 &old_sub.root,
5263 &old_sub.session,
5264 route_id,
5265 "resubscribe",
5266 );
5267 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5268 }
5269 if !identity.trust.allows_bash_observation() {
5270 bg_subs.remove(&route_id);
5271 bg_wake_pending.remove(&route_id);
5272 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5273 let denied_sub = BgSub {
5274 corr: frame.header.corr,
5275 ver: frame.header.ver,
5276 flags: frame.header.flags,
5277 root: identity.root.clone(),
5278 session: identity.session.clone(),
5279 };
5280 metrics.record_bg_subscription_ended(
5281 &identity.root,
5282 &identity.session,
5283 route_id,
5284 "subscribe-denied",
5285 );
5286 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5287 return Ok(());
5288 }
5289 bg_subs.insert(
5290 route_id,
5291 BgSub {
5292 corr: frame.header.corr,
5293 ver: frame.header.ver,
5294 flags: frame.header.flags,
5295 root: identity.root.clone(),
5296 session: identity.session.clone(),
5297 },
5298 );
5299 insert_bg_subscription_index(
5300 bg_sub_by_session,
5301 identity.root.clone(),
5302 identity.session.clone(),
5303 route_id,
5304 );
5305 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5306 push::arm_bg_wake(
5307 identity.root.clone(),
5308 identity.session.clone(),
5309 route_id,
5310 bg_wake_pending,
5311 bg_wake_epoch,
5312 metrics,
5313 );
5314 return Ok(());
5315 }
5316
5317 let RouteRequest::ToolCall(call) = route_request else {
5318 unreachable!("background event subscription returned above")
5319 };
5320 let bare_name = call.name;
5321 let arguments = strip_agent_preview_arg_owned(call.arguments);
5322 let format_context = crate::subc_format::FormatContext::from_tool_call(
5323 &bare_name,
5324 &arguments,
5325 identity.project_root.as_path(),
5326 );
5327
5328 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5329 let bind_trust = identity.trust;
5330 let diagnostics_on_edit = live_roots
5331 .get(&identity.root)
5332 .map(|meta| meta.diagnostics_on_edit)
5333 .unwrap_or(false);
5334
5335 let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5336 && arguments
5337 .get("sandbox")
5338 .or_else(|| {
5339 arguments
5340 .get("params")
5341 .and_then(|params| params.get("sandbox"))
5342 })
5343 .and_then(Value::as_str)
5344 == Some("host");
5345 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5346 let response = Response::error(
5347 request_id.clone(),
5348 "sandbox_escalation_denied",
5349 "sandbox host escalation is unavailable to untrusted principals",
5350 );
5351 let text = crate::subc_format::format_response_with_context(
5352 &bare_name,
5353 &response,
5354 &format_context,
5355 );
5356 let result = ToolCallResult { text, response };
5357 let response_frame = build_tool_response_frame_with_limit(
5358 frame.header.ver,
5359 route_id,
5360 frame.header.corr,
5361 frame.header.flags,
5362 &result,
5363 bind_trust,
5364 tool_response_body_limit,
5365 )?;
5366 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5367 }
5368
5369 if matches!(bind_trust, BindTrust::Untrusted)
5370 && is_bash_family_tool(&bare_name)
5371 && (!matches!(bare_name.as_str(), "bash" | "powershell")
5372 || !identity.consumer_elicitation_capable)
5373 {
5374 let response = bash::bash_denied_untrusted_response(request_id.clone());
5375 let text = crate::subc_format::format_response_with_context(
5376 &bare_name,
5377 &response,
5378 &format_context,
5379 );
5380 let result = ToolCallResult { text, response };
5381 let response_frame = build_tool_response_frame_with_limit(
5382 frame.header.ver,
5383 route_id,
5384 frame.header.corr,
5385 frame.header.flags,
5386 &result,
5387 bind_trust,
5388 tool_response_body_limit,
5389 )?;
5390 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5391 }
5392
5393 if !is_subc_agent_core_tool(&bare_name)
5401 && !is_subc_native_plumbing_tool(&bare_name)
5402 && !allow_native_passthrough
5403 {
5404 log::warn!(
5405 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5406 bare_name,
5407 frame.header.channel
5408 );
5409 let response = Response::error(
5410 request_id.clone(),
5411 "unknown_tool",
5412 format!("tool {:?} is not in the AFT tool manifest", bare_name),
5413 );
5414 let text = crate::subc_format::format_response_with_context(
5415 &bare_name,
5416 &response,
5417 &format_context,
5418 );
5419 let result = ToolCallResult { text, response };
5420 let response_frame = build_tool_response_frame_with_limit(
5421 frame.header.ver,
5422 route_id,
5423 frame.header.corr,
5424 frame.header.flags,
5425 &result,
5426 bind_trust,
5427 tool_response_body_limit,
5428 )?;
5429 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5430 }
5431
5432 if matches!(bare_name.as_str(), "bash" | "powershell") {
5433 if matches!(bind_trust, BindTrust::Untrusted) {
5434 let plan = match bash::prepare_bash_elicitation_plan(
5435 &arguments,
5436 identity.project_root.as_path(),
5437 ) {
5438 Ok(plan) => plan,
5439 Err(error) => {
5440 let response = Response::error(request_id.clone(), error.code, error.message);
5441 let text = crate::subc_format::format_response_with_context(
5442 &bare_name,
5443 &response,
5444 &format_context,
5445 );
5446 let result = ToolCallResult { text, response };
5447 let response_frame = build_tool_response_frame_with_limit(
5448 frame.header.ver,
5449 route_id,
5450 frame.header.corr,
5451 frame.header.flags,
5452 &result,
5453 bind_trust,
5454 tool_response_body_limit,
5455 )?;
5456 return send_reliable_writer_frame(
5457 tx,
5458 metrics,
5459 response_frame,
5460 "tool response",
5461 )
5462 .await;
5463 }
5464 };
5465
5466 let reverse_corr =
5467 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5468 let ask_frame = build_bash_elicitation_request_frame(
5469 frame.header.ver,
5470 route_id,
5471 reverse_corr,
5472 frame.header.flags,
5473 &plan.command,
5474 &plan.asks,
5475 )?;
5476
5477 let meta = live_roots
5478 .entry(identity.root.clone())
5479 .or_insert_with(|| RootMeta::new(Instant::now()));
5480 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5481 meta.reactivate_bound();
5482
5483 let route_cancel =
5484 route_bash_cancels
5485 .entry(route_id)
5486 .or_insert_with(|| bash::RouteBashCancel {
5487 token: PersistentCancelSignal::new(),
5488 active_waits: 0,
5489 });
5490 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5491 let cancel = bash::BashWaitCancel {
5492 connection: connection_cancel.clone(),
5493 route: route_cancel.token.clone(),
5494 };
5495 pending_bash_asks.insert(
5496 ReverseCorrKey {
5497 route: route_id,
5498 corr: reverse_corr,
5499 },
5500 PendingBashAsk {
5501 route: route_id,
5502 tool_corr: frame.header.corr,
5503 tool_flags: frame.header.flags,
5504 tool_ver: frame.header.ver,
5505 root: identity.root.clone(),
5506 project_root: identity.project_root.clone(),
5507 session_id: identity.session.clone(),
5508 spawn_principal: identity.spawn_principal.clone(),
5509 edit_slot_survives: call.edit_slot_survives,
5510 request_id,
5511 arguments,
5512 format_context,
5513 cancel,
5514 grants: plan.grants,
5515 expires_at: Instant::now() + bash_elicitation_timeout(),
5516 },
5517 );
5518 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5519 .await;
5520 }
5521
5522 let meta = live_roots
5523 .entry(identity.root.clone())
5524 .or_insert_with(|| RootMeta::new(Instant::now()));
5525 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5526 meta.reactivate_bound();
5527
5528 let route_cancel =
5529 route_bash_cancels
5530 .entry(route_id)
5531 .or_insert_with(|| bash::RouteBashCancel {
5532 token: PersistentCancelSignal::new(),
5533 active_waits: 0,
5534 });
5535 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5536 let cancel = bash::BashWaitCancel {
5537 connection: connection_cancel.clone(),
5538 route: route_cancel.token.clone(),
5539 };
5540
5541 bash::submit_deferred_bash(
5542 executor,
5543 bash_deferred_tx,
5544 bash_poll_touch_tx,
5545 metrics,
5546 dispatch,
5547 identity.root.clone(),
5548 identity.project_root.clone(),
5549 identity.session.clone(),
5550 request_id,
5551 route_id,
5552 frame.header.corr,
5553 frame.header.flags,
5554 frame.header.ver,
5555 arguments,
5556 format_context,
5557 cancel,
5558 bind_trust,
5559 identity.spawn_principal.clone(),
5560 call.edit_slot_survives,
5561 None,
5562 );
5563 return Ok(());
5564 }
5565
5566 let lane = command_lane(&bare_name);
5567 let tool_call_context = ToolCallContext {
5568 project_root: identity.project_root.clone(),
5569 session_id: Some(identity.session.clone()),
5570 request_id: request_id.clone(),
5571 diagnostics_on_edit,
5572 preview: call.preview,
5573 edit_slot_survives: call.edit_slot_survives,
5574 report_registration_downgrade: true,
5575 };
5576
5577 let uses_deferred_response_seam = bare_name == "inspect"
5578 || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5579 if uses_deferred_response_seam {
5580 let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5581 let response = Response::error(
5582 &request_id,
5583 "actor_not_registered",
5584 "executor actor is not registered",
5585 );
5586 let text = crate::subc_format::format_response_with_context(
5587 &bare_name,
5588 &response,
5589 &format_context,
5590 );
5591 let result = ToolCallResult { text, response };
5592 let response_frame = build_tool_response_frame_with_limit(
5593 frame.header.ver,
5594 route_id,
5595 frame.header.corr,
5596 frame.header.flags,
5597 &result,
5598 bind_trust,
5599 tool_response_body_limit,
5600 )?;
5601 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5602 };
5603 let identity_for_run = identity.clone();
5604 let request_id_for_force = request_id.clone();
5605 let format_context_for_run = format_context.clone();
5606 let bare_name_for_run = bare_name.clone();
5607 let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5608 phase_trace.mark_executor_submitted();
5609 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5610 phase_trace.mark_job_admitted();
5611 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5612 let run = || match prepare_tool_call(
5613 &bare_name_for_run,
5614 arguments,
5615 &format_context_for_run,
5616 &tool_call_context,
5617 ctx,
5618 Some(&mut phase_trace),
5619 ) {
5620 Err(result) => {
5621 let response = result.response;
5622 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5623 text: result.text,
5624 phase_trace,
5625 });
5626 response
5627 }
5628 Ok(prepared) => {
5629 let outcome = if bare_name_for_run == "inspect" {
5630 crate::commands::inspect::handle_inspect_deferred_with_restriction(
5631 &prepared.request,
5632 Arc::clone(&deferred_ctx),
5633 matches!(bind_trust, BindTrust::Untrusted),
5634 )
5635 } else {
5636 crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5637 &prepared.request,
5638 Arc::clone(&deferred_ctx),
5639 matches!(bind_trust, BindTrust::Untrusted),
5640 )
5641 };
5642 match outcome {
5643 DispatchOutcome::Deferred(pending) => {
5644 let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5645 pending,
5646 surface_downgraded: prepared.surface_downgraded,
5647 phase_trace,
5648 });
5649 Response::success(
5650 request_id_for_force.clone(),
5651 json!({ "response_deferred": true }),
5652 )
5653 }
5654 DispatchOutcome::Immediate(response) => {
5655 phase_trace.mark_execute_done();
5656 let finalizer = |response: &mut Response| {
5657 crate::response_finalize::finalize_response_with_bg_completions(
5658 response,
5659 ctx,
5660 &identity_for_run.session,
5661 &bare_name_for_run,
5662 bind_trust.allows_bash_observation(),
5663 );
5664 };
5665 let result = finish_tool_call_response(
5666 &bare_name_for_run,
5667 &format_context_for_run,
5668 response,
5669 prepared.surface_downgraded,
5670 Some(&finalizer),
5671 Some(&mut phase_trace),
5672 );
5673 let response = result.response;
5674 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5675 text: result.text,
5676 phase_trace,
5677 });
5678 response
5679 }
5680 }
5681 }
5682 };
5683 if matches!(bind_trust, BindTrust::Untrusted) {
5684 ctx.with_force_restrict(&request_id_for_force, run)
5685 } else {
5686 run()
5687 }
5688 })
5689 });
5690 let deferred_setup_guard =
5691 PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5692 let rx = submit_active_tool_call(
5693 executor.as_ref(),
5694 active_tool_calls,
5695 route_id,
5696 frame.header.corr,
5697 identity.root.clone(),
5698 lane,
5699 request_id.clone(),
5700 RouteDetachPolicy::CancelOnDetach,
5701 job,
5702 );
5703
5704 let completion_tx = tx.clone();
5705 let completion_shutdown = Arc::clone(shutdown);
5706 let completion_metrics = Arc::clone(metrics);
5707 let active_tool_calls = Arc::clone(active_tool_calls);
5708 let deferred_response_tx = deferred_response_tx.clone();
5709 let route = route_id;
5710 let corr = frame.header.corr;
5711 let flags = frame.header.flags;
5712 let ver = frame.header.ver;
5713 let root = identity.root.clone();
5714 let session_id = identity.session.clone();
5715 tokio::spawn(async move {
5716 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5717 let _deferred_setup = deferred_setup_guard;
5718 let response = await_executor_response(rx, request_id.clone()).await;
5719 match setup_rx.await {
5720 Ok(DeferredSetupOutcome::Deferred {
5721 pending,
5722 surface_downgraded,
5723 phase_trace,
5724 }) => {
5725 let pending = PendingSubcResponse {
5726 route,
5727 corr,
5728 flags,
5729 ver,
5730 root,
5731 session_id,
5732 bare_name,
5733 format_context,
5734 bind_trust,
5735 pending,
5736 surface_downgraded,
5737 phase_trace,
5738 };
5739 if let Err(error) = deferred_response_tx.send(pending) {
5740 if let Some(cancellation) = &error.0.pending.cancellation {
5741 cancellation.request_cancel();
5742 }
5743 finish_active_tool_call(&active_tool_calls, route, corr);
5744 }
5745 }
5746 Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5747 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5748 return;
5749 }
5750 let result = ToolCallResult { text, response };
5751 let fatal = note_fatal_panic_response(&result.response);
5752 match build_tool_response_frame_with_limit(
5753 ver,
5754 route,
5755 corr,
5756 flags,
5757 &result,
5758 bind_trust,
5759 tool_response_body_limit,
5760 ) {
5761 Ok(response_frame) => {
5762 let trace = ToolResponseWriteTrace::new(
5763 phase_trace,
5764 bare_name.clone(),
5765 identity.project_root.clone(),
5766 identity.session.clone(),
5767 route.channel,
5768 corr,
5769 );
5770 if let Err(error) = send_traced_tool_response_frame(
5771 &completion_tx,
5772 &completion_metrics,
5773 response_frame,
5774 trace,
5775 )
5776 .await
5777 {
5778 log::warn!(
5779 "subc attach: failed to queue deferred-seam setup response: {error}"
5780 );
5781 }
5782 }
5783 Err(error) => {
5784 log::error!(
5785 "subc attach: failed to build deferred-seam setup response: {error}"
5786 );
5787 }
5788 }
5789 if fatal {
5790 signal_fatal_teardown(
5791 &completion_tx,
5792 Some(route),
5793 ver,
5794 corr,
5795 &completion_shutdown,
5796 &completion_metrics,
5797 )
5798 .await;
5799 }
5800 }
5801 Err(_) => {
5802 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5803 return;
5804 }
5805 let text = crate::subc_format::format_response_with_context(
5806 &bare_name,
5807 &response,
5808 &format_context,
5809 );
5810 let result = ToolCallResult { text, response };
5811 if let Ok(response_frame) = build_tool_response_frame_with_limit(
5812 ver,
5813 route,
5814 corr,
5815 flags,
5816 &result,
5817 bind_trust,
5818 tool_response_body_limit,
5819 ) {
5820 let _ = send_reliable_writer_frame(
5821 &completion_tx,
5822 &completion_metrics,
5823 response_frame,
5824 "deferred setup failure",
5825 )
5826 .await;
5827 }
5828 }
5829 }
5830 });
5831 return Ok(());
5832 }
5833
5834 let bare_name_for_frame = bare_name.clone();
5835 let identity_for_run = identity.clone();
5836 let completion_session = identity.session.clone();
5837 let completion_root = identity.project_root.clone();
5838 let request_id_for_force = request_id.clone();
5839 let format_context_for_frame = format_context.clone();
5840 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5841 phase_trace.mark_executor_submitted();
5842 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5843 phase_trace.mark_job_admitted();
5844 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5845 let run = || {
5846 let finalizer = |response: &mut Response| {
5847 crate::response_finalize::finalize_response_with_bg_completions(
5848 response,
5849 ctx,
5850 &identity_for_run.session,
5851 &bare_name,
5852 bind_trust.allows_bash_observation(),
5853 );
5854 };
5855 match run_tool_call(
5856 &bare_name,
5857 arguments,
5858 &format_context,
5859 &tool_call_context,
5860 ctx,
5861 &dispatch,
5862 Some(&finalizer),
5863 Some(&mut phase_trace),
5864 ) {
5865 ToolCallOutcome::Unary(result) => {
5866 let response = result.response;
5867 let _ = tool_call_tx.send(ToolCallCompletion {
5868 text: result.text,
5869 phase_trace,
5870 });
5871 response
5872 }
5873 }
5874 };
5875 if matches!(bind_trust, BindTrust::Untrusted) {
5876 ctx.with_force_restrict(&request_id_for_force, run)
5877 } else {
5878 run()
5879 }
5880 })
5881 });
5882 let rx = submit_active_tool_call(
5883 executor.as_ref(),
5884 active_tool_calls,
5885 route_id,
5886 frame.header.corr,
5887 identity.root.clone(),
5888 lane,
5889 request_id.clone(),
5890 RouteDetachPolicy::RetainForReplay,
5891 job,
5892 );
5893 let completion_tx = tx.clone();
5894 let completion_shutdown = Arc::clone(shutdown);
5895 let route = route_id;
5896 let corr = frame.header.corr;
5897 let flags = frame.header.flags;
5898 let ver = frame.header.ver;
5899 let completion_metrics = Arc::clone(metrics);
5900 let active_tool_calls = Arc::clone(active_tool_calls);
5901 tokio::spawn(async move {
5902 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5903 let response = await_executor_response(rx, request_id.clone()).await;
5904 let (text, phase_trace) = match tool_call_rx.await {
5905 Ok(completion) => (completion.text, Some(completion.phase_trace)),
5906 Err(_) => (
5907 crate::subc_format::format_response_with_context(
5908 &bare_name_for_frame,
5909 &response,
5910 &format_context_for_frame,
5911 ),
5912 None,
5913 ),
5914 };
5915 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5916 return;
5917 }
5918 let result = ToolCallResult { text, response };
5919 let fatal = note_fatal_panic_response(&result.response);
5920 match build_tool_response_frame_with_limit(
5921 ver,
5922 route,
5923 corr,
5924 flags,
5925 &result,
5926 bind_trust,
5927 tool_response_body_limit,
5928 ) {
5929 Ok(response_frame) => {
5930 let send_result = if let Some(phase_trace) = phase_trace {
5931 let trace = ToolResponseWriteTrace::new(
5932 phase_trace,
5933 bare_name_for_frame,
5934 completion_root,
5935 completion_session,
5936 route.channel,
5937 corr,
5938 );
5939 send_traced_tool_response_frame(
5940 &completion_tx,
5941 &completion_metrics,
5942 response_frame,
5943 trace,
5944 )
5945 .await
5946 } else {
5947 send_reliable_writer_frame(
5948 &completion_tx,
5949 &completion_metrics,
5950 response_frame,
5951 "tool response",
5952 )
5953 .await
5954 };
5955 if let Err(error) = send_result {
5956 log::warn!("subc attach: failed to queue tool response frame: {error}");
5957 }
5958 }
5959 Err(error) => {
5960 log::error!("subc attach: failed to build tool response frame: {error}");
5961 }
5962 }
5963 if fatal {
5964 signal_fatal_teardown(
5965 &completion_tx,
5966 Some(route),
5967 ver,
5968 corr,
5969 &completion_shutdown,
5970 &completion_metrics,
5971 )
5972 .await;
5973 }
5974 });
5975 Ok(())
5976}
5977
5978fn submit_maintenance_job(
5979 executor: &Arc<Executor>,
5980 root_id: ProjectRootId,
5981 kind: MaintenanceDrainKind,
5982 bg_sessions_to_check: Vec<(String, u64)>,
5983 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5984 metrics: &Arc<DispatchPathMetrics>,
5985) {
5986 let request_id = format!(
5987 "subc-maintenance-drain-{}-{}",
5988 kind.label(),
5989 root_id.as_path().to_string_lossy()
5990 );
5991 let response_id = request_id.clone();
5992 let completion_root_id = root_id.clone();
5993 let maintenance_generation = executor
5994 .actor_context(&root_id)
5995 .map(|ctx| ctx.configure_generation())
5996 .unwrap_or(0);
5997 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5998 let lane = Lane::MaintenanceCommit;
6002 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
6003 let outcome = match kind {
6004 MaintenanceDrainKind::Watcher => {
6005 let drained = runtime_drain::drain_watcher_events_bounded(
6006 ctx,
6007 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
6008 );
6009 MaintenanceJobOutcome {
6010 empty_bg_sessions: Vec::new(),
6011 unacked_bg_keys: None,
6012 requeue_kind: drained.has_more.then_some(kind),
6013 }
6014 }
6015 MaintenanceDrainKind::Lsp => {
6016 let drained = runtime_drain::drain_lsp_events_bounded(
6017 ctx,
6018 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6019 );
6020 MaintenanceJobOutcome {
6021 empty_bg_sessions: Vec::new(),
6022 unacked_bg_keys: None,
6023 requeue_kind: drained.has_more.then_some(kind),
6024 }
6025 }
6026 MaintenanceDrainKind::ConfigureTail => {
6027 runtime_drain::drain_deferred_configure_maintenance(ctx);
6028 runtime_drain::drain_configure_warning_events(ctx);
6029 MaintenanceJobOutcome::default()
6030 }
6031 MaintenanceDrainKind::CompletionDrains => {
6032 runtime_drain::drain_search_index_events(ctx);
6033 runtime_drain::drain_callgraph_store_events(ctx);
6034 runtime_drain::drain_semantic_index_events(ctx);
6035 runtime_drain::drain_semantic_refresh_events(ctx);
6036 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6037 let empty_bg_sessions = bg_sessions_to_check
6038 .into_iter()
6039 .filter(|(session, _)| {
6040 !ctx.bash_background().has_unacked_wakes_for_session(session)
6041 })
6042 .collect();
6043 MaintenanceJobOutcome {
6044 empty_bg_sessions,
6045 unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6046 requeue_kind: None,
6047 }
6048 }
6049 };
6050 let requeued = outcome.requeue_kind.is_some();
6051 let _ = outcome_tx.send(outcome);
6052 Response::success(
6053 response_id,
6054 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6055 )
6056 });
6057 let rx = match kind {
6058 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6059 root_id,
6060 lane,
6061 request_id.clone(),
6062 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6063 job,
6064 ),
6065 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6066 root_id,
6067 lane,
6068 request_id.clone(),
6069 crate::executor::MaintenanceCoalesceKey::LspDrain,
6070 job,
6071 ),
6072 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6073 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6074 }
6075 };
6076 let completion_tx = completion_tx.clone();
6077 let completion_metrics = Arc::clone(metrics);
6078 tokio::spawn(async move {
6079 let _response_task = ResponseTaskGuard::new(&completion_metrics);
6080 let response = await_executor_response(rx, request_id).await;
6081 let outcome = outcome_rx.await.unwrap_or_default();
6082 let _ = send_counted_channel(
6083 &completion_tx,
6084 &completion_metrics.maintenance_queued,
6085 MaintenanceCompletion {
6086 root_id: completion_root_id,
6087 kind,
6088 response,
6089 empty_bg_sessions: outcome.empty_bg_sessions,
6090 unacked_bg_keys: outcome.unacked_bg_keys,
6091 requeue_kind: outcome.requeue_kind,
6092 },
6093 )
6094 .await;
6095 });
6096}
6097
6098async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6099 rx.await
6100 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6101}
6102
6103async fn deliver_resolved_subc_response(
6104 tx: &WriterSender,
6105 mut resolved: ResolvedSubcResponse,
6106 routes: &HashMap<RouteChannel, RouteIdentity>,
6107 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6108 executor: &Executor,
6109 active_tool_calls: &ActiveToolCalls,
6110 shutdown: &Arc<Notify>,
6111 metrics: &DispatchPathMetrics,
6112 tool_response_body_limit: usize,
6113) -> Result<(), SubcError> {
6114 let entry = &mut resolved.entry;
6115 finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6116 if let Some(meta) = live_roots.get_mut(&entry.root) {
6117 meta.note_activity();
6118 }
6119
6120 let Some(identity) = routes.get(&entry.route) else {
6121 log::debug!(
6122 "subc attach: dropping deferred {} response {} for unbound route {}",
6123 entry.bare_name,
6124 entry.pending.request_id,
6125 entry.route
6126 );
6127 return Ok(());
6128 };
6129 let Some(ctx) = executor.actor_context(&entry.root) else {
6130 return Ok(());
6131 };
6132 entry.phase_trace.mark_execute_done();
6133 let finalizer = |response: &mut Response| {
6134 crate::response_finalize::finalize_response_with_bg_completions(
6135 response,
6136 &ctx,
6137 &entry.session_id,
6138 &entry.bare_name,
6139 entry.bind_trust.allows_bash_observation(),
6140 );
6141 };
6142 let result = finish_tool_call_response(
6143 &entry.bare_name,
6144 &entry.format_context,
6145 resolved.response,
6146 entry.surface_downgraded,
6147 Some(&finalizer),
6148 Some(&mut entry.phase_trace),
6149 );
6150 let fatal = note_fatal_panic_response(&result.response);
6151 let response_frame = build_tool_response_frame_with_limit(
6152 entry.ver,
6153 entry.route,
6154 entry.corr,
6155 entry.flags,
6156 &result,
6157 identity.trust,
6158 tool_response_body_limit,
6159 )?;
6160 let trace = ToolResponseWriteTrace::new(
6161 std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6162 entry.bare_name.clone(),
6163 identity.project_root.clone(),
6164 entry.session_id.clone(),
6165 entry.route.channel,
6166 entry.corr,
6167 );
6168 send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6169 if fatal {
6170 signal_fatal_teardown(
6171 tx,
6172 Some(entry.route),
6173 entry.ver,
6174 entry.corr,
6175 shutdown,
6176 metrics,
6177 )
6178 .await;
6179 }
6180 Ok(())
6181}
6182
6183async fn signal_fatal_teardown(
6184 tx: &WriterSender,
6185 route: Option<RouteChannel>,
6186 ver: u8,
6187 corr: u64,
6188 shutdown: &Arc<Notify>,
6189 metrics: &DispatchPathMetrics,
6190) {
6191 if let Some(route) = route {
6192 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6193 if let Err(error) = send_frame(tx, metrics, frame).await {
6194 log::warn!(
6195 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6196 );
6197 }
6198 }
6199 }
6200 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6201 if let Err(error) = send_frame(tx, metrics, frame).await {
6202 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6203 }
6204 }
6205 shutdown.notify_one();
6206}
6207#[derive(Debug, Deserialize)]
6208#[serde(untagged)]
6209enum RouteRequest {
6210 BgEvents(BgEventsRequest),
6211 ToolCall(ToolCallRequest),
6212}
6213
6214#[derive(Debug, Deserialize)]
6215struct BgEventsRequest {
6216 op: BgEventsOp,
6217}
6218
6219#[derive(Debug, Deserialize)]
6220#[serde(rename_all = "snake_case")]
6221enum BgEventsOp {
6222 BgEvents,
6223}
6224
6225#[derive(Debug, Deserialize)]
6226struct ToolCallRequest {
6227 name: String,
6228 #[serde(default)]
6229 arguments: Value,
6230 #[serde(default)]
6232 edit_slot_survives: Option<bool>,
6233 #[serde(default)]
6238 preview: bool,
6239}
6240
6241#[cfg(test)]
6242pub(crate) mod test_support {
6243 use super::*;
6244 use crate::bash_background::BgTaskStatus;
6245 use crate::protocol::{
6246 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6247 ProgressFrame, StatusChangedFrame,
6248 };
6249 use serde_json::json;
6250
6251 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6252 let dir = tempfile::Builder::new()
6253 .prefix(name)
6254 .tempdir()
6255 .expect("temp root");
6256 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6257 (dir, root)
6258 }
6259
6260 pub(super) fn test_ctx() -> Arc<AppContext> {
6261 Arc::new(AppContext::new(
6262 Box::new(crate::parser::TreeSitterProvider::new()),
6263 crate::config::Config::default(),
6264 ))
6265 }
6266
6267 fn inspect_context(root: &Path) -> Arc<AppContext> {
6268 inspect_context_with_timeout(root, None)
6269 }
6270
6271 fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6272 let mut config = crate::config::Config::default();
6273 config.project_root = Some(root.to_path_buf());
6274 if let Some(timeout_ms) = timeout_ms {
6275 config.inspect.diagnostics_timeout_ms = timeout_ms;
6276 }
6277 let ctx = Arc::new(AppContext::new(
6278 Box::new(crate::parser::TreeSitterProvider::new()),
6279 config,
6280 ));
6281 ctx.set_harness(crate::harness::Harness::Opencode);
6282 ctx
6283 }
6284
6285 fn inspect_request(id: &str) -> RawRequest {
6286 serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6287 }
6288
6289 fn submit_deferred_inspect_setup(
6290 executor: &Arc<Executor>,
6291 root: &ProjectRootId,
6292 ctx: &Arc<AppContext>,
6293 request_id: &str,
6294 ) -> (PendingResponse, JobCancellation) {
6295 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6296 let request = inspect_request(request_id);
6297 let inspect_ctx = Arc::clone(ctx);
6298 let (_rx, cancellation) = executor.submit_cancellable_async(
6299 root.clone(),
6300 Lane::SerialLspStatus,
6301 request_id.to_string(),
6302 Box::new(move |_| {
6303 let DispatchOutcome::Deferred(pending) =
6304 crate::commands::inspect::handle_inspect_deferred_with_restriction(
6305 &request,
6306 inspect_ctx,
6307 true,
6308 )
6309 else {
6310 panic!("inspect setup must defer")
6311 };
6312 pending_tx.send(pending).expect("send pending inspect");
6313 Response::success("inspect-setup", json!({}))
6314 }),
6315 );
6316 let pending = pending_rx
6317 .recv_timeout(Duration::from_secs(1))
6318 .expect("inspect setup leaves the executor lane");
6319 let deadline = Instant::now() + Duration::from_secs(1);
6320 while !executor.actor_is_idle(root) {
6321 assert!(
6322 Instant::now() < deadline,
6323 "inspect setup kept lane counters live"
6324 );
6325 std::thread::sleep(Duration::from_millis(5));
6326 }
6327 (pending, cancellation)
6328 }
6329
6330 fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6331 let deadline = Instant::now() + Duration::from_secs(60);
6332 loop {
6333 if let Some(response) = (pending.poll)(ctx) {
6334 return response;
6335 }
6336 assert!(Instant::now() < deadline, "inspect terminal timed out");
6337 std::thread::sleep(Duration::from_millis(5));
6338 }
6339 }
6340
6341 fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6342 let source_dir = root.join("src");
6343 std::fs::create_dir_all(&source_dir).expect("create source dir");
6344 std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6345 .expect("write Cargo manifest");
6346 let source = source_dir.join("main.rs");
6347 std::fs::write(&source, "fn main() {}\n").expect("write source");
6348 let binary = root.join("cold-navigation-server");
6349 std::fs::write(&binary, b"fixture").expect("write server placeholder");
6350
6351 let ctx = inspect_context(root);
6352 ctx.lsp()
6353 .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6354 (ctx, source)
6355 }
6356
6357 fn navigation_request(id: &str, source: &Path) -> RawRequest {
6358 serde_json::from_value(json!({
6359 "id": id,
6360 "command": "lsp_hover",
6361 "file": source.display().to_string(),
6362 "line": 1,
6363 "character": 1,
6364 }))
6365 .expect("navigation request")
6366 }
6367
6368 fn submit_deferred_navigation_setup(
6369 executor: &Arc<Executor>,
6370 root: &ProjectRootId,
6371 ctx: &Arc<AppContext>,
6372 source: &Path,
6373 request_id: &str,
6374 ) -> (PendingResponse, JobCancellation) {
6375 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6376 let request = navigation_request(request_id, source);
6377 let navigation_ctx = Arc::clone(ctx);
6378 let (_rx, cancellation) = executor.submit_cancellable_async(
6379 root.clone(),
6380 Lane::SerialLspStatus,
6381 request_id.to_string(),
6382 Box::new(move |_| {
6383 let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6384 handle_lsp_navigation_deferred_with_restriction(
6385 &request,
6386 navigation_ctx,
6387 true,
6388 )
6389 else {
6390 panic!("cold navigation setup must defer")
6391 };
6392 pending_tx.send(pending).expect("send pending navigation");
6393 Response::success("navigation-setup", json!({}))
6394 }),
6395 );
6396 let pending = pending_rx
6397 .recv_timeout(Duration::from_secs(1))
6398 .expect("navigation setup leaves the executor lane");
6399 let deadline = Instant::now() + Duration::from_secs(1);
6400 while !executor.actor_is_idle(root) {
6401 assert!(
6402 Instant::now() < deadline,
6403 "navigation setup kept lane counters live"
6404 );
6405 std::thread::sleep(Duration::from_millis(5));
6406 }
6407 (pending, cancellation)
6408 }
6409
6410 fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6411 let deadline = Instant::now() + Duration::from_secs(1);
6412 loop {
6413 if let Some(response) = (pending.poll)(ctx) {
6414 return response;
6415 }
6416 assert!(Instant::now() < deadline, "navigation terminal timed out");
6417 std::thread::sleep(Duration::from_millis(5));
6418 }
6419 }
6420
6421 #[test]
6422 fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6423 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6424 let executor = Arc::new(Executor::new());
6425 let (dir, root) = test_root("deferred-inspect-storm");
6426 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6427 let ctx = inspect_context(dir.path());
6428 executor.register_actor(root.clone(), Arc::clone(&ctx));
6429 let (started_rx, release_tx) =
6430 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6431 let (mut pending, _cancellation) =
6432 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6433 started_rx
6434 .recv_timeout(Duration::from_secs(1))
6435 .expect("deferred inspect body starts");
6436
6437 for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6438 let response = executor.submit(
6439 root.clone(),
6440 Lane::Mutating,
6441 request_id.to_string(),
6442 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6443 );
6444 assert!(
6445 response
6446 .recv_timeout(Duration::from_secs(1))
6447 .expect("writer admits while inspect remains deferred")
6448 .success
6449 );
6450 }
6451 assert_eq!(
6452 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6453 1,
6454 "writer admissions must not finish the detached inspect"
6455 );
6456
6457 release_tx.send(()).expect("release inspect body");
6458 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6459 assert!(
6460 terminal.data.get("inspect_terminal").is_some(),
6461 "inspect must still produce its terminal: {:?}",
6462 terminal.data
6463 );
6464 }
6465
6466 #[test]
6467 fn deferred_inspect_honors_the_shared_request_deadline() {
6468 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6469 let executor = Arc::new(Executor::new());
6470 let (dir, root) = test_root("deferred-inspect-deadline");
6471 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6472 let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6473 executor.register_actor(root.clone(), Arc::clone(&ctx));
6474 let (started_rx, _release_tx) =
6475 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6476 let started = Instant::now();
6477 let (mut pending, _cancellation) =
6478 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6479 started_rx
6480 .recv_timeout(Duration::from_secs(1))
6481 .expect("deferred inspect body starts");
6482
6483 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6484 assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6485 assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6486 assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6487 assert!(
6488 started.elapsed() < Duration::from_secs(8),
6489 "deferred inspect missed its terminal reserve: {:?}",
6490 started.elapsed()
6491 );
6492 }
6493
6494 #[test]
6495 fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6496 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6497 let executor = Arc::new(Executor::new());
6498 let (dir, root) = test_root("deferred-cold-navigation");
6499 let (ctx, source) = cold_navigation_context(dir.path());
6500 executor.register_actor(root.clone(), Arc::clone(&ctx));
6501 let (started_rx, _release_tx) =
6502 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6503 let (mut pending, cancellation) = submit_deferred_navigation_setup(
6504 &executor,
6505 &root,
6506 &ctx,
6507 &source,
6508 "subc-cold-navigation",
6509 );
6510 started_rx
6511 .recv_timeout(Duration::from_secs(1))
6512 .expect("detached navigation reaches its cold-start gate");
6513
6514 for (lane, request_id) in [
6515 (Lane::PureRead, "subc-navigation-read"),
6516 (Lane::Mutating, "subc-navigation-write"),
6517 ] {
6518 let response = executor.submit(
6519 root.clone(),
6520 lane,
6521 request_id.to_string(),
6522 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6523 );
6524 assert!(
6525 response
6526 .recv_timeout(Duration::from_secs(1))
6527 .expect("unrelated work admits while navigation remains deferred")
6528 .success
6529 );
6530 }
6531 assert_eq!(
6532 crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6533 1,
6534 "lane admissions must not finish the detached navigation"
6535 );
6536
6537 cancellation.request_cancel();
6538 let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6539 assert!(!terminal.success);
6540 let deadline = Instant::now() + Duration::from_secs(1);
6541 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6542 assert!(
6543 Instant::now() < deadline,
6544 "cancelled navigation worker did not settle"
6545 );
6546 std::thread::sleep(Duration::from_millis(5));
6547 }
6548 assert!(executor.actor_is_idle(&root));
6549 }
6550
6551 #[test]
6552 fn cancelling_pending_navigation_removes_entry_without_reply() {
6553 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6554 let executor = Arc::new(Executor::new());
6555 let (dir, root) = test_root("cancelled-pending-navigation");
6556 let (ctx, source) = cold_navigation_context(dir.path());
6557 executor.register_actor(root.clone(), Arc::clone(&ctx));
6558 let (started_rx, _release_tx) =
6559 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6560 let (pending, _cancellation) = submit_deferred_navigation_setup(
6561 &executor,
6562 &root,
6563 &ctx,
6564 &source,
6565 "subc-cancel-navigation",
6566 );
6567 started_rx
6568 .recv_timeout(Duration::from_secs(1))
6569 .expect("detached navigation reaches its cancellation gate");
6570
6571 let route = RouteChannel {
6572 channel: 17,
6573 epoch: 1,
6574 };
6575 let mut registry = PendingSubcResponses::default();
6576 registry.register(PendingSubcResponse {
6577 route,
6578 corr: 71,
6579 flags: Flags::new(false, Priority::Passive, false),
6580 ver: PROTOCOL_VERSION,
6581 root: root.clone(),
6582 session_id: "navigation-cancel-session".to_string(),
6583 bare_name: "lsp_hover".to_string(),
6584 format_context: crate::subc_format::FormatContext::from_tool_call(
6585 "lsp_hover",
6586 &json!({}),
6587 dir.path(),
6588 ),
6589 bind_trust: BindTrust::FirstParty,
6590 pending,
6591 surface_downgraded: false,
6592 phase_trace: PhaseTrace::new(Instant::now()),
6593 });
6594
6595 assert!(registry.cancel_request(route, 71));
6596 assert!(registry.is_empty());
6597 let deadline = Instant::now() + Duration::from_secs(1);
6598 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6599 assert!(
6600 Instant::now() < deadline,
6601 "pending cancellation did not settle the detached worker"
6602 );
6603 std::thread::sleep(Duration::from_millis(5));
6604 }
6605 assert!(
6606 registry.poll_ready(executor.as_ref()).is_empty(),
6607 "a cancelled navigation must not leak a reply"
6608 );
6609 }
6610
6611 #[test]
6612 fn same_root_deferred_inspects_are_single_flight() {
6613 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6614 let executor = Arc::new(Executor::new());
6615 let (dir, root) = test_root("single-flight-deferred-inspect");
6616 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6617 let ctx = inspect_context(dir.path());
6618 executor.register_actor(root.clone(), Arc::clone(&ctx));
6619 let (started_rx, release_tx) =
6620 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6621 let (mut first, _first_cancellation) =
6622 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6623 started_rx
6624 .recv_timeout(Duration::from_secs(1))
6625 .expect("first inspect owns the root flight");
6626 let (mut second, second_cancellation) =
6627 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6628
6629 assert_eq!(
6630 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6631 1,
6632 "only one detached body may run for a root"
6633 );
6634 assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6635 second_cancellation.request_cancel();
6636 let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6637 assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6638 assert_eq!(
6639 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6640 1,
6641 "cancelling the queued request must not release the active flight"
6642 );
6643
6644 release_tx.send(()).expect("release first inspect");
6645 let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6646 assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6647 assert_eq!(
6648 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6649 0
6650 );
6651 }
6652
6653 #[test]
6654 fn route_abandonment_cancels_detached_inspect_thread() {
6655 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6656 let executor = Arc::new(Executor::new());
6657 let (dir, root) = test_root("cancelled-deferred-inspect");
6658 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6659 let ctx = inspect_context(dir.path());
6660 executor.register_actor(root.clone(), Arc::clone(&ctx));
6661 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6662 let route = RouteChannel {
6663 channel: 7,
6664 epoch: 1,
6665 };
6666 let (started_rx, _release_tx) =
6667 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6668 let (mut pending, cancellation) =
6669 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6670 active.lock().expect("active tool call map").insert(
6671 (route, 41),
6672 ActiveToolCall {
6673 root_id: root.clone(),
6674 cancellation,
6675 detach_policy: RouteDetachPolicy::CancelOnDetach,
6676 },
6677 );
6678 started_rx
6679 .recv_timeout(Duration::from_secs(1))
6680 .expect("detached inspect reaches cancellation gate");
6681 assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6682
6683 assert!(cancel_active_tool_call(
6684 &active,
6685 executor.as_ref(),
6686 route,
6687 41,
6688 "test route abandonment"
6689 ));
6690 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6691 assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6692 assert_eq!(
6693 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6694 0
6695 );
6696 assert!(executor.actor_is_idle(&root));
6697 assert!(active.lock().expect("active tool call map").is_empty());
6698 let restriction_deadline = Instant::now() + Duration::from_secs(1);
6699 while ctx.request_force_restrict("subc-inspect-abandoned") {
6700 assert!(
6701 Instant::now() < restriction_deadline,
6702 "detached force-restrict guard leaked"
6703 );
6704 std::thread::sleep(Duration::from_millis(5));
6705 }
6706 }
6707
6708 #[test]
6709 fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6710 let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6715 pool_size: 3,
6716 read_cap: 2,
6717 actor_cap: 2,
6718 heavy_permits: 1,
6719 drr_quantum: 1,
6720 }));
6721 let (_dir, root) = test_root("cancelled-interactive-search");
6722 executor.register_actor(root.clone(), test_ctx());
6723 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6724 let route = RouteChannel {
6725 channel: 9,
6726 epoch: 1,
6727 };
6728
6729 let disabled_iterations = Arc::new(AtomicUsize::new(0));
6730 let disabled_probe = Arc::clone(&disabled_iterations);
6731 let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6732 let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6733 root.clone(),
6734 Lane::PureRead,
6735 "untracked-search".to_string(),
6736 Box::new(move |_| {
6737 disabled_started_tx
6738 .send(())
6739 .expect("signal untracked search");
6740 let deadline = Instant::now() + Duration::from_secs(5);
6741 while !crate::commands::semantic_search::search_cancellation_requested() {
6742 if Instant::now() >= deadline {
6743 return Response::error(
6744 "untracked-search",
6745 "test_timeout",
6746 "untracked search did not receive cancellation",
6747 );
6748 }
6749 disabled_probe.fetch_add(1, Ordering::Relaxed);
6750 std::thread::yield_now();
6751 }
6752 Response::error(
6753 "untracked-search",
6754 "request_cancelled",
6755 "cancelled at search checkpoint",
6756 )
6757 }),
6758 );
6759 disabled_started_rx
6760 .recv_timeout(Duration::from_secs(1))
6761 .expect("untracked search starts");
6762 assert_eq!(
6763 apply_route_work_disposition(
6764 &active,
6765 executor.as_ref(),
6766 route,
6767 RouteWorkDisposition::Abandon,
6768 "disabled cancellation wiring",
6769 ),
6770 0
6771 );
6772 let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6773 std::thread::sleep(Duration::from_millis(10));
6774 assert!(
6775 disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6776 "without route registration the search keeps computing"
6777 );
6778 disabled_cancellation.request_cancel();
6779 let disabled_response = disabled_rx
6780 .blocking_recv()
6781 .expect("untracked search settles after explicit cleanup");
6782 assert_eq!(disabled_response.data["code"], "request_cancelled");
6783
6784 let tracked_iterations = Arc::new(AtomicUsize::new(0));
6785 let tracked_probe = Arc::clone(&tracked_iterations);
6786 let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6787 let tracked_rx = submit_active_tool_call(
6788 executor.as_ref(),
6789 &active,
6790 route,
6791 42,
6792 root.clone(),
6793 Lane::PureRead,
6794 "tracked-search".to_string(),
6795 RouteDetachPolicy::RetainForReplay,
6796 Box::new(move |_| {
6797 tracked_started_tx.send(()).expect("signal tracked search");
6798 let deadline = Instant::now() + Duration::from_secs(5);
6799 while !crate::commands::semantic_search::search_cancellation_requested() {
6800 if Instant::now() >= deadline {
6801 return Response::error(
6802 "tracked-search",
6803 "test_timeout",
6804 "tracked search did not receive cancellation",
6805 );
6806 }
6807 tracked_probe.fetch_add(1, Ordering::Relaxed);
6808 std::thread::yield_now();
6809 }
6810 Response::error(
6811 "tracked-search",
6812 "request_cancelled",
6813 "cancelled at search checkpoint",
6814 )
6815 }),
6816 );
6817 tracked_started_rx
6818 .recv_timeout(Duration::from_secs(1))
6819 .expect("tracked search starts");
6820
6821 let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6822 let terminal_rx = submit_active_tool_call(
6823 executor.as_ref(),
6824 &active,
6825 route,
6826 43,
6827 root.clone(),
6828 Lane::PureRead,
6829 "teardown-terminal".to_string(),
6830 RouteDetachPolicy::CancelOnDetach,
6831 Box::new(move |_| {
6832 terminal_started_tx
6833 .send(())
6834 .expect("signal teardown-terminal call");
6835 let deadline = Instant::now() + Duration::from_secs(5);
6836 while !crate::executor::current_job_cancelled() {
6837 if Instant::now() >= deadline {
6838 return Response::error(
6839 "teardown-terminal",
6840 "test_timeout",
6841 "terminal call did not receive cancellation",
6842 );
6843 }
6844 std::thread::yield_now();
6845 }
6846 Response::error(
6847 "teardown-terminal",
6848 "request_cancelled",
6849 "cancelled for terminal-emitting teardown",
6850 )
6851 }),
6852 );
6853 terminal_started_rx
6854 .recv_timeout(Duration::from_secs(1))
6855 .expect("teardown-terminal call starts");
6856 assert_eq!(
6857 apply_route_work_disposition(
6858 &active,
6859 executor.as_ref(),
6860 route,
6861 RouteWorkDisposition::RetainForReplay,
6862 "test route detach",
6863 ),
6864 1,
6865 "only the replayable search remains active after route detach"
6866 );
6867 let terminal_response = terminal_rx
6868 .blocking_recv()
6869 .expect("teardown-terminal call stops at cancellation checkpoint");
6870 assert_eq!(terminal_response.data["code"], "request_cancelled");
6871 let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6872 std::thread::sleep(Duration::from_millis(10));
6873 assert!(
6874 tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6875 "route detach must retain work whose response can replay after rebind"
6876 );
6877 assert_eq!(
6878 apply_route_work_disposition(
6879 &active,
6880 executor.as_ref(),
6881 route,
6882 RouteWorkDisposition::Abandon,
6883 "test session purge",
6884 ),
6885 1
6886 );
6887 let tracked_response = tracked_rx
6888 .blocking_recv()
6889 .expect("tracked search stops at cancellation checkpoint");
6890 assert_eq!(tracked_response.data["code"], "request_cancelled");
6891 assert!(active.lock().expect("active tool calls").is_empty());
6892
6893 let deadline = Instant::now() + Duration::from_secs(1);
6894 while !executor.actor_is_idle(&root) {
6895 assert!(
6896 Instant::now() < deadline,
6897 "cancelled search must release the PureRead lane"
6898 );
6899 std::thread::sleep(Duration::from_millis(2));
6900 }
6901 }
6902
6903 #[test]
6904 fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6905 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6906 let executor = Arc::new(Executor::new());
6907 let (dir, root) = test_root("shutdown-deferred-inspect");
6908 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6909 let ctx = inspect_context(dir.path());
6910 executor.register_actor(root.clone(), Arc::clone(&ctx));
6911 let (started_rx, _release_tx) =
6912 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6913 let (pending, cancellation) =
6914 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6915 let route = RouteChannel {
6916 channel: 8,
6917 epoch: 1,
6918 };
6919 started_rx
6920 .recv_timeout(Duration::from_secs(1))
6921 .expect("detached inspect reaches shutdown gate");
6922 let mut registry = PendingSubcResponses::default();
6923 registry.register(PendingSubcResponse {
6924 route,
6925 corr: 42,
6926 flags: Flags::new(false, Priority::Passive, false),
6927 ver: PROTOCOL_VERSION,
6928 root: root.clone(),
6929 session_id: "shutdown-session".to_string(),
6930 bare_name: "inspect".to_string(),
6931 format_context: crate::subc_format::FormatContext::from_tool_call(
6932 "inspect",
6933 &json!({}),
6934 dir.path(),
6935 ),
6936 bind_trust: BindTrust::FirstParty,
6937 pending,
6938 surface_downgraded: false,
6939 phase_trace: PhaseTrace::new(Instant::now()),
6940 });
6941 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6942 (route, 42),
6943 ActiveToolCall {
6944 root_id: root.clone(),
6945 cancellation,
6946 detach_policy: RouteDetachPolicy::CancelOnDetach,
6947 },
6948 )])));
6949
6950 let resolved = registry.drain_on_shutdown(executor.as_ref());
6951 assert!(registry.is_empty());
6952 assert_eq!(resolved.len(), 1);
6953 assert_eq!(
6954 resolved[0].response.data["failure_reason"],
6955 "daemon_shutdown"
6956 );
6957 finish_active_tool_call(&active, route, 42);
6958 let deadline = Instant::now() + Duration::from_secs(1);
6959 while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6960 assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6961 std::thread::sleep(Duration::from_millis(5));
6962 }
6963 assert!(active.lock().expect("active calls").is_empty());
6964 assert!(executor.actor_is_idle(&root));
6965 }
6966
6967 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6968 let deadline = Instant::now() + Duration::from_secs(30);
6969 loop {
6970 let observed = ctx.watcher_registry_count();
6971 if observed == expected {
6972 return;
6973 }
6974 assert!(
6975 Instant::now() < deadline,
6976 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6977 );
6978 std::thread::sleep(Duration::from_millis(50));
6979 }
6980 }
6981
6982 pub(super) fn reap_until_forgotten(
6992 root: &ProjectRootId,
6993 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6994 pending_binds: &HashMap<RouteChannel, PendingBind>,
6995 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6996 executor: &Arc<Executor>,
6997 metrics: &DispatchPathMetrics,
6998 ) -> IdleReapOutcome {
6999 let deadline = Instant::now() + Duration::from_secs(30);
7000 loop {
7001 let outcome = reap_idle_roots(
7002 Instant::now(),
7003 live_roots,
7004 pending_binds,
7005 root_channels,
7006 executor,
7007 metrics,
7008 );
7009 if outcome.forgotten_deleted_roots.contains(root) {
7010 return outcome;
7011 }
7012 assert!(
7013 Instant::now() < deadline,
7014 "deleted root was never forgotten: {root:?}"
7015 );
7016 std::thread::sleep(Duration::from_millis(10));
7017 }
7018 }
7019
7020 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7021 let deadline = Instant::now() + Duration::from_secs(30);
7022 loop {
7023 let observed = app.actor_root_count();
7024 if observed == expected {
7025 return;
7026 }
7027 assert!(
7028 Instant::now() < deadline,
7029 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7030 );
7031 std::thread::sleep(Duration::from_millis(50));
7032 }
7033 }
7034
7035 pub(super) fn status_frame(seq: u64) -> PushFrame {
7036 status_frame_with_session(seq, None)
7037 }
7038
7039 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7040 PushFrame::StatusChanged(StatusChangedFrame {
7041 frame_type: "status_changed",
7042 session_id: session_id.map(str::to_string),
7043 snapshot: json!({ "seq": seq }),
7044 })
7045 }
7046
7047 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7048 completion_frame_with_session(task_id, "session-1")
7049 }
7050
7051 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7052 PushFrame::BashCompleted(BashCompletedFrame {
7053 frame_type: "bash_completed",
7054 task_id: task_id.to_string(),
7055 session_id: session_id.to_string(),
7056 status: BgTaskStatus::Completed,
7057 exit_code: Some(0),
7058 command: format!("echo {task_id}"),
7059 output_preview: String::new(),
7060 bash_output_list_envelope: None,
7061 output_truncated: false,
7062 original_tokens: None,
7063 compressed_tokens: None,
7064 tokens_skipped: false,
7065 status_reason: None,
7066 live_descendants: Some(Vec::new()),
7067 live_descendants_omitted: 0,
7068 live_descendants_summary: None,
7069 })
7070 }
7071
7072 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7073 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7074 }
7075
7076 pub(super) fn long_running_frame_with_session(
7077 task_id: &str,
7078 session_id: &str,
7079 elapsed_ms: u64,
7080 ) -> PushFrame {
7081 PushFrame::BashLongRunning(BashLongRunningFrame {
7082 frame_type: "bash_long_running",
7083 task_id: task_id.to_string(),
7084 session_id: session_id.to_string(),
7085 command: format!("sleep {elapsed_ms}"),
7086 elapsed_ms,
7087 })
7088 }
7089
7090 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7091 PushFrame::BashPatternMatch(BashPatternMatchFrame {
7092 frame_type: "bash_pattern_match",
7093 task_id: "task-pattern".to_string(),
7094 session_id: session_id.to_string(),
7095 watch_id: "watch-1".to_string(),
7096 match_text: "needle".to_string(),
7097 match_offset: 7,
7098 context: "haystack needle".to_string(),
7099 once: true,
7100 reason: "pattern_match",
7101 })
7102 }
7103
7104 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7105 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7106 frame_type: "configure_warnings",
7107 session_id: session_id.map(str::to_string),
7108 project_root: "/tmp/subc-test".to_string(),
7109 warnings: Vec::new(),
7110 })
7111 }
7112
7113 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7114 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7115 }
7116
7117 pub(super) fn route_identity_with_trust(
7118 root: &ProjectRootId,
7119 session_id: &str,
7120 trust: BindTrust,
7121 ) -> RouteIdentity {
7122 RouteIdentity(Arc::new(RouteIdentityData {
7123 root: root.clone(),
7124 project_root: root.as_path().to_path_buf(),
7125 harness: "opencode".to_string(),
7126 session: session_id.to_string(),
7127 trust,
7128 spawn_principal: AuthenticatedPrincipal::RouteBind {
7129 trust: trust.sandbox_trust(),
7130 route_channel: 0,
7131 route_epoch: 0,
7132 project_root: root.as_path().to_path_buf(),
7133 harness: "opencode".to_string(),
7134 session_id: session_id.to_string(),
7135 principal_id: Some(match trust {
7136 BindTrust::FirstParty => "direct".to_string(),
7137 BindTrust::Untrusted => "unverified".to_string(),
7138 }),
7139 },
7140 consumer_elicitation_capable: false,
7141 }))
7142 }
7143
7144 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7145 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7146 }
7147
7148 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7149 match frame {
7150 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7151 _ => None,
7152 }
7153 }
7154
7155 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7156 match frame {
7157 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7158 _ => None,
7159 }
7160 }
7161
7162 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7163 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7164 body.get("task_id")
7165 .and_then(serde_json::Value::as_str)
7166 .map(str::to_string)
7167 }
7168}
7169
7170#[cfg(test)]
7171mod tests {
7172 use super::test_support::{
7173 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7174 wait_for_actor_root_count, wait_for_watcher_count,
7175 };
7176 use super::*;
7177 use crate::bash_background::BgTaskStatus;
7178
7179 #[test]
7183 fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7184 assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7185 assert!(matches!(
7186 module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7187 Err(SubcError::ConnectionLost)
7188 ));
7189 assert!(matches!(
7190 module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7191 Err(SubcError::ActorFatal)
7192 ));
7193 }
7194
7195 #[test]
7196 fn fatal_panic_responses_are_detected_and_noted() {
7197 let panic = Response::error(
7198 "req-fatal",
7199 "actor_fatal",
7200 "start byte index 7 is not a char boundary",
7201 );
7202 assert!(note_fatal_panic_response(&panic));
7203 let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7204 assert!(!note_fatal_panic_response(&ordinary));
7205 }
7206
7207 fn attach_error(kind: io::ErrorKind) -> SubcError {
7208 SubcError::Connect {
7209 endpoint: "127.0.0.1:1".to_string(),
7210 source: io::Error::new(kind, "constructed attach failure"),
7211 }
7212 }
7213
7214 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7215 SubcError::Auth {
7216 endpoint: "127.0.0.1:1".to_string(),
7217 source: subc_transport::AuthError::Io {
7218 stage: subc_transport::AuthStage::ServerProof,
7219 source: io::Error::new(kind, "constructed auth failure"),
7220 },
7221 }
7222 }
7223
7224 fn cpu_hunt_process_cpu_us() -> u64 {
7225 #[cfg(unix)]
7226 {
7227 let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7228 if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7230 let usage = unsafe { usage.assume_init() };
7231 return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7232 + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7233 }
7234 }
7235 0
7236 }
7237
7238 #[test]
7239 fn route_bind_does_not_recompute_fleet_health() {
7240 let runtime = tokio::runtime::Builder::new_current_thread()
7241 .enable_all()
7242 .build()
7243 .expect("bind runtime");
7244 runtime.block_on(async {
7245 let (dir, root) = test_root("bind-health-work-count");
7246 let app = App::default_shared();
7247 let executor = Arc::new(Executor::new());
7248 let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7249 let mut fixture_dirs = Vec::new();
7250 if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7253 let copy = std::fs::canonicalize(copy).expect("copied store exists");
7254 let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7255 .parent()
7256 .expect("crates directory")
7257 .parent()
7258 .expect("checkout directory")
7259 .canonicalize()
7260 .expect("canonical checkout");
7261 assert!(copy.starts_with(project.join("target")));
7262 for index in 0..36 {
7263 let actor = if index == 0 {
7264 Arc::clone(&ctx)
7265 } else {
7266 Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7267 };
7268 actor.update_config(|config| config.project_root = Some(project.clone()));
7269 *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7270 crate::callgraph_store::CallGraphStore::open_readonly(
7271 copy.clone(),
7272 project.clone(),
7273 )
7274 .expect("open copied graph")
7275 .expect("copied graph ready"),
7276 ));
7277 if index > 0 {
7278 let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7279 assert!(executor.register_actor(id, actor));
7280 fixture_dirs.push(fixture);
7281 }
7282 }
7283 }
7284 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7285 let cache = HealthRollupCache::new();
7286 if !fixture_dirs.is_empty() {
7287 let started = Instant::now();
7288 let cpu = cpu_hunt_process_cpu_us();
7289 cache.refresh(&executor, &app);
7290 eprintln!(
7291 "cpu_hunt health_rollup wall_us={} cpu_us={}",
7292 started.elapsed().as_micros(),
7293 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7294 );
7295 let started = Instant::now();
7296 let cpu = cpu_hunt_process_cpu_us();
7297 std::hint::black_box(ctx.build_status_snapshot());
7298 eprintln!(
7299 "cpu_hunt status wall_us={} cpu_us={}",
7300 started.elapsed().as_micros(),
7301 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7302 );
7303 }
7304 let refreshes_before = cache.refresh_count_for_test();
7305 let metrics = Arc::new(DispatchPathMetrics::new());
7306 let (writer_tx, _writer_rx) = mpsc::channel(8);
7307 let (completion_tx, mut completion_rx) = mpsc::channel(8);
7308 let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7309 let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7310 let senders = PushSenders {
7311 lossy_tx,
7312 reliable_tx,
7313 lossy_overflow: Arc::new(push::LossyOverflow::default()),
7314 lossy_seq: Arc::new(AtomicU64::new(0)),
7315 fleet_status_client: FleetStatusClient::channel(1).0,
7316 };
7317 let request = ModuleControlRequest::RouteBind {
7318 route_channel: 1,
7319 epoch: 1,
7320 target: RouteTarget::ToolProvider {
7321 module_id: "aft".to_string(),
7322 },
7323 identity: subc_protocol::BindIdentity {
7324 project_root: root.as_path().to_path_buf(),
7325 harness: "opencode".to_string(),
7326 session: "bind-health-work-count".to_string(),
7327 },
7328 principal: Some(subc_protocol::Principal::Direct),
7329 consumer_capabilities: None,
7330 admission_facts: Default::default(),
7331 };
7332 let frame = Frame::build_with_version(
7333 PROTOCOL_VERSION,
7334 FrameType::Request,
7335 control_flags(),
7336 0,
7337 0,
7338 1,
7339 serde_json::to_vec(&request).expect("bind body"),
7340 )
7341 .expect("bind frame");
7342 let mut pending_binds = HashMap::new();
7343 let started = Instant::now();
7344 let cpu = cpu_hunt_process_cpu_us();
7345 handle_control_request(
7346 &writer_tx,
7347 &frame,
7348 &app,
7349 &executor,
7350 &mut HashMap::new(),
7351 &mut pending_binds,
7352 &mut HashMap::new(),
7353 &mut HashMap::new(),
7354 &mut HashSet::new(),
7355 &mut HashMap::new(),
7356 &mut HashMap::new(),
7357 &mut HashMap::new(),
7358 &mut HashMap::new(),
7359 &mut HashMap::new(),
7360 &mut HashMap::new(),
7361 &Arc::new(StdMutex::new(HashMap::new())),
7362 &mut PendingSubcResponses::default(),
7363 &mut RetryBuffer::new(),
7364 &mut HashMap::new(),
7365 &Arc::new(Notify::new()),
7366 &completion_tx,
7367 &metrics,
7368 None,
7369 &cache,
7370 &senders,
7371 |request, _| Response::success(request.id, json!({})),
7372 Some(&dir.path().join("absent-user-config.json")),
7373 usize::MAX,
7374 )
7375 .await
7376 .expect("admit route bind");
7377 eprintln!(
7378 "route_bind_health admission_us={} cpu_us={} refreshes={}",
7379 started.elapsed().as_micros(),
7380 cpu_hunt_process_cpu_us().saturating_sub(cpu),
7381 cache.refresh_count_for_test() - refreshes_before
7382 );
7383 let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7384 .await
7385 .expect("configure completes")
7386 .expect("completion delivered");
7387 assert!(completion.configure_response.success);
7388 assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7389 assert_eq!(
7390 cache.refresh_count_for_test() - refreshes_before,
7391 0,
7392 "route admission must not perform a fleet-wide health census"
7393 );
7394 });
7395 }
7396
7397 #[test]
7398 fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7399 let (dir, root) = test_root("health-does-not-lock-bash-db");
7400 let executor = Arc::new(Executor::new());
7401 let ctx = test_ctx();
7402 ctx.set_harness(crate::harness::Harness::Opencode);
7403 let db = Arc::new(StdMutex::new(
7404 crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7405 ));
7406 ctx.bash_background().set_db_pool(Arc::clone(&db));
7407 executor.register_actor(root, ctx);
7408
7409 let guard = db.lock().expect("hold bash-background DB mutex");
7410 let app = App::default_shared();
7411 let metrics = Arc::new(DispatchPathMetrics::new());
7412 let health_rollup_cache = Arc::new(HealthRollupCache::new());
7413 let frame = Frame::build_with_version(
7414 PROTOCOL_VERSION,
7415 FrameType::Request,
7416 control_flags(),
7417 0,
7418 0,
7419 77,
7420 Vec::new(),
7421 )
7422 .expect("health request frame");
7423 let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7424 let (done_tx, done_rx) = std::sync::mpsc::channel();
7425 let join = std::thread::spawn(move || {
7426 let runtime = tokio::runtime::Builder::new_current_thread()
7427 .enable_all()
7428 .build()
7429 .expect("health test runtime");
7430 let result = runtime.block_on(send_cached_health_response(
7431 &writer_tx,
7432 &frame,
7433 &app,
7434 &executor,
7435 &HashMap::new(),
7436 &metrics,
7437 &health_rollup_cache,
7438 ));
7439 done_tx.send(result).expect("report health result");
7440 });
7441
7442 let result = done_rx
7443 .recv_timeout(Duration::from_millis(500))
7444 .expect("channel-0 health blocked on bash-background DB mutex");
7445 result.expect("send cached health response");
7446 drop(guard);
7447 join.join().expect("health thread");
7448 }
7449
7450 #[test]
7451 fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7452 let (_dir_a, root_a) = test_root("health-metric-root-a");
7453 let (_dir_b, root_b) = test_root("health-metric-root-b");
7454 let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7455 let snapshots = HashMap::from([
7456 (root_a, HashSet::from([duplicate_key.to_string()])),
7457 (root_b, HashSet::from([duplicate_key.to_string()])),
7458 ]);
7459 let metrics = DispatchPathMetrics::new();
7460
7461 record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7462
7463 assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7464 }
7465
7466 #[test]
7467 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7468 let transient_errors = vec![
7469 attach_error(io::ErrorKind::ConnectionRefused),
7470 attach_error(io::ErrorKind::TimedOut),
7471 attach_error(io::ErrorKind::ConnectionReset),
7472 auth_io_error(io::ErrorKind::ConnectionAborted),
7473 auth_io_error(io::ErrorKind::BrokenPipe),
7474 SubcError::Auth {
7475 endpoint: "127.0.0.1:1".to_string(),
7476 source: subc_transport::AuthError::UnexpectedEof {
7477 stage: subc_transport::AuthStage::ServerProof,
7478 expected: 4,
7479 actual: 0,
7480 },
7481 },
7482 SubcError::Auth {
7483 endpoint: "127.0.0.1:1".to_string(),
7484 source: subc_transport::AuthError::Timeout {
7485 stage: subc_transport::AuthStage::ServerProof,
7486 deadline: AUTH_DEADLINE,
7487 },
7488 },
7489 ];
7490 for error in &transient_errors {
7491 assert_eq!(
7492 classify_attach_error(error),
7493 AttachErrorClass::Transient,
7494 "expected transient: {error}"
7495 );
7496 }
7497
7498 let permanent_errors = vec![
7499 attach_error(io::ErrorKind::PermissionDenied),
7500 auth_io_error(io::ErrorKind::InvalidData),
7501 SubcError::Auth {
7502 endpoint: "127.0.0.1:1".to_string(),
7503 source: subc_transport::AuthError::InvalidServerProof,
7504 },
7505 SubcError::Auth {
7506 endpoint: "127.0.0.1:1".to_string(),
7507 source: subc_transport::AuthError::DaemonIdMismatch,
7508 },
7509 SubcError::ConnectionFile {
7510 path: PathBuf::from("subc-connection.json"),
7511 source: subc_transport::ConnectionFileError::Invalid {
7512 reason: "constructed invalid file".to_string(),
7513 },
7514 },
7515 SubcError::NoEndpoint {
7516 path: PathBuf::from("subc-connection.json"),
7517 },
7518 SubcError::InvalidEndpoint {
7519 path: PathBuf::from("subc-connection.json"),
7520 endpoint: "not-an-ip:1234".to_string(),
7521 },
7522 ];
7523 for error in &permanent_errors {
7524 assert_eq!(
7525 classify_attach_error(error),
7526 AttachErrorClass::Permanent,
7527 "expected permanent: {error}"
7528 );
7529 }
7530 }
7531
7532 #[test]
7533 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7534 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7535 let conn_path = conn_dir.path().join("subc-connection.json");
7536 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7537 listener
7538 .set_nonblocking(true)
7539 .expect("set listener nonblocking");
7540 let port = listener.local_addr().expect("listener addr").port();
7541 connection_file::write_atomic(
7542 &conn_path,
7543 &connection_file::ConnectionInfo {
7544 schema: connection_file::SCHEMA_VERSION,
7545 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7546 endpoints: vec![connection_file::Endpoint {
7547 host: "127.0.0.1".to_string(),
7548 port,
7549 }],
7550 key: vec![0x42; subc_transport::KEY_LEN],
7551 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7552 pid: std::process::id(),
7553 daemon_ver: "subc-test".to_string(),
7554 },
7555 )
7556 .expect("write connection file");
7557
7558 let runtime = tokio::runtime::Builder::new_current_thread()
7559 .enable_all()
7560 .build()
7561 .expect("test runtime");
7562 let result = runtime.block_on(connect_and_authenticate_with_policy(
7563 &conn_path,
7564 AttachRetryPolicy {
7565 budget: Duration::from_secs(1),
7566 initial_backoff: Duration::from_millis(5),
7567 max_backoff: Duration::from_millis(10),
7568 jitter_percent: 0,
7569 },
7570 None,
7571 ));
7572 assert!(matches!(
7573 result,
7574 Err(SubcError::ConnectionFile {
7575 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7576 ..
7577 })
7578 ));
7579 assert!(matches!(
7580 listener.accept(),
7581 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7582 ));
7583 }
7584
7585 #[test]
7586 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7587 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7588 let conn_path = conn_dir.path().join("subc-connection.json");
7589 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7590 let port = listener.local_addr().expect("reserved addr").port();
7591 drop(listener);
7592 connection_file::write_atomic(
7593 &conn_path,
7594 &connection_file::ConnectionInfo {
7595 schema: connection_file::SCHEMA_VERSION,
7596 wire_version: Some(PROTOCOL_VERSION),
7597 endpoints: vec![connection_file::Endpoint {
7598 host: "127.0.0.1".to_string(),
7599 port,
7600 }],
7601 key: vec![0x42; subc_transport::KEY_LEN],
7602 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7603 pid: std::process::id(),
7604 daemon_ver: "subc-test".to_string(),
7605 },
7606 )
7607 .expect("write connection file");
7608
7609 let policy = AttachRetryPolicy {
7610 budget: Duration::from_millis(40),
7611 initial_backoff: Duration::from_millis(5),
7612 max_backoff: Duration::from_millis(10),
7613 jitter_percent: 0,
7614 };
7615 let runtime = tokio::runtime::Builder::new_current_thread()
7616 .enable_all()
7617 .build()
7618 .expect("test runtime");
7619 let started_at = Instant::now();
7620 let result = runtime.block_on(connect_and_authenticate_with_policy(
7621 &conn_path, policy, None,
7622 ));
7623 let elapsed = started_at.elapsed();
7624 let error = match result {
7625 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7626 Err(error) => error,
7627 };
7628
7629 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7630 assert!(
7631 elapsed >= Duration::from_millis(35),
7632 "retry budget ended too early: {elapsed:?}"
7633 );
7634 assert!(
7635 elapsed < Duration::from_secs(1),
7636 "retry budget was not bounded: {elapsed:?}"
7637 );
7638 }
7639
7640 fn due_maintenance_jobs_without_actor_context(
7641 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7642 budget: usize,
7643 pending_bind_roots: &HashSet<ProjectRootId>,
7644 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7645 due_maintenance_jobs(
7646 live_roots,
7647 None,
7648 &HashMap::new(),
7649 &BgWakePending::new(),
7650 budget,
7651 pending_bind_roots,
7652 )
7653 }
7654
7655 fn actor_ctx_with_dirty_search_index(
7656 root: &Path,
7657 storage: &Path,
7658 file_name: &str,
7659 old_contents: &str,
7660 new_contents: &str,
7661 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7662 let file = root.join(file_name);
7663 std::fs::write(&file, old_contents).expect("write source");
7664 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7665 let ctx = Arc::new(AppContext::new(
7666 Box::new(crate::parser::TreeSitterProvider::new()),
7667 Config {
7668 project_root: Some(root.to_path_buf()),
7669 storage_dir: Some(storage.to_path_buf()),
7670 ..Config::default()
7671 },
7672 ));
7673 ctx.set_canonical_cache_root(canonical_root.clone());
7674
7675 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7676 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7677 let git_head = index.stored_git_head().map(str::to_owned);
7678 index.write_to_disk(&cache_dir, git_head.as_deref());
7679
7680 std::fs::write(&file, new_contents).expect("edit source");
7681 index.update_file(&file);
7682 *ctx.search_index()
7683 .write()
7684 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7685 (ctx, canonical_root, cache_dir)
7686 }
7687
7688 #[test]
7689 fn graceful_shutdown_flushes_every_actor_search_index() {
7690 let storage = tempfile::tempdir().expect("storage tempdir");
7691 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7692 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7693 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7694 root1_dir.path(),
7695 storage.path(),
7696 "alpha.txt",
7697 "old actor one token\n",
7698 "new actor one token\n",
7699 );
7700 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7701 root2_dir.path(),
7702 storage.path(),
7703 "beta.txt",
7704 "old actor two token\n",
7705 "new actor two token\n",
7706 );
7707
7708 let executor = Executor::new();
7709 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7710 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7711
7712 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7713
7714 let mut restored1 =
7715 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7716 .expect("load flushed root one index");
7717 restored1.ready = true;
7718 assert_eq!(
7719 restored1
7720 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7721 .matches
7722 .len(),
7723 1,
7724 "graceful subc shutdown should flush the first root's trigram delta"
7725 );
7726
7727 let mut restored2 =
7728 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7729 .expect("load flushed root two index");
7730 restored2.ready = true;
7731 assert_eq!(
7732 restored2
7733 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7734 .matches
7735 .len(),
7736 1,
7737 "graceful subc shutdown should flush every registered root"
7738 );
7739 }
7740
7741 #[test]
7742 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7743 let _ = env_logger::builder().is_test(true).try_init();
7744 let (root_dir, root) = test_root("idle-root-reaper");
7745 let storage = tempfile::tempdir().expect("storage tempdir");
7746 std::fs::write(
7747 root_dir.path().join("main.rs"),
7748 "fn entry() { leaf(); }\nfn leaf() {}\n",
7749 )
7750 .expect("source file");
7751 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7752 let app = App::default_shared();
7753 let ctx = Arc::new(AppContext::from_app(
7754 Arc::clone(&app),
7755 Config {
7756 project_root: Some(canonical_root.clone()),
7757 storage_dir: Some(storage.path().to_path_buf()),
7758 callgraph_store: true,
7759 search_index: true,
7760 ..Config::default()
7761 },
7762 ));
7763 ctx.set_canonical_cache_root(canonical_root.clone());
7764 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7765 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7766 assert!(ctx
7767 .ensure_callgraph_store()
7768 .expect("build callgraph store")
7769 .is_some());
7770
7771 let cache_dir =
7772 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7773 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7774 let git_head = index.stored_git_head().map(str::to_owned);
7775 index.write_to_disk(&cache_dir, git_head.as_deref());
7776 *ctx.search_index()
7777 .write()
7778 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7779 let seeded_generation =
7782 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7783 .expect("seeded artifact generation");
7784 crate::cache_freshness::record_verify_completed(
7785 &canonical_root,
7786 crate::cache_freshness::VerifyArtifact::Search,
7787 Some(seeded_generation),
7788 );
7789 assert!(
7790 matches!(
7791 crate::cache_freshness::warm_verify_plan(
7792 canonical_root.as_path(),
7793 crate::cache_freshness::VerifyArtifact::Search,
7794 Some(seeded_generation),
7795 ),
7796 crate::cache_freshness::WarmVerifyPlan::Skip
7797 ),
7798 "memo must be warm before eviction for the downgrade assertion to bite"
7799 );
7800
7801 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7802 let _dispatch_tx = dispatch_tx;
7803 let shutdown = Arc::new(AtomicBool::new(false));
7804 let thread_shutdown = Arc::clone(&shutdown);
7805 let join = std::thread::spawn(move || {
7806 while !thread_shutdown.load(Ordering::SeqCst) {
7807 std::thread::yield_now();
7808 }
7809 });
7810 ctx.install_watcher_runtime(
7811 dispatch_rx,
7812 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7813 );
7814 wait_for_watcher_count(&ctx, 1);
7815
7816 let executor = Arc::new(Executor::new());
7817 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7818 ctx.mark_subc_unbound();
7819 let mut live_roots = HashMap::new();
7820 let mut meta = RootMeta::new(Instant::now());
7821 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7822 meta.unbound_quiesced = true;
7823 live_roots.insert(root.clone(), meta);
7824
7825 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7826 assert!(message.contains("evicted idle root"));
7827 assert!(message.contains("freed ~"));
7828 assert!(message.contains("semantic"));
7829 assert!(!message.contains("semantic not estimated retained"));
7830 assert!(message.contains("trigram"));
7831 assert!(message.contains("retained: bash"));
7832 assert!(message.contains("parser_pool"));
7833
7834 assert_eq!(
7835 reap_idle_roots(
7836 Instant::now(),
7837 &mut live_roots,
7838 &HashMap::new(),
7839 &HashMap::new(),
7840 &executor,
7841 &DispatchPathMetrics::new(),
7842 )
7843 .evicted,
7844 1
7845 );
7846 assert!(ctx.search_index().read().unwrap().is_none());
7847 wait_for_watcher_count(&ctx, 0);
7848 assert!(
7853 matches!(
7854 crate::cache_freshness::warm_verify_plan(
7855 canonical_root.as_path(),
7856 crate::cache_freshness::VerifyArtifact::Search,
7857 Some(seeded_generation),
7858 ),
7859 crate::cache_freshness::WarmVerifyPlan::Strict
7860 ),
7861 "idle eviction must force strict re-verification"
7862 );
7863 assert!(
7864 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7865 );
7866 ctx.mark_subc_bound();
7867 assert!(ctx
7868 .ensure_callgraph_store()
7869 .expect("reopen callgraph store")
7870 .is_some());
7871 assert!(live_roots[&root].idle_artifacts_evicted);
7872 }
7873
7874 #[test]
7875 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7876 let (_root_dir, root) = test_root("idle-root-ttl-gate");
7877 let ctx = test_ctx();
7878 let executor = Arc::new(Executor::new());
7879 assert!(executor.register_actor(root.clone(), ctx));
7880 let ctx = executor.actor_context(&root).expect("actor context");
7881 ctx.mark_subc_unbound();
7882 let now = Instant::now();
7883 let mut meta = RootMeta::new(now);
7884 meta.unbound_quiesced = true;
7885 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7886
7887 assert_eq!(
7891 reap_idle_roots(
7892 now,
7893 &mut live_roots,
7894 &HashMap::new(),
7895 &HashMap::new(),
7896 &executor,
7897 &DispatchPathMetrics::new(),
7898 )
7899 .evicted,
7900 0
7901 );
7902 assert!(!live_roots[&root].idle_artifacts_evicted);
7903
7904 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7910 assert_eq!(
7911 reap_idle_roots(
7912 now + IDLE_ROOT_TTL,
7913 &mut live_roots,
7914 &HashMap::new(),
7915 &HashMap::new(),
7916 &executor,
7917 &DispatchPathMetrics::new(),
7918 )
7919 .evicted,
7920 1
7921 );
7922 assert!(live_roots[&root].idle_artifacts_evicted);
7923 assert!(
7924 ctx.take_pending_search_index_paths().is_empty(),
7925 "TTL eviction must dispose retained pending reconciliation paths"
7926 );
7927 }
7928
7929 #[test]
7930 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7931 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7932 let ctx = test_ctx();
7933 let executor = Arc::new(Executor::new());
7934 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7935 ctx.mark_subc_unbound();
7936
7937 let pending = root.as_path().join("edited-while-unbound.rs");
7943 ctx.add_pending_search_index_paths([pending.clone()]);
7944 let dirty_source = root.as_path().join("dirty.rs");
7945 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7946 let mut dirty = crate::search_index::SearchIndex::new();
7947 dirty.ready = true;
7948 dirty.update_file(&dirty_source);
7949 *ctx.search_index()
7950 .write()
7951 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7952 assert!(ctx.artifact_eviction_blocked());
7953
7954 let mut live_roots = HashMap::new();
7955 let mut meta = RootMeta::new(Instant::now());
7956 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7957 meta.unbound_quiesced = true;
7958 live_roots.insert(root.clone(), meta);
7959
7960 assert_eq!(
7961 reap_idle_roots(
7962 Instant::now(),
7963 &mut live_roots,
7964 &HashMap::new(),
7965 &HashMap::new(),
7966 &executor,
7967 &DispatchPathMetrics::new(),
7968 )
7969 .evicted,
7970 0,
7971 "the dirty index must still block this eviction"
7972 );
7973 assert_eq!(
7974 ctx.take_pending_search_index_paths(),
7975 vec![pending],
7976 "a blocked eviction must restore the taken pending paths"
7977 );
7978 }
7979
7980 #[test]
7981 fn idle_reap_with_bound_route_keeps_watcher_running() {
7982 let (_root_dir, root) = test_root("bound-root-reap-gate");
7983 let ctx = test_ctx();
7984 let executor = Arc::new(Executor::new());
7985 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7986
7987 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7988 let _dispatch_tx = dispatch_tx;
7989 let shutdown = Arc::new(AtomicBool::new(false));
7990 let thread_shutdown = Arc::clone(&shutdown);
7991 let join = std::thread::spawn(move || {
7992 while !thread_shutdown.load(Ordering::SeqCst) {
7993 std::thread::yield_now();
7994 }
7995 });
7996 ctx.install_watcher_runtime(
7997 dispatch_rx,
7998 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7999 );
8000
8001 let mut meta = RootMeta::new(Instant::now());
8002 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
8003 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8004 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
8005 assert_eq!(
8006 reap_idle_roots(
8007 Instant::now(),
8008 &mut live_roots,
8009 &HashMap::new(),
8010 &bound,
8011 &executor,
8012 &DispatchPathMetrics::new(),
8013 )
8014 .evicted,
8015 0
8016 );
8017 wait_for_watcher_count(&ctx, 1);
8018 ctx.stop_watcher_runtime_in_background();
8019 wait_for_watcher_count(&ctx, 0);
8020 }
8021
8022 #[test]
8023 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8024 let (root_dir, root) = test_root("deleted-bound-root-reap");
8025 let executor = Arc::new(Executor::new());
8026 assert!(executor.register_actor(root.clone(), test_ctx()));
8027 root_dir.close().expect("delete project root");
8028
8029 let route = route_key(19, 3);
8030 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8031 let cancel_signal = PersistentCancelSignal::new();
8032 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8033 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8034 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8035 let mut route_bash_cancels = HashMap::from([(
8036 route,
8037 bash::RouteBashCancel {
8038 token: cancel_signal.clone(),
8039 active_waits: 0,
8040 },
8041 )]);
8042 let metrics = DispatchPathMetrics::new();
8043
8044 let first = reap_idle_roots(
8045 Instant::now(),
8046 &mut live_roots,
8047 &HashMap::new(),
8048 &root_channels,
8049 &executor,
8050 &metrics,
8051 );
8052 assert!(first.forgotten_deleted_roots.is_empty());
8053 assert!(executor.actor_registered(&root));
8054
8055 let mut forgotten = Vec::new();
8056 for _ in 0..100 {
8057 let outcome = reap_idle_roots(
8058 Instant::now(),
8059 &mut live_roots,
8060 &HashMap::new(),
8061 &root_channels,
8062 &executor,
8063 &metrics,
8064 );
8065 if !outcome.forgotten_deleted_roots.is_empty() {
8066 forgotten = outcome.forgotten_deleted_roots;
8067 break;
8068 }
8069 std::thread::sleep(Duration::from_millis(10));
8070 }
8071 assert_eq!(forgotten, vec![root.clone()]);
8072 assert!(!executor.actor_registered(&root));
8073
8074 let mut retry_buffer = HashMap::new();
8075 let mut reclaimed_routes = ReclaimedRoutes::default();
8076 let mut session_identity = HashMap::new();
8077 let mut push_buffer = HashMap::new();
8078 let mut bg_subs = HashMap::from([(
8079 route,
8080 BgSub {
8081 corr: 77,
8082 ver: PROTOCOL_VERSION,
8083 flags: control_flags(),
8084 root: root.clone(),
8085 session: "deleted-route".to_string(),
8086 },
8087 )]);
8088 let mut bg_sub_by_session = HashMap::from([(
8089 (root.clone(), "deleted-route".to_string()),
8090 HashSet::from([route]),
8091 )]);
8092 let mut bg_wake_pending =
8093 BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8094 let mut bg_wake_epoch = HashMap::new();
8095 let mut pending_bash_asks = HashMap::new();
8096 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8097 health::take_bg_observability_logs_for_test();
8098 purge_deleted_root_residents(
8099 &root,
8100 &mut routes,
8101 &mut root_channels,
8102 &mut installed_route_epochs,
8103 &mut route_bash_cancels,
8104 &active_tool_calls,
8105 executor.as_ref(),
8106 &mut retry_buffer,
8107 &mut reclaimed_routes,
8108 &mut session_identity,
8109 &mut push_buffer,
8110 &mut bg_subs,
8111 &mut bg_sub_by_session,
8112 &mut bg_wake_pending,
8113 &mut bg_wake_epoch,
8114 &mut pending_bash_asks,
8115 &metrics,
8116 );
8117
8118 assert!(routes.is_empty());
8119 assert!(root_channels.is_empty());
8120 assert!(installed_route_epochs.is_empty());
8121 assert!(route_bash_cancels.is_empty());
8122 assert!(reclaimed_routes.contains(route));
8123 assert!(cancel_signal.is_cancelled());
8124 assert_eq!(
8125 health::take_bg_observability_logs_for_test(),
8126 vec![format!(
8127 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8128 root.as_path().display()
8129 )]
8130 );
8131 }
8132
8133 #[test]
8140 fn live_root_with_bound_route_is_never_reclaimed() {
8141 let (_root_dir, root) = test_root("live-bound-root-retained");
8142 let ctx = test_ctx();
8143 ctx.mark_subc_unbound();
8144 let executor = Arc::new(Executor::new());
8145 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8146
8147 let route = RouteChannel {
8148 channel: 7,
8149 epoch: 1,
8150 };
8151 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8152 meta.unbound_quiesced = true;
8153 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8154 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8155
8156 for _ in 0..3 {
8161 let outcome = reap_idle_roots(
8162 Instant::now(),
8163 &mut live_roots,
8164 &HashMap::new(),
8165 &root_channels,
8166 &executor,
8167 &DispatchPathMetrics::new(),
8168 );
8169 assert!(
8170 outcome.forgotten_deleted_roots.is_empty(),
8171 "a root whose directory exists must never be forgotten"
8172 );
8173 }
8174
8175 assert!(live_roots.contains_key(&root), "live root must be retained");
8176 assert!(
8177 executor.actor_registered(&root),
8178 "live root's actor must survive"
8179 );
8180 assert!(
8181 root.as_path().exists(),
8182 "test vehicle must keep the directory alive; otherwise this control proves nothing"
8183 );
8184 }
8185
8186 #[test]
8187 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8188 let (root_dir, root) = test_root("deleted-root-first-observation");
8189 let ctx = test_ctx();
8190 ctx.mark_subc_unbound();
8191 let executor = Arc::new(Executor::new());
8192 assert!(executor.register_actor(root.clone(), ctx));
8193 root_dir.close().expect("delete project root");
8194
8195 let mut meta = RootMeta::new(Instant::now());
8196 meta.unbound_quiesced = true;
8197 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8198 let outcome = reap_idle_roots(
8199 Instant::now(),
8200 &mut live_roots,
8201 &HashMap::new(),
8202 &HashMap::new(),
8203 &executor,
8204 &DispatchPathMetrics::new(),
8205 );
8206
8207 assert!(outcome.forgotten_deleted_roots.is_empty());
8208 assert!(live_roots.contains_key(&root));
8209 assert!(executor.actor_registered(&root));
8210 }
8211
8212 fn spawn_background_for_root(
8213 ctx: &AppContext,
8214 root: &ProjectRootId,
8215 storage: &tempfile::TempDir,
8216 session_id: &str,
8217 ) -> (String, u32) {
8218 let command = if cfg!(windows) {
8224 "ping -n 31 127.0.0.1 > nul"
8226 } else {
8227 "sleep 30"
8228 };
8229 let task_id = ctx
8230 .bash_background()
8231 .spawn(
8232 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8233 command,
8234 session_id.to_string(),
8235 storage.path().to_path_buf(),
8236 HashMap::new(),
8237 Some(Duration::from_secs(60)),
8238 storage.path().to_path_buf(),
8239 8,
8240 true,
8241 false,
8242 Some(root.as_path().to_path_buf()),
8243 )
8244 .expect("spawn background task");
8245 let snapshot = ctx
8246 .bash_background()
8247 .status(
8248 &task_id,
8249 session_id,
8250 Some(root.as_path()),
8251 Some(storage.path()),
8252 0,
8253 )
8254 .expect("background task status");
8255 (task_id, snapshot.child_pid.expect("background child pid"))
8256 }
8257
8258 fn wait_for_background_exit(pid: u32) {
8259 let deadline = Instant::now() + Duration::from_secs(5);
8260 while crate::bash_background::process::is_process_alive(pid) {
8261 assert!(
8262 Instant::now() < deadline,
8263 "background task process survived kill"
8264 );
8265 std::thread::sleep(Duration::from_millis(20));
8266 }
8267 }
8268
8269 #[test]
8270 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8271 let (root_dir, root) = test_root("deleted-root-background-task");
8272 let storage = tempfile::tempdir().expect("task storage");
8273 let ctx = test_ctx();
8274 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8275 assert!(crate::bash_background::process::is_process_alive(pid));
8276
8277 let executor = Arc::new(Executor::new());
8278 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8279 assert!(executor.actor_is_idle(&root));
8280 root_dir.close().expect("delete project root");
8281 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8282 let pending_binds = HashMap::new();
8283 let root_channels = HashMap::new();
8284 let metrics = DispatchPathMetrics::new();
8285
8286 let first = reap_idle_roots(
8287 Instant::now(),
8288 &mut live_roots,
8289 &pending_binds,
8290 &root_channels,
8291 &executor,
8292 &metrics,
8293 );
8294 assert!(first.forgotten_deleted_roots.is_empty());
8295 assert!(crate::bash_background::process::is_process_alive(pid));
8296
8297 let outcome = reap_until_forgotten(
8298 &root,
8299 &mut live_roots,
8300 &pending_binds,
8301 &root_channels,
8302 &executor,
8303 &metrics,
8304 );
8305 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8306 wait_for_background_exit(pid);
8307
8308 let snapshot = ctx
8309 .bash_background()
8310 .status(
8311 &task_id,
8312 "reclaim-session",
8313 Some(root.as_path()),
8314 Some(storage.path()),
8315 0,
8316 )
8317 .expect("reclaimed task status");
8318 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8319 assert_eq!(
8320 snapshot.info.status_reason.as_deref(),
8321 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8322 );
8323 assert_eq!(
8324 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8325 crate::bash_background::registry::ROOT_RECLAIMED_REASON
8326 );
8327 let completion = ctx
8328 .bash_background()
8329 .drain_completions_for_session(Some("reclaim-session"))
8330 .pop()
8331 .expect("reclaimed task completion");
8332 assert_eq!(
8333 completion.status_reason.as_deref(),
8334 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8335 );
8336 }
8337
8338 #[test]
8339 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8340 let (root_dir, root) = test_root("existing-root-background-task");
8341 let storage = tempfile::tempdir().expect("task storage");
8342 let ctx = test_ctx();
8343 ctx.mark_subc_unbound();
8344 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8345
8346 let executor = Arc::new(Executor::new());
8347 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8348 let mut meta = RootMeta::new(
8349 Instant::now()
8350 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8351 .expect("old root timestamp"),
8352 );
8353 meta.unbound_quiesced = true;
8354 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8355 let pending_binds = HashMap::new();
8356 let root_channels = HashMap::new();
8357 let metrics = DispatchPathMetrics::new();
8358
8359 for _ in 0..8 {
8360 reap_idle_roots(
8361 Instant::now(),
8362 &mut live_roots,
8363 &pending_binds,
8364 &root_channels,
8365 &executor,
8366 &metrics,
8367 );
8368 std::thread::sleep(Duration::from_millis(10));
8369 }
8370 assert!(root_dir.path().exists());
8371 assert!(crate::bash_background::process::is_process_alive(pid));
8372 let snapshot = ctx
8373 .bash_background()
8374 .status(
8375 &task_id,
8376 "existing-session",
8377 Some(root.as_path()),
8378 Some(storage.path()),
8379 0,
8380 )
8381 .expect("existing task status");
8382 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8383 let _ = ctx.bash_background().kill(&task_id, "existing-session");
8384 wait_for_background_exit(pid);
8385 }
8386
8387 #[test]
8388 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8389 let (root_dir, root) = test_root("restored-root-background-task");
8390 let storage = tempfile::tempdir().expect("task storage");
8391 let ctx = test_ctx();
8392 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8393
8394 let executor = Arc::new(Executor::new());
8395 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8396 root_dir.close().expect("delete project root");
8397 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8398 let pending_binds = HashMap::new();
8399 let root_channels = HashMap::new();
8400 let metrics = DispatchPathMetrics::new();
8401
8402 let first = reap_idle_roots(
8403 Instant::now(),
8404 &mut live_roots,
8405 &pending_binds,
8406 &root_channels,
8407 &executor,
8408 &metrics,
8409 );
8410 assert!(first.forgotten_deleted_roots.is_empty());
8411 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8412 let second = reap_idle_roots(
8413 Instant::now(),
8414 &mut live_roots,
8415 &pending_binds,
8416 &root_channels,
8417 &executor,
8418 &metrics,
8419 );
8420 assert!(second.forgotten_deleted_roots.is_empty());
8421 assert!(crate::bash_background::process::is_process_alive(pid));
8422 let _ = ctx.bash_background().kill(&task_id, "restored-session");
8423 wait_for_background_exit(pid);
8424 }
8425
8426 #[test]
8427 fn observing_root_again_resets_deleted_sweep_confirmation() {
8428 let (root_dir, root) = test_root("deleted-root-observation-reset");
8429 let ctx = test_ctx();
8430 ctx.mark_subc_unbound();
8431 let executor = Arc::new(Executor::new());
8432 assert!(executor.register_actor(root.clone(), ctx));
8433 root_dir.close().expect("delete project root");
8434
8435 let mut meta = RootMeta::new(Instant::now());
8436 meta.unbound_quiesced = true;
8437 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8438 let pending_binds = HashMap::new();
8439 let root_channels = HashMap::new();
8440 let metrics = DispatchPathMetrics::new();
8441
8442 let first = reap_idle_roots(
8443 Instant::now(),
8444 &mut live_roots,
8445 &pending_binds,
8446 &root_channels,
8447 &executor,
8448 &metrics,
8449 );
8450 assert!(first.forgotten_deleted_roots.is_empty());
8451
8452 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8453 reap_idle_roots(
8454 Instant::now(),
8455 &mut live_roots,
8456 &pending_binds,
8457 &root_channels,
8458 &executor,
8459 &metrics,
8460 );
8461 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8462
8463 let after_reset = reap_idle_roots(
8464 Instant::now(),
8465 &mut live_roots,
8466 &pending_binds,
8467 &root_channels,
8468 &executor,
8469 &metrics,
8470 );
8471 assert!(after_reset.forgotten_deleted_roots.is_empty());
8472 assert!(live_roots.contains_key(&root));
8473 assert!(executor.actor_registered(&root));
8474 }
8475
8476 #[test]
8477 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8478 let (root_dir, root) = test_root("deleted-root-reap");
8479 let app = App::default_shared();
8480 let ctx = Arc::new(AppContext::from_app(
8481 Arc::clone(&app),
8482 Config {
8483 project_root: Some(root.as_path().to_path_buf()),
8484 ..Config::default()
8485 },
8486 ));
8487 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8488 ctx.mark_subc_unbound();
8489 let executor = Arc::new(Executor::new());
8490 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8491 assert_eq!(app.actor_root_count(), 1);
8492 drop(ctx);
8493 root_dir.close().expect("delete project root");
8494
8495 let mut meta = RootMeta::new(Instant::now());
8496 meta.unbound_quiesced = true;
8497 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8498 let outcome = reap_until_forgotten(
8499 &root,
8500 &mut live_roots,
8501 &HashMap::new(),
8502 &HashMap::new(),
8503 &executor,
8504 &DispatchPathMetrics::new(),
8505 );
8506 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8507 assert!(!executor.actor_registered(&root));
8508 assert!(!live_roots.contains_key(&root));
8509 wait_for_actor_root_count(&app, 0);
8510
8511 let status_ctx = AppContext::from_app(app, Config::default());
8512 let status = status_ctx.build_status_snapshot();
8513 assert_eq!(status["runtime"]["live_actor_roots"], 0);
8514 assert_eq!(status["runtime"]["open_routes"], 0);
8515 }
8516
8517 #[test]
8518 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8519 let (root_dir, root) = test_root("deleted-root-reap-census");
8520 let executor = Arc::new(Executor::new());
8521 assert!(executor.register_actor(root.clone(), test_ctx()));
8522 root_dir.close().expect("delete project root");
8523
8524 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8525 let metrics = DispatchPathMetrics::new();
8526 let outcome = reap_idle_roots(
8527 Instant::now(),
8528 &mut live_roots,
8529 &HashMap::new(),
8530 &HashMap::new(),
8531 &executor,
8532 &metrics,
8533 );
8534 assert_eq!(outcome.evicted, 0);
8535
8536 let app = crate::context::App::default_shared();
8537 let health_rollup_cache = HealthRollupCache::new();
8538 health_rollup_cache.refresh(&executor, &app);
8539 let report = build_health_report(
8540 &health_rollup_cache,
8541 &executor,
8542 &HashMap::new(),
8543 &metrics,
8544 &app,
8545 );
8546 let reap = report
8547 .metrics
8548 .as_ref()
8549 .and_then(|metrics| metrics.get("reap"))
8550 .expect("reap health metrics");
8551 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8552 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8553 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8554 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8555 }
8556
8557 #[test]
8558 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8559 let (root_dir, root) = test_root("connection-exit-deleted-root");
8560 let executor = Arc::new(Executor::new());
8561 assert!(executor.register_actor(root.clone(), test_ctx()));
8562
8563 let route = route_key(11, 1);
8564 let mut meta = RootMeta::new(Instant::now());
8565 meta.maintenance_pending = true;
8566 meta.maintenance_queued_kinds
8567 .push_back(MaintenanceDrainKind::CompletionDrains);
8568 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8569 let mut pending_binds = HashMap::new();
8570 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8571 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8572 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8573 let mut route_bash_cancels = HashMap::new();
8574 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8575
8576 quiesce_connection_roots(
8577 &mut live_roots,
8578 &mut pending_binds,
8579 &mut routes,
8580 &mut root_channels,
8581 &mut installed_route_epochs,
8582 &mut route_bash_cancels,
8583 &active_tool_calls,
8584 &executor,
8585 );
8586 assert!(live_roots[&root].unbound_quiesced);
8587 assert!(!live_roots[&root].maintenance_pending);
8588 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8589 assert!(routes.is_empty());
8590 assert!(root_channels.is_empty());
8591
8592 root_dir.close().expect("delete project root");
8593 let metrics = DispatchPathMetrics::new();
8594 let outcome = reap_until_forgotten(
8595 &root,
8596 &mut live_roots,
8597 &pending_binds,
8598 &root_channels,
8599 &executor,
8600 &metrics,
8601 );
8602 let mut session_identity = HashMap::new();
8603 let mut push_buffer = HashMap::new();
8604 let mut bg_subs = HashMap::new();
8605 let mut bg_sub_by_session = HashMap::new();
8606 let mut bg_wake_pending = BgWakePending::new();
8607 let mut bg_wake_epoch = HashMap::new();
8608 let mut pending_bash_asks = HashMap::new();
8609 let mut retry_buffer = HashMap::new();
8610 let mut reclaimed_routes = ReclaimedRoutes::default();
8611 for forgotten in &outcome.forgotten_deleted_roots {
8612 purge_deleted_root_residents(
8613 forgotten,
8614 &mut routes,
8615 &mut root_channels,
8616 &mut installed_route_epochs,
8617 &mut route_bash_cancels,
8618 &active_tool_calls,
8619 executor.as_ref(),
8620 &mut retry_buffer,
8621 &mut reclaimed_routes,
8622 &mut session_identity,
8623 &mut push_buffer,
8624 &mut bg_subs,
8625 &mut bg_sub_by_session,
8626 &mut bg_wake_pending,
8627 &mut bg_wake_epoch,
8628 &mut pending_bash_asks,
8629 &metrics,
8630 );
8631 }
8632
8633 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8634 assert!(!executor.actor_registered(&root));
8635 assert!(!live_roots.contains_key(&root));
8636 }
8637
8638 #[test]
8639 fn unbound_root_quiesces_maintenance_without_removing_actor() {
8640 let (_root_dir, root) = test_root("unbound-root-quiesce");
8641 let ctx = test_ctx();
8642 let executor = Arc::new(Executor::new());
8643 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8644 let mut meta = RootMeta::new(Instant::now());
8645 meta.maintenance_pending = true;
8646 meta.maintenance_jobs_in_flight = 1;
8647 meta.maintenance_queued_kinds
8648 .push_back(MaintenanceDrainKind::ConfigureTail);
8649 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8650 *ctx.search_index()
8652 .write()
8653 .unwrap_or_else(std::sync::PoisonError::into_inner) =
8654 Some(crate::search_index::SearchIndex::new());
8655 ctx.set_cache_writer_capabilities(true, true);
8656 let pending = root.as_path().join("pending.rs");
8657 ctx.add_pending_search_index_paths([pending.clone()]);
8658 let canonical_root = root.as_path().to_path_buf();
8662 let artifact = canonical_root.join("cache.bin");
8663 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8664 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8665 crate::cache_freshness::record_verify_completed(
8666 &canonical_root,
8667 crate::cache_freshness::VerifyArtifact::Search,
8668 seeded_generation,
8669 );
8670 assert!(matches!(
8671 crate::cache_freshness::warm_verify_plan(
8672 &canonical_root,
8673 crate::cache_freshness::VerifyArtifact::Search,
8674 seeded_generation,
8675 ),
8676 crate::cache_freshness::WarmVerifyPlan::Skip
8677 ));
8678 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8681 let _dispatch_tx = dispatch_tx;
8682 let shutdown = Arc::new(AtomicBool::new(false));
8683 let thread_shutdown = Arc::clone(&shutdown);
8684 let join = std::thread::spawn(move || {
8685 while !thread_shutdown.load(Ordering::SeqCst) {
8686 std::thread::yield_now();
8687 }
8688 });
8689 ctx.install_watcher_runtime(
8690 dispatch_rx,
8691 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8692 );
8693 assert!(ctx.watcher_runtime_active());
8694
8695 quiesce_unbound_root(&root, &mut live_roots, &executor);
8696 let meta = &live_roots[&root];
8697 assert!(meta.unbound_quiesced);
8698 assert!(ctx.subc_unbound_quiesced());
8699 assert!(meta.maintenance_pending);
8700 assert!(meta.maintenance_queued_kinds.is_empty());
8701 assert!(executor.actor_registered(&root));
8702 assert!(
8706 ctx.search_index()
8707 .read()
8708 .unwrap_or_else(std::sync::PoisonError::into_inner)
8709 .is_some(),
8710 "quiesce must not evict resident artifacts"
8711 );
8712 assert_eq!(
8713 ctx.pending_callgraph_store_force_token(),
8714 None,
8715 "quiesce must not force a callgraph rebuild"
8716 );
8717 assert_eq!(
8718 ctx.take_pending_search_index_paths(),
8719 vec![pending],
8720 "quiesce must retain pending watcher-derived paths"
8721 );
8722 assert!(
8723 matches!(
8724 crate::cache_freshness::warm_verify_plan(
8725 &canonical_root,
8726 crate::cache_freshness::VerifyArtifact::Search,
8727 seeded_generation,
8728 ),
8729 crate::cache_freshness::WarmVerifyPlan::Skip
8730 ),
8731 "quiesce must not invalidate the warm verify memo"
8732 );
8733 assert!(
8734 ctx.watcher_runtime_active(),
8735 "quiesce must not stop a running watcher"
8736 );
8737 ctx.stop_watcher_runtime();
8738
8739 let meta = live_roots.get_mut(&root).expect("root metadata");
8740 note_maintenance_completion(
8741 meta,
8742 Some(MaintenanceDrainKind::ConfigureTail),
8743 false,
8744 meta.unbound_quiesced,
8745 );
8746 assert!(!meta.maintenance_pending);
8747 assert!(meta.maintenance_queued_kinds.is_empty());
8748 }
8749
8750 #[test]
8751 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8752 let (_dir, root) = test_root("same-root-replacement");
8753 let route = route_key(7, 1);
8754 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8755 let root_channels = HashMap::new();
8756
8757 assert!(!route_removal_will_quiesce_root(
8758 &root,
8759 route,
8760 &installed_channels,
8761 false,
8762 Some(&root),
8763 ));
8764 assert!(route_removal_will_quiesce_root(
8765 &root,
8766 route,
8767 &installed_channels,
8768 false,
8769 None,
8770 ));
8771 assert!(!should_quiesce_removed_root(
8772 &root,
8773 &root_channels,
8774 false,
8775 Some(&root),
8776 ));
8777 assert!(should_quiesce_removed_root(
8778 &root,
8779 &root_channels,
8780 false,
8781 None,
8782 ));
8783 assert!(!should_quiesce_removed_root(
8784 &root,
8785 &root_channels,
8786 true,
8787 None,
8788 ));
8789 }
8790
8791 #[test]
8792 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8793 let (_root_dir, root) = test_root("unbound-root-route-count");
8794 let executor = Arc::new(Executor::new());
8795 assert!(executor.register_actor(root.clone(), test_ctx()));
8796 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8797 let mut root_channels = HashMap::from([(
8798 root.clone(),
8799 HashSet::from([route_key(7, 1), route_key(8, 1)]),
8800 )]);
8801
8802 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8803 if !root_channels.contains_key(&root) {
8804 quiesce_unbound_root(&root, &mut live_roots, &executor);
8805 }
8806 assert!(!live_roots[&root].unbound_quiesced);
8807
8808 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8809 if !root_channels.contains_key(&root) {
8810 quiesce_unbound_root(&root, &mut live_roots, &executor);
8811 }
8812 assert!(live_roots[&root].unbound_quiesced);
8813
8814 live_roots
8815 .get_mut(&root)
8816 .expect("root metadata")
8817 .note_activity();
8818 assert!(
8819 live_roots[&root].unbound_quiesced,
8820 "late asynchronous activity must not reactivate an unbound root"
8821 );
8822
8823 live_roots
8824 .get_mut(&root)
8825 .expect("root metadata")
8826 .reactivate_bound();
8827 assert!(!live_roots[&root].unbound_quiesced);
8828 }
8829
8830 #[test]
8831 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8832 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8833 let (_active_dir, active_root) = test_root("allocator-relief-active");
8834 let now = Instant::now();
8835 let mut live_roots = HashMap::new();
8836 let mut idle = RootMeta::new(now);
8837 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8838 live_roots.insert(idle_root, idle);
8839 let executor = Executor::new();
8840 assert!(process_has_been_idle(now, &live_roots, &executor));
8841
8842 live_roots.insert(active_root.clone(), RootMeta::new(now));
8843 assert!(!process_has_been_idle(now, &live_roots, &executor));
8844
8845 let active = live_roots
8846 .get_mut(&active_root)
8847 .expect("active root metadata");
8848 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8849 active.active_bash_waits = 1;
8850 assert!(!process_has_been_idle(now, &live_roots, &executor));
8851 }
8852
8853 #[test]
8854 fn pressure_relief_log_reports_before_and_after_measurements() {
8855 let allocator = crate::memory::AllocatorMemorySnapshot {
8856 status: "measured",
8857 bytes_in_use: Some(8 * 1024 * 1024),
8858 size_allocated: Some(12 * 1024 * 1024),
8859 retained_slack_bytes: Some(4 * 1024 * 1024),
8860 not_estimated: None,
8861 };
8862 let relief = crate::memory::AllocatorPressureRelief {
8863 bytes_released: 3 * 1024 * 1024,
8864 rss_before_bytes: Some(20 * 1024 * 1024),
8865 rss_after_bytes: Some(17 * 1024 * 1024),
8866 allocator_before: allocator.clone(),
8867 allocator_after: crate::memory::AllocatorMemorySnapshot {
8868 size_allocated: Some(9 * 1024 * 1024),
8869 retained_slack_bytes: Some(1024 * 1024),
8870 ..allocator
8871 },
8872 };
8873 let message = pressure_relief_label(&relief);
8874 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8875 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8876 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8877 assert!(message.contains("reported 3.0 MB released"));
8878 }
8879
8880 #[test]
8881 fn due_maintenance_jobs_skip_poisoned_roots() {
8882 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8883 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8884 let mut live_roots = HashMap::new();
8885 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8886 let mut poisoned_meta = RootMeta::new(Instant::now());
8887 poisoned_meta.maintenance_poisoned = true;
8888 live_roots.insert(poisoned_root.clone(), poisoned_meta);
8889
8890 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8891 &mut live_roots,
8892 MAINTENANCE_SUBMIT_BUDGET,
8893 &HashSet::new(),
8894 );
8895
8896 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8897 assert!(due.iter().all(|(root, _)| root == &healthy_root));
8898 assert!(!deferred);
8899 assert!(live_roots[&healthy_root].maintenance_pending);
8900 assert_eq!(
8901 live_roots[&healthy_root].maintenance_jobs_in_flight,
8902 INITIAL_MAINTENANCE_JOB_COUNT
8903 );
8904 assert!(!live_roots[&poisoned_root].maintenance_pending);
8905 }
8906
8907 #[test]
8908 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8909 let (_dir, root) = test_root("maintenance-unbound");
8910 let mut meta = RootMeta::new(Instant::now());
8911 meta.unbound_quiesced = true;
8912 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8913
8914 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8915 &mut live_roots,
8916 MAINTENANCE_SUBMIT_BUDGET,
8917 &HashSet::new(),
8918 );
8919
8920 assert!(due.is_empty());
8921 assert!(!deferred);
8922 assert!(!live_roots[&root].maintenance_pending);
8923 }
8924
8925 #[test]
8926 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8927 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8928 let ctx = test_ctx();
8929 assert!(!ctx.completion_drains_have_work());
8930
8931 let executor = Executor::new();
8932 assert!(executor.register_actor(root.clone(), ctx));
8933 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8934 let session = "idle-session".to_string();
8935 let channel = route_key(17, 1);
8936 let metrics = DispatchPathMetrics::new();
8937 let bg_sub_by_session =
8938 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8939 let mut bg_wake_pending = BgWakePending::new();
8940
8941 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8942 &mut live_roots,
8943 Some(&executor),
8944 &bg_sub_by_session,
8945 &bg_wake_pending,
8946 MAINTENANCE_SUBMIT_BUDGET,
8947 &HashSet::new(),
8948 );
8949 assert!(idle_tick_jobs.is_empty());
8950 assert!(!deferred);
8951 assert!(!live_roots[&root].maintenance_pending);
8952
8953 let mut bg_wake_epoch = HashMap::new();
8956 push::arm_bg_wake(
8957 root.clone(),
8958 session,
8959 channel,
8960 &mut bg_wake_pending,
8961 &mut bg_wake_epoch,
8962 &metrics,
8963 );
8964 let (next_tick_jobs, deferred) = due_maintenance_jobs(
8965 &mut live_roots,
8966 Some(&executor),
8967 &bg_sub_by_session,
8968 &bg_wake_pending,
8969 MAINTENANCE_SUBMIT_BUDGET,
8970 &HashSet::new(),
8971 );
8972 assert_eq!(
8973 next_tick_jobs,
8974 vec![(root, MaintenanceDrainKind::CompletionDrains)]
8975 );
8976 assert!(!deferred);
8977 }
8978
8979 async fn assert_slow_configure_tail_admission(
8980 config: crate::executor::ExecutorConfig,
8981 shape: &'static str,
8982 ) {
8983 let root_dir = tempfile::tempdir().unwrap();
8984 let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8985 let root = ProjectRootId::from_path(&root_path).unwrap();
8986 let ctx = test_ctx();
8987 ctx.mark_subc_bound();
8988 let storage_root = ctx.storage_dir();
8989 ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8990 generation: ctx.configure_generation(),
8991 root_path: root_path.clone(),
8992 canonical_cache_root: root_path.clone(),
8993 harness: crate::harness::Harness::Opencode,
8994 storage_root: storage_root.clone(),
8995 harness_dir: storage_root.join("opencode"),
8996 session_id: "first-search-admission".to_string(),
8997 home_match: false,
8998 format_tool_cache_clear_needed: false,
8999 run_bash_replay: false,
9000 refresh_project_runtime: false,
9001 sync_bash_compress_flag: false,
9002 reset_filter_registry: false,
9003 clear_failed_spawns: false,
9004 warm_callgraph_store: false,
9005 supersede_search_artifact_persistence: false,
9006 supersede_callgraph_artifact_persistence: false,
9007 supersede_semantic_artifact_persistence: false,
9008 search_artifact_load_start: None,
9009 semantic_artifact_load_start: None,
9010 })
9011 .expect("queue configure maintenance");
9012 let (_gate, maintenance_reached, release_maintenance) =
9013 crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9014 root_path.clone(),
9015 );
9016
9017 let expected_pool_size = config.pool_size;
9018 let expected_actor_cap = config.actor_cap;
9019 let executor = Arc::new(Executor::with_config(config));
9020 assert_eq!(executor.pool_size(), expected_pool_size);
9021 assert_eq!(executor.actor_cap(), expected_actor_cap);
9022 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9023 let metrics = Arc::new(DispatchPathMetrics::new());
9024 let (completion_tx, mut completion_rx) = mpsc::channel(2);
9025 submit_maintenance_job(
9026 &executor,
9027 root.clone(),
9028 MaintenanceDrainKind::ConfigureTail,
9029 Vec::new(),
9030 &completion_tx,
9031 &metrics,
9032 );
9033 maintenance_reached
9034 .recv_timeout(Duration::from_secs(2))
9035 .expect("configure tail reached gate");
9036
9037 let admission_started = std::time::Instant::now();
9038 let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9039 let search = executor.submit_async(
9040 root.clone(),
9041 Lane::HeavyInit,
9042 "first-search".to_string(),
9043 Box::new(move |_ctx| {
9044 search_admitted_tx
9045 .send(())
9046 .expect("signal search admission");
9047 Response::success("first-search", json!({}))
9048 }),
9049 );
9050 let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9051 let mutation = executor.submit_async(
9052 root,
9053 Lane::Mutating,
9054 "queued-mutation".to_string(),
9055 Box::new(move |_ctx| {
9056 mutation_started_tx.send(()).expect("signal mutation start");
9057 Response::success("queued-mutation", json!({}))
9058 }),
9059 );
9060
9061 let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9067 let admission_elapsed = admission_started.elapsed();
9068 let mutation_waited = mutation_started_rx.try_recv().is_err();
9069 release_maintenance
9070 .send(())
9071 .expect("release configure maintenance");
9072 search_admission
9073 .expect("first search must admit while configure maintenance remains gated");
9074 eprintln!(
9075 "first-search admission while configure maintenance is gated ({shape}): {}ms",
9076 admission_elapsed.as_millis()
9077 );
9078 assert!(
9079 mutation_waited,
9080 "mutating work must wait for configure maintenance to release its read epoch"
9081 );
9082 tokio::time::timeout(Duration::from_secs(5), search)
9083 .await
9084 .expect("first search completion timed out")
9085 .expect("first search completion channel closed");
9086 tokio::time::timeout(Duration::from_secs(5), mutation)
9087 .await
9088 .expect("mutation completion timed out")
9089 .expect("mutation completion channel closed");
9090 tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9091 .await
9092 .expect("configure-tail completion timed out")
9093 .expect("configure-tail completion channel closed");
9094 }
9095
9096 #[tokio::test]
9097 async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9098 assert_slow_configure_tail_admission(
9099 crate::executor::ExecutorConfig {
9100 pool_size: 2,
9101 read_cap: 1,
9102 actor_cap: 1,
9103 heavy_permits: 1,
9104 drr_quantum: 1,
9105 },
9106 "pool=2 actor_cap=1",
9107 )
9108 .await;
9109 assert_slow_configure_tail_admission(
9110 crate::executor::ExecutorConfig {
9111 pool_size: 4,
9112 read_cap: 3,
9113 actor_cap: 3,
9114 heavy_permits: 3,
9115 drr_quantum: 1,
9116 },
9117 "pool=4 actor_cap=3",
9118 )
9119 .await;
9120 }
9121
9122 #[tokio::test]
9123 async fn subc_configure_tail_precedes_completed_search_install() {
9124 let root_dir = tempfile::tempdir().unwrap();
9125 let storage = tempfile::tempdir().unwrap();
9126 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9127 let (ctx, ignored_path) =
9128 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9129 let ctx = Arc::new(ctx);
9130 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9131 &ctx,
9132 &ignored_path
9133 ));
9134
9135 let executor = Arc::new(Executor::new());
9136 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9137 let metrics = Arc::new(DispatchPathMetrics::new());
9138 let (completion_tx, mut completion_rx) = mpsc::channel(4);
9139 submit_maintenance_job(
9140 &executor,
9141 root.clone(),
9142 MaintenanceDrainKind::ConfigureTail,
9143 Vec::new(),
9144 &completion_tx,
9145 &metrics,
9146 );
9147 submit_maintenance_job(
9148 &executor,
9149 root,
9150 MaintenanceDrainKind::CompletionDrains,
9151 Vec::new(),
9152 &completion_tx,
9153 &metrics,
9154 );
9155
9156 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9157 .await
9158 .expect("configure-tail completion timed out")
9159 .expect("configure-tail completion channel closed");
9160 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9161 .await
9162 .expect("completion-drains completion timed out")
9163 .expect("completion-drains completion channel closed");
9164 assert!(first.response.id.contains("configure-tail"));
9165 assert!(second.response.id.contains("completion-drains"));
9166 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9167 &ctx,
9168 &ignored_path
9169 ));
9170 assert_eq!(
9171 ctx.search_index()
9172 .read()
9173 .unwrap_or_else(std::sync::PoisonError::into_inner)
9174 .as_ref()
9175 .expect("completed search index installed")
9176 .file_count(),
9177 0,
9178 "configure must install the ignore matcher before pending paths replay"
9179 );
9180 ctx.stop_watcher_runtime();
9181 }
9182
9183 #[test]
9184 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9185 let (_dir, root) = test_root("maintenance-post-bind");
9186 let mut live_roots = HashMap::new();
9187 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9188
9189 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9190 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9191
9192 let meta = live_roots.get(&root).expect("root metadata");
9193 assert!(meta.maintenance_pending);
9194 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9195 assert_eq!(
9196 meta.maintenance_queued_kinds
9197 .iter()
9198 .copied()
9199 .collect::<Vec<_>>(),
9200 vec![
9201 MaintenanceDrainKind::ConfigureTail,
9202 MaintenanceDrainKind::CompletionDrains,
9203 ]
9204 );
9205
9206 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9207 &mut live_roots,
9208 MAINTENANCE_SUBMIT_BUDGET,
9209 &HashSet::new(),
9210 );
9211
9212 assert_eq!(
9213 due,
9214 vec![
9215 (root.clone(), MaintenanceDrainKind::ConfigureTail),
9216 (root.clone(), MaintenanceDrainKind::CompletionDrains),
9217 ]
9218 );
9219 assert!(!deferred);
9220 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9221 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9222 }
9223
9224 #[test]
9225 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9226 let mut live_roots = HashMap::new();
9227 let mut root_ids = Vec::new();
9228 let mut _dirs = Vec::new();
9229 for index in 0..4 {
9230 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9231 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9232 root_ids.push(root_id);
9233 _dirs.push(dir);
9234 }
9235
9236 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9237 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9238 &mut live_roots,
9239 small_budget,
9240 &HashSet::new(),
9241 );
9242
9243 assert_eq!(first_due.len(), small_budget);
9244 assert!(first_deferred);
9245 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9246 assert!(first_due_set
9247 .iter()
9248 .all(|root| live_roots[root].maintenance_pending));
9249 assert!(first_due_set
9250 .iter()
9251 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9252
9253 let all_roots: HashSet<_> = root_ids.into_iter().collect();
9254 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9255 assert!(deferred_roots
9256 .iter()
9257 .all(|root| !live_roots[root].maintenance_pending));
9258 }
9259
9260 #[test]
9261 fn due_maintenance_jobs_defers_pending_bind_roots() {
9262 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9263 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9264 let mut live_roots = HashMap::new();
9265 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9266 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9267 let pending_bind_roots = HashSet::from([bind_root.clone()]);
9268
9269 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9270 &mut live_roots,
9271 usize::MAX,
9272 &pending_bind_roots,
9273 );
9274
9275 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9276 assert!(due.iter().all(|(root, _)| root == &healthy_root));
9277 assert!(!deferred);
9278 assert!(!live_roots[&bind_root].maintenance_pending);
9279 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9280 }
9281
9282 #[test]
9283 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9284 let (_dir, root) = test_root("maintenance-requeue");
9285 let mut live_roots = HashMap::new();
9286 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9287 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9288 &mut live_roots,
9289 usize::MAX,
9290 &HashSet::new(),
9291 );
9292 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9293 assert!(due.iter().all(|(due_root, _)| due_root == &root));
9294 assert!(!deferred);
9295
9296 let meta = live_roots.get_mut(&root).unwrap();
9297 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9298 assert!(meta.maintenance_pending);
9299 assert_eq!(
9300 meta.maintenance_jobs_in_flight,
9301 INITIAL_MAINTENANCE_JOB_COUNT - 1
9302 );
9303 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9304
9305 let (requeued, deferred) =
9306 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9307 assert_eq!(
9308 requeued,
9309 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9310 );
9311 assert!(!deferred);
9312 let meta = live_roots.get_mut(&root).unwrap();
9313 assert_eq!(
9314 meta.maintenance_jobs_in_flight,
9315 INITIAL_MAINTENANCE_JOB_COUNT
9316 );
9317 assert!(meta.maintenance_queued_kinds.is_empty());
9318
9319 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9320 note_maintenance_completion(meta, None, false, false);
9321 }
9322 assert!(!meta.maintenance_pending);
9323 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9324 }
9325
9326 #[test]
9327 fn maintenance_requeue_drops_while_bind_is_pending() {
9328 let (_dir, root) = test_root("maintenance-bind-requeue");
9329 let mut live_roots = HashMap::new();
9330 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9331 let (due, _) = due_maintenance_jobs_without_actor_context(
9332 &mut live_roots,
9333 usize::MAX,
9334 &HashSet::new(),
9335 );
9336 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9337
9338 let meta = live_roots.get_mut(&root).unwrap();
9339 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9340
9341 assert_eq!(
9342 meta.maintenance_jobs_in_flight,
9343 INITIAL_MAINTENANCE_JOB_COUNT - 1
9344 );
9345 assert!(meta.maintenance_queued_kinds.is_empty());
9346 assert!(meta.maintenance_pending);
9347 }
9348
9349 #[test]
9350 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9351 let mut meta = RootMeta::new(Instant::now());
9352 meta.unbound_quiesced = true;
9353
9354 assert!(!should_requiesce_after_maintenance(
9355 &meta,
9356 MaintenanceDrainKind::Lsp,
9357 false,
9358 ));
9359 assert!(!should_requiesce_after_maintenance(
9360 &meta,
9361 MaintenanceDrainKind::ConfigureTail,
9362 true,
9363 ));
9364 assert!(should_requiesce_after_maintenance(
9365 &meta,
9366 MaintenanceDrainKind::ConfigureTail,
9367 false,
9368 ));
9369 }
9370
9371 #[test]
9372 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9373 let (_dir, root) = test_root("maintenance-fatal");
9374 let mut live_roots = HashMap::new();
9375 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9376 let (due, _) = due_maintenance_jobs_without_actor_context(
9377 &mut live_roots,
9378 usize::MAX,
9379 &HashSet::new(),
9380 );
9381 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9382
9383 let meta = live_roots.get_mut(&root).unwrap();
9384 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9385 assert!(meta.maintenance_poisoned);
9386 assert!(meta.maintenance_queued_kinds.is_empty());
9387
9388 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9389 note_maintenance_completion(meta, None, false, false);
9390 }
9391 assert!(!meta.maintenance_pending);
9392 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9393 }
9394
9395 #[test]
9396 fn trust_for_principal_matrix() {
9397 assert_eq!(
9398 trust_for_principal(&Some(Principal::Direct)),
9399 BindTrust::FirstParty
9400 );
9401 for module_id in [
9408 "llm-runner",
9409 "aft",
9410 "broca",
9411 "alfonso-core",
9412 "prefrontal",
9413 "prefrontal-core",
9414 ] {
9415 assert_eq!(
9416 trust_for_principal(&Some(Principal::Reserved {
9417 module_id: module_id.to_string(),
9418 })),
9419 BindTrust::FirstParty,
9420 "reserved module id '{module_id}' must resolve to first-party trust"
9421 );
9422 }
9423 assert_eq!(
9424 trust_for_principal(&Some(Principal::Reserved {
9425 module_id: "subc-mcp".to_string(),
9426 })),
9427 BindTrust::Untrusted
9428 );
9429 assert_eq!(
9430 trust_for_principal(&Some(Principal::Reserved {
9431 module_id: "anything-unknown".to_string(),
9432 })),
9433 BindTrust::Untrusted
9434 );
9435 assert_eq!(
9436 trust_for_principal(&Some(Principal::Unverified)),
9437 BindTrust::Untrusted
9438 );
9439 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9440 }
9441
9442 #[test]
9443 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9444 let principal = Some(Principal::Direct);
9445 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9446 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9447
9448 assert_eq!(
9449 trust_for_bind(fingerprint_a, &principal),
9450 BindTrust::Untrusted
9451 );
9452 assert_eq!(
9453 trust_for_bind(fingerprint_b, &principal),
9454 BindTrust::Untrusted
9455 );
9456 }
9457
9458 #[test]
9465 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9466 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9467 assert_eq!(
9468 trust_for_bind(harness, &Some(Principal::Direct)),
9469 BindTrust::FirstParty,
9470 "a direct principal must stay first-party on {harness}"
9471 );
9472 assert_eq!(
9473 trust_for_bind(harness, &Some(Principal::Unverified)),
9474 BindTrust::Untrusted,
9475 "an unverified principal must stay untrusted on {harness}"
9476 );
9477 assert_eq!(
9478 trust_for_bind(harness, &None),
9479 BindTrust::Untrusted,
9480 "an absent principal must fail closed on {harness}"
9481 );
9482 assert_eq!(
9483 trust_for_bind(
9484 harness,
9485 &Some(Principal::Reserved {
9486 module_id: "subc-mcp".to_string(),
9487 })
9488 ),
9489 BindTrust::Untrusted,
9490 "a non-allowlisted reserved module must stay untrusted on {harness}"
9491 );
9492 }
9493 }
9494
9495 #[tokio::test]
9496 async fn persistent_cancel_resolves_when_fired_before_await() {
9497 let signal = PersistentCancelSignal::new();
9501 signal.cancel();
9502 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9504 .await
9505 .expect("cancelled() must resolve when cancel fired beforehand");
9506
9507 let racing = PersistentCancelSignal::new();
9509 let racing_for_task = racing.clone();
9510 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9511 racing.cancel();
9512 tokio::time::timeout(Duration::from_secs(1), waiter)
9513 .await
9514 .expect("cancelled() must resolve when cancel races the await")
9515 .expect("waiter task panicked");
9516 }
9517
9518 #[test]
9519 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9520 let installed = HashMap::from([(7, 9)]);
9521 let mut reclaimed = ReclaimedRoutes::default();
9522 reclaimed.insert(route_key(8, 1));
9523 for ty in [
9524 FrameType::Request,
9525 FrameType::Response,
9526 FrameType::Error,
9527 FrameType::Push,
9528 FrameType::Cancel,
9529 FrameType::Goodbye,
9530 ] {
9531 let body = if ty.is_pure_header() {
9532 Vec::new()
9533 } else {
9534 br#"{}"#.to_vec()
9535 };
9536 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9537 assert!(
9538 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9539 "{ty:?}"
9540 );
9541 }
9542
9543 let reclaimed_request = Frame::build(
9544 FrameType::Request,
9545 control_flags(),
9546 8,
9547 1,
9548 42,
9549 br#"{}"#.to_vec(),
9550 )
9551 .unwrap();
9552 assert!(ingress_route_should_be_processed(
9553 &installed,
9554 &reclaimed,
9555 &reclaimed_request
9556 ));
9557
9558 let never_installed = Frame::build(
9559 FrameType::Request,
9560 control_flags(),
9561 9,
9562 1,
9563 43,
9564 br#"{}"#.to_vec(),
9565 )
9566 .unwrap();
9567 assert!(!ingress_route_should_be_processed(
9568 &installed,
9569 &reclaimed,
9570 &never_installed
9571 ));
9572
9573 let current = Frame::build(
9574 FrameType::Request,
9575 control_flags(),
9576 7,
9577 9,
9578 43,
9579 br#"{}"#.to_vec(),
9580 )
9581 .unwrap();
9582 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9583 assert!(ingress_route_should_be_processed(
9584 &installed, &reclaimed, ¤t
9585 ));
9586 assert!(ingress_route_should_be_processed(
9587 &installed, &reclaimed, &control
9588 ));
9589 assert_eq!(installed, HashMap::from([(7, 9)]));
9590 }
9591
9592 #[tokio::test]
9593 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9594 let (_dir, root) = test_root("route-bind-b2-ordering");
9595 let route = route_key(7, 3);
9596 let identity = RouteIdentity(Arc::new(RouteIdentityData {
9597 root: root.clone(),
9598 project_root: root.as_path().to_path_buf(),
9599 harness: "opencode".to_string(),
9600 session: "b2-session".to_string(),
9601 trust: BindTrust::FirstParty,
9602 spawn_principal: AuthenticatedPrincipal::FirstParty,
9603 consumer_elicitation_capable: false,
9604 }));
9605 let replay_key = push::ReplayKey::from_identity(&identity);
9606 let completion = RouteBindCompletion {
9607 route,
9608 identity,
9609 bind_root_id: root.clone(),
9610 inserted_new_actor: false,
9611 configure_response: Response::success("subc-bind-7", json!({})),
9612 diagnostics_on_edit: false,
9613 ver: PROTOCOL_VERSION,
9614 corr: 91,
9615 flags: control_flags(),
9616 };
9617 let mut pending_binds = HashMap::from([(
9618 route,
9619 PendingBind {
9620 bind_root_id: root,
9621 inserted_new_actor: false,
9622 cancelled: false,
9623 configure_request_id: "subc-bind-7".to_string(),
9624 started_at: Instant::now(),
9625 warned_half_deadline: false,
9626 deadline_reported: false,
9627 corr: 91,
9628 ver: PROTOCOL_VERSION,
9629 flags: control_flags(),
9630 cancellation: crate::executor::JobCancellation::new(),
9631 },
9632 )]);
9633 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9634 let mut push_buffer =
9635 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9636 let (writer_tx, mut writer_rx) = mpsc::channel(8);
9637 let metrics = Arc::new(DispatchPathMetrics::new());
9638 let executor = Arc::new(Executor::new());
9639 let standing_actor =
9640 standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9641
9642 handle_route_bind_completion(
9643 &writer_tx,
9644 completion,
9645 &mut HashMap::new(),
9646 &mut HashMap::new(),
9647 &mut HashMap::new(),
9648 &mut push_buffer,
9649 &mut HashMap::new(),
9650 &mut pending_binds,
9651 &mut installed_route_epochs,
9652 &executor,
9653 &standing_actor,
9654 &Arc::new(Notify::new()),
9655 &metrics,
9656 None,
9657 )
9658 .await
9659 .unwrap();
9660
9661 let ack = writer_rx.try_recv().expect("RouteBindAck");
9662 assert_eq!(ack.header.ty, FrameType::Response);
9663 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9664 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9665 assert_eq!(route_frame.header.ty, FrameType::Push);
9666 assert_eq!(
9667 (route_frame.header.channel, route_frame.header.epoch),
9668 (route.channel, route.epoch)
9669 );
9670 }
9671}