1pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38 finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39 PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46 ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47 StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51 MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55 MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71const HELLO_CORR: u64 = 1;
73
74const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106 MaintenanceDrainKind::Watcher,
107 MaintenanceDrainKind::Lsp,
108 MaintenanceDrainKind::ConfigureTail,
109 MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135 channel: u16,
136 epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 write!(f, "{}@{}", self.channel, self.epoch)
142 }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156 build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157 DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158 ReapBlockerCensus, ResponseTaskGuard,
159};
160use self::manifest::{
161 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
162 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
163};
164pub use self::wire::SubcError;
165
166#[doc(hidden)]
171#[derive(Clone, Debug, PartialEq, Eq)]
172pub enum SubcLifecycleEvent {
173 AttachDecision {
174 attempt: u32,
175 will_retry: bool,
176 },
177 RouteDetached {
178 route_channel: u16,
179 route_epoch: u32,
180 session_id: String,
181 },
182 ReliableCompletionRetained {
183 task_id: String,
184 session_id: String,
185 },
186 ReliableCompletionReplayed {
187 route_channel: u16,
188 route_epoch: u32,
189 task_id: String,
190 session_id: String,
191 },
192}
193
194#[doc(hidden)]
196#[derive(Clone)]
197pub struct SubcTestLifecycleProbe {
198 events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
199}
200
201impl SubcTestLifecycleProbe {
202 #[doc(hidden)]
203 pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
204 Self { events_tx }
205 }
206
207 fn attach_decision(&self, attempt: u32, will_retry: bool) {
208 let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
209 attempt,
210 will_retry,
211 });
212 }
213
214 fn route_detached(&self, route: RouteChannel, session_id: &str) {
215 let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
216 route_channel: route.channel,
217 route_epoch: route.epoch,
218 session_id: session_id.to_string(),
219 });
220 }
221
222 fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
223 let _ = self
224 .events_tx
225 .send(SubcLifecycleEvent::ReliableCompletionRetained {
226 task_id: task_id.to_string(),
227 session_id: session_id.to_string(),
228 });
229 }
230
231 fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
232 let _ = self
233 .events_tx
234 .send(SubcLifecycleEvent::ReliableCompletionReplayed {
235 route_channel: route.channel,
236 route_epoch: route.epoch,
237 task_id: task_id.to_string(),
238 session_id: session_id.to_string(),
239 });
240 }
241}
242
243pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
247 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
248}
249use self::wire::{
250 build_error_frame, build_goodbye_frame, build_tool_response_frame,
251 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
252 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
253 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
254};
255
256struct DecodedFrame {
257 frame: Frame,
258 phase_trace: PhaseTrace,
259}
260
261struct ToolCallCompletion {
262 text: String,
263 phase_trace: PhaseTrace,
264}
265
266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
267enum RouteDetachPolicy {
268 RetainForReplay,
269 CancelOnDetach,
270}
271
272#[derive(Clone)]
273struct ActiveToolCall {
274 root_id: ProjectRootId,
275 cancellation: JobCancellation,
276 detach_policy: RouteDetachPolicy,
277}
278
279type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
280
281struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
282
283impl PendingDeferredSetupGuard {
284 fn new(count: Arc<AtomicUsize>) -> Self {
285 count.fetch_add(1, Ordering::SeqCst);
286 Self(count)
287 }
288}
289
290impl Drop for PendingDeferredSetupGuard {
291 fn drop(&mut self) {
292 self.0.fetch_sub(1, Ordering::SeqCst);
293 }
294}
295
296enum DeferredSetupOutcome {
297 Immediate {
298 text: String,
299 phase_trace: PhaseTrace,
300 },
301 Deferred {
302 pending: PendingResponse,
303 surface_downgraded: bool,
304 phase_trace: PhaseTrace,
305 },
306}
307
308struct PendingSubcResponse {
309 route: RouteChannel,
310 corr: u64,
311 flags: Flags,
312 ver: u8,
313 root: ProjectRootId,
314 session_id: String,
315 bare_name: String,
316 format_context: crate::subc_format::FormatContext,
317 bind_trust: BindTrust,
318 pending: PendingResponse,
319 surface_downgraded: bool,
320 phase_trace: PhaseTrace,
321}
322
323struct ResolvedSubcResponse {
324 entry: PendingSubcResponse,
325 response: Response,
326}
327
328#[derive(Default)]
329struct PendingSubcResponses {
330 entries: Vec<PendingSubcResponse>,
331}
332
333impl PendingSubcResponses {
334 fn register(&mut self, pending: PendingSubcResponse) {
335 self.entries.retain(|entry| {
336 let keep = entry.route != pending.route || entry.corr != pending.corr;
337 if !keep {
338 if let Some(cancellation) = &entry.pending.cancellation {
339 cancellation.request_cancel();
340 }
341 }
342 keep
343 });
344 self.entries.push(pending);
345 }
346
347 fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
348 let mut ready = Vec::new();
349 let mut waiting = Vec::with_capacity(self.entries.len());
350 for mut entry in self.entries.drain(..) {
351 let response = executor
352 .actor_context(&entry.root)
353 .and_then(|ctx| (entry.pending.poll)(&ctx));
354 if let Some(response) = response {
355 ready.push(ResolvedSubcResponse { entry, response });
356 } else {
357 waiting.push(entry);
358 }
359 }
360 self.entries = waiting;
361 ready
362 }
363
364 fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
365 let mut cancelled = false;
366 self.entries.retain(|entry| {
367 let keep = entry.route != route || entry.corr != corr;
368 if !keep {
369 cancelled = true;
370 if let Some(cancellation) = &entry.pending.cancellation {
371 cancellation.request_cancel();
372 }
373 }
374 keep
375 });
376 cancelled
377 }
378
379 fn drain_route(
380 &mut self,
381 route: RouteChannel,
382 executor: &Executor,
383 ) -> Vec<ResolvedSubcResponse> {
384 self.drain_matching(executor, |entry| entry.route == route)
385 }
386
387 fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
388 self.drain_matching(executor, |_| true)
389 }
390
391 fn drain_matching(
392 &mut self,
393 executor: &Executor,
394 matches: impl Fn(&PendingSubcResponse) -> bool,
395 ) -> Vec<ResolvedSubcResponse> {
396 let mut resolved = Vec::new();
397 let mut waiting = Vec::with_capacity(self.entries.len());
398 for mut entry in self.entries.drain(..) {
399 if !matches(&entry) {
400 waiting.push(entry);
401 continue;
402 }
403 if let Some(cancellation) = &entry.pending.cancellation {
404 cancellation.request_cancel();
405 }
406 if let Some(ctx) = executor.actor_context(&entry.root) {
407 if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
408 let response = on_shutdown(&ctx);
409 resolved.push(ResolvedSubcResponse { entry, response });
410 }
411 }
412 }
413 self.entries = waiting;
414 resolved
415 }
416
417 fn is_empty(&self) -> bool {
418 self.entries.is_empty()
419 }
420}
421
422#[derive(Clone)]
423struct PushSenders {
424 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
425 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
426 lossy_overflow: Arc<push::LossyOverflow>,
427 lossy_seq: Arc<AtomicU64>,
428 fleet_status_client: FleetStatusClient,
429}
430
431#[derive(Clone)]
432struct PersistentCancelSignal {
433 inner: Arc<PersistentCancelInner>,
434}
435
436struct PersistentCancelInner {
437 cancelled: AtomicBool,
438 notify: Notify,
439}
440
441impl PersistentCancelSignal {
442 fn new() -> Self {
443 Self {
444 inner: Arc::new(PersistentCancelInner {
445 cancelled: AtomicBool::new(false),
446 notify: Notify::new(),
447 }),
448 }
449 }
450
451 fn cancel(&self) {
452 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
453 self.inner.notify.notify_waiters();
454 }
455 }
456
457 fn is_cancelled(&self) -> bool {
458 self.inner.cancelled.load(Ordering::SeqCst)
459 }
460
461 async fn cancelled(&self) {
462 loop {
470 let notified = self.inner.notify.notified();
471 tokio::pin!(notified);
472 notified.as_mut().enable();
473 if self.is_cancelled() {
474 return;
475 }
476 notified.await;
477 }
478 }
479}
480
481fn submit_active_tool_call(
482 executor: &Executor,
483 active: &ActiveToolCalls,
484 route: RouteChannel,
485 corr: u64,
486 root_id: ProjectRootId,
487 lane: Lane,
488 request_id: String,
489 detach_policy: RouteDetachPolicy,
490 job: crate::executor::ExecutorJob,
491) -> oneshot::Receiver<Response> {
492 let (rx, cancellation) =
493 executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
494 active
495 .lock()
496 .unwrap_or_else(std::sync::PoisonError::into_inner)
497 .insert(
498 (route, corr),
499 ActiveToolCall {
500 root_id,
501 cancellation,
502 detach_policy,
503 },
504 );
505 rx
506}
507
508fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
509 active
510 .lock()
511 .unwrap_or_else(std::sync::PoisonError::into_inner)
512 .remove(&(route, corr))
513 .is_some()
514}
515
516fn active_tool_call_is_registered(
517 active: &ActiveToolCalls,
518 route: RouteChannel,
519 corr: u64,
520) -> bool {
521 active
522 .lock()
523 .unwrap_or_else(std::sync::PoisonError::into_inner)
524 .contains_key(&(route, corr))
525}
526
527fn cancel_active_tool_call(
528 active: &ActiveToolCalls,
529 executor: &Executor,
530 route: RouteChannel,
531 corr: u64,
532 reason: &str,
533) -> bool {
534 let call = active
535 .lock()
536 .unwrap_or_else(std::sync::PoisonError::into_inner)
537 .remove(&(route, corr));
538 let Some(call) = call else {
539 return false;
540 };
541 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
542 log::debug!(
543 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
544 );
545 true
546}
547
548#[derive(Clone, Copy, Debug, PartialEq, Eq)]
549enum RouteWorkDisposition {
550 RetainForReplay,
551 RetainStartedForReplay,
552 Abandon,
553}
554
555fn apply_route_work_disposition(
556 active: &ActiveToolCalls,
557 executor: &Executor,
558 route: RouteChannel,
559 disposition: RouteWorkDisposition,
560 reason: &str,
561) -> usize {
562 if disposition != RouteWorkDisposition::Abandon {
563 let route_calls = active
564 .lock()
565 .unwrap_or_else(std::sync::PoisonError::into_inner)
566 .iter()
567 .filter(|((call_route, _), _)| *call_route == route)
568 .map(|(key, call)| (*key, call.clone()))
569 .collect::<Vec<_>>();
570 let mut retained = 0usize;
571 let mut cancelled_before_execution = 0usize;
572 let mut cancelled_terminal = 0usize;
573
574 for (key, call) in route_calls {
575 let remove = match (call.detach_policy, disposition) {
576 (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
577 retained += 1;
578 false
579 }
580 (
581 RouteDetachPolicy::RetainForReplay,
582 RouteWorkDisposition::RetainStartedForReplay,
583 ) => {
584 match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
585 PreExecutionCancelOutcome::AlreadyStarted => {
586 retained += 1;
587 false
588 }
589 PreExecutionCancelOutcome::QueuedRemoved
590 | PreExecutionCancelOutcome::DispatchedCancelled => {
591 cancelled_before_execution += 1;
592 true
593 }
594 }
595 }
596 (RouteDetachPolicy::CancelOnDetach, _) => {
597 executor.cancel_job(&call.root_id, &call.cancellation);
598 cancelled_terminal += 1;
599 true
600 }
601 (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
602 };
603 if remove {
604 active
605 .lock()
606 .unwrap_or_else(std::sync::PoisonError::into_inner)
607 .remove(&key);
608 }
609 }
610 log::debug!(
611 "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
612 );
613 return retained;
614 }
615
616 let cancelled = {
617 let mut calls = active
618 .lock()
619 .unwrap_or_else(std::sync::PoisonError::into_inner);
620 let mut cancelled = Vec::new();
621 calls.retain(|(call_route, _), call| {
622 if *call_route == route {
623 cancelled.push(call.clone());
624 false
625 } else {
626 true
627 }
628 });
629 cancelled
630 };
631 for call in &cancelled {
632 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
633 log::debug!(
634 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
635 );
636 }
637 cancelled.len()
638}
639
640fn cancel_all_active_tool_calls(
641 active: &ActiveToolCalls,
642 executor: &Executor,
643 reason: &str,
644) -> usize {
645 let cancelled = {
646 let mut calls = active
647 .lock()
648 .unwrap_or_else(std::sync::PoisonError::into_inner);
649 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
650 };
651 for call in &cancelled {
652 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
653 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
654 }
655 cancelled.len()
656}
657
658#[derive(Debug, Clone, Copy, PartialEq, Eq)]
659pub enum BindTrust {
660 FirstParty,
661 Untrusted,
662}
663
664impl BindTrust {
665 fn allows_bash_observation(self) -> bool {
666 matches!(self, Self::FirstParty)
667 }
668
669 fn label(self) -> &'static str {
670 match self {
671 Self::FirstParty => "first_party",
672 Self::Untrusted => "untrusted",
673 }
674 }
675
676 fn sandbox_trust(self) -> PrincipalTrust {
677 match self {
678 Self::FirstParty => PrincipalTrust::FirstParty,
679 Self::Untrusted => PrincipalTrust::Untrusted,
680 }
681 }
682}
683
684pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
685 match principal {
686 Some(Principal::Direct) => BindTrust::FirstParty,
687 Some(Principal::Reserved { module_id })
698 if module_id == "llm-runner"
699 || module_id == "aft"
700 || module_id == "broca"
701 || module_id == "alfonso-core"
702 || module_id == "prefrontal"
703 || module_id == "prefrontal-core" =>
704 {
705 BindTrust::FirstParty
706 }
707 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
708 BindTrust::Untrusted
709 }
710 }
711}
712
713fn harness_forces_untrusted(harness: &str) -> bool {
714 harness.starts_with("fed:")
715}
716
717pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
718 if harness_forces_untrusted(harness) {
719 BindTrust::Untrusted
720 } else {
721 trust_for_principal(principal)
722 }
723}
724
725fn principal_id(principal: &Option<Principal>) -> Option<String> {
726 match principal {
727 Some(Principal::Direct) => Some("direct".to_string()),
728 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
729 Some(Principal::Unverified) => Some("unverified".to_string()),
730 None => None,
731 }
732}
733
734fn principal_label(principal: &Option<Principal>) -> String {
735 principal_id(principal).unwrap_or_else(|| "absent".to_string())
736}
737
738#[derive(Debug)]
739struct RootMeta {
745 maintenance_pending: bool,
746 maintenance_jobs_in_flight: usize,
747 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
748 maintenance_last_submitted: Option<Instant>,
749 maintenance_poisoned: bool,
750 last_touched: Instant,
751 diagnostics_on_edit: bool,
752 active_bash_waits: usize,
753 idle_artifacts_evicted: bool,
754 unbound_quiesced: bool,
755 consecutive_missing_sweeps: u8,
756}
757
758#[derive(Debug)]
759struct PendingBind {
760 bind_root_id: ProjectRootId,
761 inserted_new_actor: bool,
762 cancelled: bool,
763 configure_request_id: String,
764 started_at: Instant,
765 warned_half_deadline: bool,
766 deadline_reported: bool,
767 corr: u64,
768 ver: u8,
769 flags: Flags,
770 cancellation: crate::executor::JobCancellation,
775}
776
777struct RouteBindCompletion {
778 route: RouteChannel,
779 identity: RouteIdentity,
780 bind_root_id: ProjectRootId,
781 inserted_new_actor: bool,
782 configure_response: Response,
783 diagnostics_on_edit: bool,
784 ver: u8,
785 corr: u64,
786 flags: Flags,
787}
788
789#[derive(Debug, Clone)]
790struct RouteIdentity(Arc<RouteIdentityData>);
791
792#[derive(Debug)]
793struct RouteIdentityData {
794 root: ProjectRootId,
795 project_root: PathBuf,
796 harness: String,
797 session: String,
798 trust: BindTrust,
799 spawn_principal: AuthenticatedPrincipal,
800 consumer_elicitation_capable: bool,
801}
802
803impl Deref for RouteIdentity {
804 type Target = RouteIdentityData;
805
806 fn deref(&self) -> &Self::Target {
807 &self.0
808 }
809}
810
811#[derive(Debug, Clone)]
812struct RetainedSessionIdentity {
813 harness: String,
814 trust: BindTrust,
815}
816
817#[derive(Clone)]
818struct BgSub {
819 corr: u64,
820 ver: u8,
821 flags: Flags,
822 root: ProjectRootId,
823 session: String,
824}
825
826#[derive(Clone, Copy, Debug)]
827struct BgWakeState {
828 next_nudge_at: Instant,
829 nudges_sent: u32,
830}
831
832impl BgWakeState {
833 fn armed(now: Instant) -> Self {
834 Self {
835 next_nudge_at: now,
836 nudges_sent: 0,
837 }
838 }
839}
840
841type BgWakePending = HashMap<RouteChannel, BgWakeState>;
842
843type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
846
847struct MaintenanceCompletion {
848 root_id: ProjectRootId,
849 kind: MaintenanceDrainKind,
850 response: Response,
851 empty_bg_sessions: Vec<(String, u64)>,
852 unacked_bg_keys: Option<HashSet<String>>,
853 requeue_kind: Option<MaintenanceDrainKind>,
854}
855
856#[derive(Clone, Copy, Debug, PartialEq, Eq)]
857enum MaintenanceDrainKind {
858 Watcher,
859 Lsp,
860 ConfigureTail,
861 CompletionDrains,
862}
863
864impl MaintenanceDrainKind {
865 fn label(self) -> &'static str {
866 match self {
867 Self::Watcher => "watcher",
868 Self::Lsp => "lsp",
869 Self::ConfigureTail => "configure-tail",
870 Self::CompletionDrains => "completion-drains",
871 }
872 }
873}
874
875#[derive(Debug, Default)]
876struct MaintenanceJobOutcome {
877 empty_bg_sessions: Vec<(String, u64)>,
878 unacked_bg_keys: Option<HashSet<String>>,
879 requeue_kind: Option<MaintenanceDrainKind>,
880}
881
882#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
883struct ReverseCorrKey {
884 route: RouteChannel,
885 corr: u64,
886}
887
888struct PendingBashAsk {
889 route: RouteChannel,
890 tool_corr: u64,
891 tool_flags: Flags,
892 tool_ver: u8,
893 root: ProjectRootId,
894 project_root: PathBuf,
895 session_id: String,
896 spawn_principal: AuthenticatedPrincipal,
897 edit_slot_survives: Option<bool>,
898 request_id: String,
899 arguments: Value,
900 format_context: crate::subc_format::FormatContext,
901 cancel: bash::BashWaitCancel,
902 grants: Vec<String>,
903 expires_at: Instant,
904}
905
906impl RootMeta {
907 fn new(now: Instant) -> Self {
908 Self {
909 maintenance_pending: false,
910 maintenance_jobs_in_flight: 0,
911 maintenance_queued_kinds: VecDeque::new(),
912 maintenance_last_submitted: None,
913 maintenance_poisoned: false,
914 last_touched: now,
915 diagnostics_on_edit: false,
916 active_bash_waits: 0,
917 idle_artifacts_evicted: false,
918 unbound_quiesced: false,
919 consecutive_missing_sweeps: 0,
920 }
921 }
922
923 fn note_activity(&mut self) {
924 self.last_touched = Instant::now();
925 }
926
927 fn reactivate_bound(&mut self) {
928 self.note_activity();
929 self.idle_artifacts_evicted = false;
930 self.unbound_quiesced = false;
931 }
932}
933
934fn due_maintenance_jobs(
935 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
936 executor: Option<&Executor>,
937 bg_sub_by_session: &BgSubsBySession,
938 bg_wake_pending: &BgWakePending,
939 budget: usize,
940 pending_bind_roots: &HashSet<ProjectRootId>,
941) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
942 let mut jobs = Vec::new();
943 let mut deferred = false;
944 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
945 roots.sort_by(|left, right| {
946 let left_last = live_roots
947 .get(left)
948 .and_then(|meta| meta.maintenance_last_submitted);
949 let right_last = live_roots
950 .get(right)
951 .and_then(|meta| meta.maintenance_last_submitted);
952 left_last
953 .cmp(&right_last)
954 .then_with(|| left.as_path().cmp(right.as_path()))
955 });
956
957 for root_id in roots {
958 let Some(meta) = live_roots.get_mut(&root_id) else {
959 continue;
960 };
961 if meta.maintenance_poisoned {
962 continue;
963 }
964
965 if pending_bind_roots.contains(&root_id) {
966 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
967 deferred = true;
968 }
969 continue;
970 }
971
972 if !meta.maintenance_pending {
973 if jobs.len() >= budget {
974 deferred = true;
975 continue;
976 }
977 let executor_actor_context =
981 executor.and_then(|executor| executor.actor_context(&root_id));
982 let root_has_pending_bg_wake =
983 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
984 sub_root == &root_id
985 && channels
986 .iter()
987 .any(|channel| bg_wake_pending.contains_key(channel))
988 });
989 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
990 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
991 .into_iter()
992 .filter(|kind| {
993 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
994 return false;
995 }
996 match kind {
997 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
998 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
999 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1000 MaintenanceDrainKind::CompletionDrains => {
1006 root_has_pending_bg_wake || ctx.completion_drains_have_work()
1007 }
1008 }
1009 })
1010 .collect(),
1011 None if meta.unbound_quiesced => Vec::new(),
1012 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1014 };
1015 if kinds_with_work.is_empty() {
1016 continue;
1017 }
1018 meta.maintenance_pending = true;
1019 meta.maintenance_queued_kinds.extend(kinds_with_work);
1020 }
1021
1022 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1023 if jobs.len() >= budget {
1024 meta.maintenance_queued_kinds.push_front(kind);
1025 deferred = true;
1026 break;
1027 }
1028 meta.maintenance_jobs_in_flight += 1;
1029 meta.maintenance_last_submitted = Some(Instant::now());
1030 jobs.push((root_id.clone(), kind));
1031 }
1032
1033 meta.maintenance_pending =
1034 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1035 }
1036
1037 (jobs, deferred)
1038}
1039
1040fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1041 match estimate.estimated_bytes {
1042 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1043 None if estimate.status == "busy" => "busy".to_string(),
1044 None => "not estimated".to_string(),
1045 }
1046}
1047
1048fn optional_memory_label(bytes: Option<u64>) -> String {
1049 bytes.map_or_else(
1050 || "not estimated".to_string(),
1051 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1052 )
1053}
1054
1055fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1056 format!(
1057 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1058 optional_memory_label(relief.rss_before_bytes),
1059 optional_memory_label(relief.rss_after_bytes),
1060 optional_memory_label(relief.allocator_before.bytes_in_use),
1061 optional_memory_label(relief.allocator_after.bytes_in_use),
1062 optional_memory_label(relief.allocator_before.size_allocated),
1063 optional_memory_label(relief.allocator_after.size_allocated),
1064 optional_memory_label(relief.allocator_before.retained_slack_bytes),
1065 optional_memory_label(relief.allocator_after.retained_slack_bytes),
1066 relief.bytes_released as f64 / (1024.0 * 1024.0),
1067 )
1068}
1069
1070fn idle_root_eviction_message(
1071 root_id: &ProjectRootId,
1072 memory: &crate::memory::RootMemorySnapshot,
1073 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1074) -> String {
1075 let freed_bytes = [
1078 &memory.semantic,
1079 &memory.trigram,
1080 &memory.symbols,
1081 &memory.callgraph,
1082 &memory.inspect,
1083 ]
1084 .iter()
1085 .filter_map(|estimate| estimate.estimated_bytes)
1086 .fold(0u64, u64::saturating_add);
1087 let mut message = format!(
1088 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1089 root_id.as_path().display(),
1090 freed_bytes as f64 / (1024.0 * 1024.0),
1091 eviction_estimate_label(&memory.semantic),
1092 eviction_estimate_label(&memory.trigram),
1093 eviction_estimate_label(&memory.symbols),
1094 eviction_estimate_label(&memory.callgraph),
1095 eviction_estimate_label(&memory.inspect),
1096 eviction_estimate_label(&memory.bash),
1097 eviction_estimate_label(&memory.lsp),
1098 eviction_estimate_label(&memory.parser_pool),
1099 );
1100 if let Some(pressure_relief) = pressure_relief {
1101 message.push_str(&pressure_relief_label(pressure_relief));
1102 }
1103 message
1104}
1105
1106fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1107 executor
1108 .actor_context(root_id)
1109 .map(|ctx| ctx.config().idle.root_ttl())
1110 .unwrap_or(IDLE_ROOT_TTL)
1111}
1112
1113fn process_has_been_idle(
1114 now: Instant,
1115 live_roots: &HashMap<ProjectRootId, RootMeta>,
1116 executor: &Executor,
1117) -> bool {
1118 !live_roots.is_empty()
1119 && live_roots.iter().all(|(root_id, meta)| {
1120 now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1121 && meta.active_bash_waits == 0
1122 && !meta.maintenance_pending
1123 && meta.maintenance_queued_kinds.is_empty()
1124 })
1125}
1126
1127fn allocator_pressure_relief_after_idle_sweep(
1128 now: Instant,
1129 live_roots: &HashMap<ProjectRootId, RootMeta>,
1130 executor: &Executor,
1131) -> Option<crate::memory::AllocatorPressureRelief> {
1132 if !process_has_been_idle(now, live_roots, executor)
1133 || live_roots.keys().any(|root_id| {
1134 executor
1135 .actor_context(root_id)
1136 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1137 })
1138 {
1139 return None;
1140 }
1141
1142 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1143 {
1144 Some(crate::memory::relieve_allocator_pressure())
1145 }
1146 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1147 {
1148 None
1149 }
1150}
1151
1152fn quiesce_unbound_root(
1153 root_id: &ProjectRootId,
1154 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1155 executor: &Arc<Executor>,
1156) {
1157 let Some(meta) = live_roots.get_mut(root_id) else {
1158 return;
1159 };
1160
1161 let ctx = executor.actor_context(root_id);
1162 if let Some(ctx) = ctx.as_ref() {
1163 ctx.mark_subc_unbound();
1167 ctx.bash_background()
1168 .replace_live_delivery_sessions(HashSet::new());
1169 }
1170 let cancelled = executor.cancel_queued_maintenance(root_id);
1171 let discarded = ctx
1179 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1180 .unwrap_or(0);
1181 meta.unbound_quiesced = true;
1182 meta.maintenance_queued_kinds.clear();
1183 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1184 log::info!(
1185 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1186 root_id.as_path().display(),
1187 cancelled,
1188 discarded
1189 );
1190}
1191
1192#[allow(clippy::too_many_arguments)]
1193fn quiesce_connection_roots(
1194 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1195 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1196 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1197 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1198 installed_route_epochs: &mut HashMap<u16, u32>,
1199 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1200 active_tool_calls: &ActiveToolCalls,
1201 executor: &Arc<Executor>,
1202) {
1203 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1204 for cancel in route_bash_cancels.values() {
1205 cancel.token.cancel();
1206 }
1207 route_bash_cancels.clear();
1208
1209 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1210 for pending in pending_binds.values_mut() {
1211 pending.cancelled = true;
1212 roots.insert(pending.bind_root_id.clone());
1213 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1214 }
1215
1216 for root_id in roots {
1220 if live_roots.contains_key(&root_id) {
1221 quiesce_unbound_root(&root_id, live_roots, executor);
1222 } else if let Some(ctx) = executor.actor_context(&root_id) {
1223 ctx.mark_subc_unbound();
1224 executor.cancel_queued_maintenance(&root_id);
1225 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1226 }
1227 }
1228
1229 routes.clear();
1230 root_channels.clear();
1231 installed_route_epochs.clear();
1232}
1233
1234#[derive(Debug, Default)]
1238struct ReclaimedRoutes {
1239 highest_epoch_by_channel: HashMap<u16, u32>,
1240}
1241
1242impl ReclaimedRoutes {
1243 fn insert(&mut self, route: RouteChannel) {
1244 self.highest_epoch_by_channel
1245 .entry(route.channel)
1246 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1247 .or_insert(route.epoch);
1248 }
1249
1250 fn contains(&self, route: RouteChannel) -> bool {
1251 self.highest_epoch_by_channel
1252 .get(&route.channel)
1253 .is_some_and(|epoch| route.epoch <= *epoch)
1254 }
1255}
1256
1257#[derive(Debug, Default)]
1258struct IdleReapOutcome {
1259 evicted: usize,
1260 forgotten_deleted_roots: Vec<ProjectRootId>,
1261}
1262
1263fn reap_idle_roots(
1264 now: Instant,
1265 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1266 pending_binds: &HashMap<RouteChannel, PendingBind>,
1267 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1268 executor: &Arc<Executor>,
1269 metrics: &DispatchPathMetrics,
1270) -> IdleReapOutcome {
1271 let pending_bind_roots = pending_binds
1272 .values()
1273 .map(|pending| pending.bind_root_id.clone())
1274 .collect::<HashSet<_>>();
1275 let mut census = ReapBlockerCensus::default();
1276 let mut candidates = Vec::new();
1277
1278 for (root_id, meta) in live_roots.iter_mut() {
1279 let deleted = !root_id.as_path().exists();
1280 if deleted {
1281 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1291 } else {
1292 meta.consecutive_missing_sweeps = 0;
1293 }
1294 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1295 let has_bound_route = root_channels
1296 .get(root_id)
1297 .is_some_and(|channels| !channels.is_empty());
1298 let has_pending_bind = pending_bind_roots.contains(root_id);
1299
1300 if deleted {
1301 let mut retained = false;
1302 if !deletion_confirmed {
1303 census.absence_unconfirmed += 1;
1304 retained = true;
1305 }
1306 if meta.active_bash_waits > 0 {
1310 census.bash_waits += 1;
1311 retained = true;
1312 }
1313 if meta.maintenance_pending {
1314 census.maintenance_pending += 1;
1315 retained = true;
1316 }
1317 if !meta.maintenance_queued_kinds.is_empty() {
1318 census.maintenance_queued += 1;
1319 retained = true;
1320 }
1321 if has_pending_bind {
1322 census.pending_binds += 1;
1323 retained = true;
1324 }
1325 match executor.try_actor_is_idle(root_id) {
1326 Some(true) => {}
1327 Some(false) => {
1328 census.actor_busy += 1;
1329 retained = true;
1330 }
1331 None => {
1332 census.actor_state_busy += 1;
1333 retained = true;
1334 }
1335 }
1336 if retained {
1337 census.deleted_retained += 1;
1338 continue;
1339 }
1340 } else {
1341 if has_bound_route
1345 || !meta.unbound_quiesced
1346 || meta.idle_artifacts_evicted
1347 || now.saturating_duration_since(meta.last_touched)
1348 < root_idle_ttl(executor, root_id)
1349 || meta.active_bash_waits > 0
1350 || meta.maintenance_pending
1351 || !meta.maintenance_queued_kinds.is_empty()
1352 || has_pending_bind
1353 || !executor.actor_is_idle(root_id)
1354 {
1355 continue;
1356 }
1357 }
1358 candidates.push((root_id.clone(), deleted));
1359 }
1360
1361 let mut reaped = Vec::new();
1362 let mut forgotten_deleted_roots = Vec::new();
1363 for (root_id, deleted) in candidates {
1364 let Some(ctx) = executor.actor_context(&root_id) else {
1365 if deleted {
1366 census.deleted_retained += 1;
1367 census.actor_busy += 1;
1368 }
1369 continue;
1370 };
1371 if deleted {
1382 ctx.bash_background()
1383 .kill_running_tasks_for_root(root_id.as_path());
1384 }
1385 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1386 if ctx.artifact_eviction_blocked() {
1387 if let Some(pending) = taken_pending {
1388 ctx.restore_pending_reconciliation_state(pending);
1389 }
1390 if deleted {
1391 census.deleted_retained += 1;
1392 census.artifact_eviction_blocked += 1;
1393 }
1394 continue;
1395 }
1396 let memory_before = ctx.memory_root_snapshot();
1397 if !ctx.evict_idle_artifacts() {
1398 if let Some(pending) = taken_pending {
1399 ctx.restore_pending_reconciliation_state(pending);
1400 }
1401 if deleted {
1402 census.deleted_retained += 1;
1403 census.artifact_eviction_failed += 1;
1404 }
1405 continue;
1406 }
1407 drop(taken_pending);
1408 ctx.stop_watcher_runtime_in_background();
1409 ctx.invalidate_artifacts_after_watcher_gap();
1412
1413 if deleted {
1414 if executor.retire_idle_actor_in_background(&root_id) {
1415 live_roots.remove(&root_id);
1416 forgotten_deleted_roots.push(root_id.clone());
1417 } else {
1418 census.deleted_retained += 1;
1419 census.actor_busy += 1;
1420 }
1421 } else {
1422 if let Some(meta) = live_roots.get_mut(&root_id) {
1423 meta.idle_artifacts_evicted = true;
1424 }
1425 ctx.release_idle_reopenable_resources_in_background();
1426 }
1427 reaped.push((root_id, memory_before));
1428 }
1429
1430 let census_changed = metrics.record_reap(census);
1433 if census_changed {
1434 log::info!(
1435 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1436 census.deleted_retained,
1437 census.blocker_histogram()
1438 );
1439 }
1440
1441 let pressure_relief = (!reaped.is_empty())
1442 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1443 .flatten();
1444 for (root_id, memory_before) in &reaped {
1445 log::info!(
1446 "{}",
1447 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1448 );
1449 }
1450 IdleReapOutcome {
1451 evicted: reaped.len(),
1452 forgotten_deleted_roots,
1453 }
1454}
1455
1456fn reap_idle_lsp_servers(
1464 now: Instant,
1465 live_roots: &HashMap<ProjectRootId, RootMeta>,
1466 executor: &Executor,
1467) {
1468 for (root_id, meta) in live_roots {
1469 let Some(ctx) = executor.actor_context(root_id) else {
1470 continue;
1471 };
1472 crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1473 }
1474}
1475
1476#[allow(clippy::too_many_arguments)]
1477fn purge_deleted_root_residents(
1478 root_id: &ProjectRootId,
1479 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1480 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1481 installed_route_epochs: &mut HashMap<u16, u32>,
1482 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1483 active_tool_calls: &ActiveToolCalls,
1484 executor: &Executor,
1485 retry_buffer: &mut RetryBuffer,
1486 reclaimed_routes: &mut ReclaimedRoutes,
1487 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1488 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1489 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1490 bg_sub_by_session: &mut BgSubsBySession,
1491 bg_wake_pending: &mut BgWakePending,
1492 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1493 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1494 metrics: &DispatchPathMetrics,
1495) {
1496 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1497 stale_routes.extend(
1498 routes
1499 .iter()
1500 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1501 );
1502 stale_routes.extend(
1503 bg_sub_by_session
1504 .iter()
1505 .filter(|((root, _), _)| root == root_id)
1506 .flat_map(|(_, routes)| routes.iter().copied()),
1507 );
1508 stale_routes.extend(
1509 pending_bash_asks
1510 .values()
1511 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1512 );
1513
1514 for route in stale_routes {
1515 reclaimed_routes.insert(route);
1516 remove_installed_route(installed_route_epochs, route);
1517 remove_route_channel(routes, root_channels, route);
1518 if let Some(cancel) = route_bash_cancels.remove(&route) {
1519 cancel.token.cancel();
1520 }
1521 apply_route_work_disposition(
1522 active_tool_calls,
1523 executor,
1524 route,
1525 RouteWorkDisposition::Abandon,
1526 "root reclaim",
1527 );
1528 retry_buffer.remove(&route);
1529 if let Some(sub) = bg_subs.remove(&route) {
1530 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1531 }
1532 bg_wake_pending.remove(&route);
1533 }
1534 root_channels.remove(root_id);
1535 session_identity.retain(|(root, _), _| root != root_id);
1536 push_buffer.retain(|key, _| &key.root != root_id);
1537 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1538 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1539 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1540 sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1541
1542 log::info!(
1543 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1544 root_id.as_path().display()
1545 );
1546}
1547
1548#[allow(clippy::too_many_arguments)]
1549fn submit_due_maintenance_jobs(
1550 executor: &Arc<Executor>,
1551 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1552 pending_binds: &HashMap<RouteChannel, PendingBind>,
1553 bg_sub_by_session: &BgSubsBySession,
1554 bg_wake_pending: &BgWakePending,
1555 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1556 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1557 metrics: &Arc<DispatchPathMetrics>,
1558) {
1559 let pending_bind_roots = pending_binds
1560 .values()
1561 .map(|pending| pending.bind_root_id.clone())
1562 .collect::<HashSet<_>>();
1563 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1564 live_roots,
1565 Some(executor),
1566 bg_sub_by_session,
1567 bg_wake_pending,
1568 MAINTENANCE_SUBMIT_BUDGET,
1569 &pending_bind_roots,
1570 );
1571 if deferred_jobs {
1572 metrics
1573 .maintenance_budget_deferrals
1574 .fetch_add(1, Ordering::Relaxed);
1575 }
1576 for (root_id, kind) in due_jobs {
1577 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1578 bg_sub_by_session
1579 .iter()
1580 .filter_map(|((root, session), _)| {
1581 if root == &root_id {
1582 Some((
1583 session.clone(),
1584 bg_wake_epoch
1585 .get(&(root_id.clone(), session.clone()))
1586 .copied()
1587 .unwrap_or(0),
1588 ))
1589 } else {
1590 None
1591 }
1592 })
1593 .collect()
1594 } else {
1595 Vec::new()
1596 };
1597 submit_maintenance_job(
1598 executor,
1599 root_id,
1600 kind,
1601 bg_sessions_to_check,
1602 maintenance_tx,
1603 metrics,
1604 );
1605 }
1606}
1607
1608fn should_requiesce_after_maintenance(
1609 meta: &RootMeta,
1610 completed_kind: MaintenanceDrainKind,
1611 bind_pending: bool,
1612) -> bool {
1613 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1614}
1615
1616fn note_maintenance_completion(
1617 meta: &mut RootMeta,
1618 requeue_kind: Option<MaintenanceDrainKind>,
1619 fatal: bool,
1620 defer_requeue: bool,
1621) {
1622 if fatal {
1623 meta.maintenance_poisoned = true;
1624 }
1625
1626 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1627 meta.maintenance_queued_kinds.push_back(kind);
1628 }
1629
1630 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1631 meta.maintenance_pending =
1632 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1633}
1634
1635fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1636 RouteChannel { channel, epoch }
1637}
1638
1639fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1640 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1641 installed_epochs.remove(&route.channel);
1642 }
1643}
1644
1645fn ingress_route_should_be_processed(
1646 installed_epochs: &HashMap<u16, u32>,
1647 reclaimed_routes: &ReclaimedRoutes,
1648 frame: &Frame,
1649) -> bool {
1650 if frame.header.channel == 0
1651 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1652 {
1653 return true;
1654 }
1655
1656 frame.header.ty == FrameType::Request
1661 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1662}
1663
1664fn bash_elicitation_timeout() -> Duration {
1665 if cfg!(debug_assertions) {
1666 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1667 if let Ok(ms) = raw.parse::<u64>() {
1668 if ms > 0 {
1669 return Duration::from_millis(ms);
1670 }
1671 }
1672 }
1673 }
1674 BASH_ELICITATION_TIMEOUT
1675}
1676
1677fn allocate_reverse_corr(
1678 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1679 route: RouteChannel,
1680 next_corr: &mut u64,
1681) -> u64 {
1682 loop {
1683 let corr = *next_corr;
1684 *next_corr = (*next_corr).wrapping_add(1).max(1);
1685 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1686 return corr;
1687 }
1688 }
1689}
1690
1691fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1692 match kind {
1693 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1694 crate::bash_permissions::PermissionKind::Bash => "bash",
1695 }
1696}
1697
1698fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1699 let mut patterns = Vec::new();
1700 let mut seen = HashSet::new();
1701 for ask in asks {
1702 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1703 if seen.insert(pattern.clone()) {
1704 patterns.push(pattern.clone());
1705 }
1706 }
1707 }
1708 patterns
1709}
1710
1711fn bash_elicitation_message(
1712 command: &str,
1713 asks: &[crate::bash_permissions::PermissionAsk],
1714) -> String {
1715 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1716 let patterns = bash_elicitation_patterns(asks);
1717 let pattern_text = if patterns.is_empty() {
1718 "no matched permission patterns".to_string()
1719 } else {
1720 patterns.join(", ")
1721 };
1722 let ask_kinds = asks
1723 .iter()
1724 .map(|ask| bash_permission_kind_label(&ask.kind))
1725 .collect::<HashSet<_>>()
1726 .into_iter()
1727 .collect::<Vec<_>>()
1728 .join(", ");
1729 if ask_kinds.is_empty() {
1730 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1731 } else {
1732 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1733 }
1734}
1735
1736fn bash_elicitation_request_body(
1737 command: &str,
1738 asks: &[crate::bash_permissions::PermissionAsk],
1739) -> Value {
1740 json!({
1741 "method": BASH_ELICITATION_CREATE_METHOD,
1742 "params": {
1743 "mode": "form",
1744 "message": bash_elicitation_message(command, asks),
1745 "requestedSchema": {
1746 "type": "object",
1747 "properties": {
1748 "decision": {
1749 "type": "string",
1750 "enum": ["allow", "deny"],
1751 "description": "Choose allow to run this bash command once, or deny to block it."
1752 }
1753 },
1754 "required": ["decision"],
1755 "additionalProperties": false
1756 },
1757 "_meta": {
1758 "aft": {
1759 "tool": "bash",
1760 "command": command,
1761 "asks": asks
1762 }
1763 }
1764 }
1765 })
1766}
1767
1768fn build_bash_elicitation_request_frame(
1769 ver: u8,
1770 route: RouteChannel,
1771 corr: u64,
1772 flags: Flags,
1773 command: &str,
1774 asks: &[crate::bash_permissions::PermissionAsk],
1775) -> Result<Frame, SubcError> {
1776 let body = bash_elicitation_request_body(command, asks);
1777 Frame::build_with_version(
1778 ver,
1779 FrameType::Request,
1780 flags,
1781 route.channel,
1782 route.epoch,
1783 corr,
1784 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1785 )
1786 .map_err(SubcError::FrameBuild)
1787}
1788
1789fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1790 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1791 return false;
1792 };
1793 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1794}
1795
1796fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1797 let Some(object) = value.as_object() else {
1798 return false;
1799 };
1800 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1801}
1802
1803fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1804 let Some(object) = value.as_object() else {
1805 return false;
1806 };
1807 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1808 return false;
1809 }
1810 let Some(content) = object.get("content").and_then(Value::as_object) else {
1811 return false;
1812 };
1813 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1814}
1815
1816#[allow(clippy::too_many_arguments)]
1817async fn settle_pending_bash_ask_denied(
1818 tx: &WriterSender,
1819 pending: PendingBashAsk,
1820 routes: &HashMap<RouteChannel, RouteIdentity>,
1821 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1822 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1823 shutdown: &Arc<Notify>,
1824 metrics: &DispatchPathMetrics,
1825) -> Result<(), SubcError> {
1826 let completion = bash::bash_denied_untrusted_completion(
1827 pending.route,
1828 pending.tool_corr,
1829 pending.tool_flags,
1830 pending.tool_ver,
1831 pending.root,
1832 pending.request_id,
1833 pending.format_context,
1834 );
1835 bash::handle_bash_deferred_completion(
1836 tx,
1837 completion,
1838 routes,
1839 live_roots,
1840 route_bash_cancels,
1841 shutdown,
1842 metrics,
1843 )
1844 .await
1845}
1846
1847fn take_pending_bash_asks_for_route(
1848 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1849 route: RouteChannel,
1850) -> Vec<PendingBashAsk> {
1851 let keys = pending_bash_asks
1852 .keys()
1853 .copied()
1854 .filter(|key| key.route == route)
1855 .collect::<Vec<_>>();
1856 keys.into_iter()
1857 .filter_map(|key| pending_bash_asks.remove(&key))
1858 .collect()
1859}
1860
1861#[allow(clippy::too_many_arguments)]
1862async fn settle_pending_bash_asks_for_route(
1863 tx: &WriterSender,
1864 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1865 route: RouteChannel,
1866 routes: &HashMap<RouteChannel, RouteIdentity>,
1867 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1868 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1869 shutdown: &Arc<Notify>,
1870 metrics: &DispatchPathMetrics,
1871) -> Result<(), SubcError> {
1872 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1873 settle_pending_bash_ask_denied(
1874 tx,
1875 pending,
1876 routes,
1877 live_roots,
1878 route_bash_cancels,
1879 shutdown,
1880 metrics,
1881 )
1882 .await?;
1883 }
1884 Ok(())
1885}
1886
1887#[allow(clippy::too_many_arguments)]
1888async fn settle_all_pending_bash_asks(
1889 tx: &WriterSender,
1890 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1891 routes: &HashMap<RouteChannel, RouteIdentity>,
1892 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1893 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1894 shutdown: &Arc<Notify>,
1895 metrics: &DispatchPathMetrics,
1896) -> Result<(), SubcError> {
1897 let pending = pending_bash_asks
1898 .drain()
1899 .map(|(_, pending)| pending)
1900 .collect::<Vec<_>>();
1901 for pending in pending {
1902 settle_pending_bash_ask_denied(
1903 tx,
1904 pending,
1905 routes,
1906 live_roots,
1907 route_bash_cancels,
1908 shutdown,
1909 metrics,
1910 )
1911 .await?;
1912 }
1913 Ok(())
1914}
1915
1916#[allow(clippy::too_many_arguments)]
1917async fn expire_pending_bash_asks(
1918 tx: &WriterSender,
1919 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1920 routes: &HashMap<RouteChannel, RouteIdentity>,
1921 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1922 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1923 shutdown: &Arc<Notify>,
1924 metrics: &DispatchPathMetrics,
1925) -> Result<(), SubcError> {
1926 let now = Instant::now();
1927 let expired = pending_bash_asks
1928 .iter()
1929 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1930 .collect::<Vec<_>>();
1931 for key in expired {
1932 if let Some(pending) = pending_bash_asks.remove(&key) {
1933 log::debug!(
1934 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1935 key.corr,
1936 pending.route
1937 );
1938 settle_pending_bash_ask_denied(
1939 tx,
1940 pending,
1941 routes,
1942 live_roots,
1943 route_bash_cancels,
1944 shutdown,
1945 metrics,
1946 )
1947 .await?;
1948 }
1949 }
1950 Ok(())
1951}
1952
1953#[allow(clippy::too_many_arguments)]
1954async fn handle_bash_elicitation_reply(
1955 tx: &WriterSender,
1956 frame: &Frame,
1957 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1958 routes: &HashMap<RouteChannel, RouteIdentity>,
1959 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1960 executor: &Arc<Executor>,
1961 shutdown: &Arc<Notify>,
1962 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1963 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1964 metrics: &Arc<DispatchPathMetrics>,
1965 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1966 dispatch: DispatchFn,
1967) -> Result<(), SubcError> {
1968 let key = ReverseCorrKey {
1969 route: route_key(frame.header.channel, frame.header.epoch),
1970 corr: frame.header.corr,
1971 };
1972 let Some(pending) = pending_bash_asks.remove(&key) else {
1973 return Ok(());
1974 };
1975
1976 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1977 if routes.contains_key(&key.route) {
1978 bash::submit_deferred_bash(
1979 executor,
1980 bash_deferred_tx,
1981 bash_poll_touch_tx,
1982 metrics,
1983 dispatch,
1984 pending.root,
1985 pending.project_root,
1986 pending.session_id,
1987 pending.request_id,
1988 pending.route,
1989 pending.tool_corr,
1990 pending.tool_flags,
1991 pending.tool_ver,
1992 pending.arguments,
1993 pending.format_context,
1994 pending.cancel,
1995 BindTrust::Untrusted,
1996 pending.spawn_principal,
1997 pending.edit_slot_survives,
1998 Some(pending.grants),
1999 );
2000 return Ok(());
2001 }
2002 log::debug!(
2003 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2004 key.corr,
2005 pending.route
2006 );
2007 }
2008
2009 settle_pending_bash_ask_denied(
2010 tx,
2011 pending,
2012 routes,
2013 live_roots,
2014 route_bash_cancels,
2015 shutdown,
2016 metrics,
2017 )
2018 .await
2019}
2020
2021#[allow(clippy::too_many_arguments)]
2022async fn cancel_pending_bash_ask_for_tool_call(
2023 tx: &WriterSender,
2024 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2025 route: RouteChannel,
2026 tool_corr: u64,
2027 routes: &HashMap<RouteChannel, RouteIdentity>,
2028 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2029 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2030 shutdown: &Arc<Notify>,
2031 metrics: &DispatchPathMetrics,
2032) -> Result<(), SubcError> {
2033 let keys = pending_bash_asks
2034 .iter()
2035 .filter_map(|(key, pending)| {
2036 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2037 })
2038 .collect::<Vec<_>>();
2039 for key in keys {
2040 if let Some(pending) = pending_bash_asks.remove(&key) {
2041 settle_pending_bash_ask_denied(
2042 tx,
2043 pending,
2044 routes,
2045 live_roots,
2046 route_bash_cancels,
2047 shutdown,
2048 metrics,
2049 )
2050 .await?;
2051 }
2052 }
2053 Ok(())
2054}
2055
2056fn remove_root_channel(
2057 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2058 root: &ProjectRootId,
2059 channel: RouteChannel,
2060) {
2061 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2062 channels.remove(&channel);
2063 channels.is_empty()
2064 } else {
2065 false
2066 };
2067 if remove_root {
2068 root_channels.remove(root);
2069 }
2070}
2071
2072fn remove_route_channel(
2073 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2074 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2075 channel: RouteChannel,
2076) -> Option<RouteIdentity> {
2077 let removed = routes.remove(&channel);
2078 if let Some(identity) = &removed {
2079 remove_root_channel(root_channels, &identity.root, channel);
2080 }
2081 removed
2082}
2083
2084fn insert_route_channel(
2085 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2086 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2087 channel: RouteChannel,
2088 identity: RouteIdentity,
2089) {
2090 if let Some(previous) = routes.insert(channel, identity.clone()) {
2091 remove_root_channel(root_channels, &previous.root, channel);
2092 }
2093 root_channels
2094 .entry(identity.root.clone())
2095 .or_default()
2096 .insert(channel);
2097}
2098
2099fn sync_bg_live_delivery_sessions(
2100 executor: &Executor,
2101 routes: &HashMap<RouteChannel, RouteIdentity>,
2102 additional_root: Option<&ProjectRootId>,
2103) {
2104 let sessions = routes
2108 .values()
2109 .filter(|identity| identity.trust.allows_bash_observation())
2110 .map(|identity| identity.session.clone())
2111 .collect::<HashSet<_>>();
2112 let mut roots = routes
2113 .values()
2114 .map(|identity| identity.root.clone())
2115 .collect::<HashSet<_>>();
2116 roots.extend(additional_root.cloned());
2117 for root in roots {
2118 if let Some(ctx) = executor.actor_context(&root) {
2119 ctx.bash_background()
2120 .replace_live_delivery_sessions(sessions.clone());
2121 }
2122 }
2123}
2124
2125fn insert_bg_subscription_index(
2126 bg_sub_by_session: &mut BgSubsBySession,
2127 root: ProjectRootId,
2128 session: String,
2129 channel: RouteChannel,
2130) {
2131 bg_sub_by_session
2132 .entry((root, session))
2133 .or_default()
2134 .insert(channel);
2135}
2136
2137fn remove_bg_subscription_index(
2138 bg_sub_by_session: &mut BgSubsBySession,
2139 channel: RouteChannel,
2140 identity: Option<&RouteIdentity>,
2141) {
2142 if let Some(identity) = identity {
2143 let key = (identity.root.clone(), identity.session.clone());
2144 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2145 channels.remove(&channel);
2146 channels.is_empty()
2147 });
2148 if remove_key {
2149 bg_sub_by_session.remove(&key);
2150 }
2151 } else {
2152 bg_sub_by_session.retain(|_, channels| {
2153 channels.remove(&channel);
2154 !channels.is_empty()
2155 });
2156 }
2157}
2158
2159fn route_removal_will_quiesce_root(
2160 root: &ProjectRootId,
2161 route: RouteChannel,
2162 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2163 has_pending_bind: bool,
2164 replacement_root: Option<&ProjectRootId>,
2165) -> bool {
2166 let removes_last_route = root_channels
2167 .get(root)
2168 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2169 removes_last_route && !has_pending_bind && replacement_root != Some(root)
2170}
2171
2172fn should_quiesce_removed_root(
2173 root: &ProjectRootId,
2174 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2175 has_pending_bind: bool,
2176 replacement_root: Option<&ProjectRootId>,
2177) -> bool {
2178 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2179}
2180
2181async fn end_bg_subscription(
2182 writer_tx: &WriterSender,
2183 metrics: &DispatchPathMetrics,
2184 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2185 bg_sub_by_session: &mut BgSubsBySession,
2186 bg_wake_pending: &mut BgWakePending,
2187 channel: RouteChannel,
2188 identity: Option<&RouteIdentity>,
2189 cause: &str,
2190) -> Result<(), SubcError> {
2191 if let Some(sub) = bg_subs.remove(&channel) {
2192 bg_wake_pending.remove(&channel);
2193 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2194 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2195 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2196 }
2197 Ok(())
2198}
2199
2200#[allow(clippy::too_many_arguments)]
2201async fn teardown_installed_route(
2202 tx: &WriterSender,
2203 metrics: &DispatchPathMetrics,
2204 executor: &Arc<Executor>,
2205 channel: RouteChannel,
2206 cancellation_reason: &str,
2207 replacement_root: Option<&ProjectRootId>,
2208 installed_route_epochs: &mut HashMap<u16, u32>,
2209 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2210 management_routes: &mut HashSet<RouteChannel>,
2211 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2212 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2213 bg_sub_by_session: &mut BgSubsBySession,
2214 bg_wake_pending: &mut BgWakePending,
2215 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2216 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2217 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2218 active_tool_calls: &ActiveToolCalls,
2219 pending_responses: &mut PendingSubcResponses,
2220 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2221 retry_buffer: &mut RetryBuffer,
2222 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2223 shutdown: &Arc<Notify>,
2224 tool_response_body_limit: usize,
2225 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2226) -> Result<(), SubcError> {
2227 remove_installed_route(installed_route_epochs, channel);
2228 management_routes.remove(&channel);
2229 let bg_end_cause = match cancellation_reason {
2230 "Goodbye" => "goodbye",
2231 "higher-epoch RouteBind" => "higher-epoch",
2232 other => other,
2233 };
2234 end_bg_subscription(
2235 tx,
2236 metrics,
2237 bg_subs,
2238 bg_sub_by_session,
2239 bg_wake_pending,
2240 channel,
2241 routes.get(&channel),
2242 bg_end_cause,
2243 )
2244 .await?;
2245 settle_pending_bash_asks_for_route(
2246 tx,
2247 pending_bash_asks,
2248 channel,
2249 routes,
2250 live_roots,
2251 route_bash_cancels,
2252 shutdown,
2253 metrics,
2254 )
2255 .await?;
2256 if let Some(cancel) = route_bash_cancels.remove(&channel) {
2257 cancel.token.cancel();
2258 }
2259 for resolved in pending_responses.drain_route(channel, executor) {
2260 deliver_resolved_subc_response(
2261 tx,
2262 resolved,
2263 routes,
2264 live_roots,
2265 executor.as_ref(),
2266 active_tool_calls,
2267 shutdown,
2268 metrics,
2269 tool_response_body_limit,
2270 )
2271 .await?;
2272 }
2273 let work_disposition = if replacement_root.is_some() {
2278 RouteWorkDisposition::RetainForReplay
2279 } else {
2280 RouteWorkDisposition::RetainStartedForReplay
2281 };
2282 apply_route_work_disposition(
2283 active_tool_calls,
2284 executor,
2285 channel,
2286 work_disposition,
2287 cancellation_reason,
2288 );
2289 if let Some(pending) = pending_binds.get_mut(&channel) {
2290 pending.cancelled = true;
2291 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2292 log::debug!(
2293 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2294 channel.channel
2295 );
2296 }
2297 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2298 if let Some(identity) = routes.get(&channel) {
2299 let has_pending_bind = pending_binds
2300 .values()
2301 .any(|pending| pending.bind_root_id == identity.root);
2302 if route_removal_will_quiesce_root(
2303 &identity.root,
2304 channel,
2305 root_channels,
2306 has_pending_bind,
2307 replacement_root,
2308 ) {
2309 if let Some(ctx) = executor.actor_context(&identity.root) {
2310 ctx.mark_subc_unbound();
2313 }
2314 }
2315 }
2316 delay_route_detach_for_test(lifecycle_probe).await;
2319 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2320 sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2321 if let Some(probe) = lifecycle_probe {
2322 probe.route_detached(channel, &identity.session);
2323 }
2324 let session_still_routed = routes
2325 .values()
2326 .any(|route| route.root == identity.root && route.session == identity.session);
2327 if !session_still_routed {
2328 if let Some(ctx) = executor.actor_context(&identity.root) {
2329 ctx.hashline_bindings()
2330 .teardown(identity.root.as_path(), &identity.session);
2331 }
2332 }
2333 if migrated > 0 {
2334 log::debug!(
2335 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2336 channel.channel
2337 );
2338 }
2339 if let Some(meta) = live_roots.get_mut(&identity.root) {
2340 let idle_for = meta.last_touched.elapsed();
2341 meta.note_activity();
2342 log::debug!(
2343 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2344 channel.channel,
2345 identity.root.as_path().display(),
2346 identity.harness,
2347 identity.session,
2348 idle_for
2349 );
2350 } else {
2351 log::debug!(
2352 "subc attach: route {} torn down for root {} harness {} session {}",
2353 channel.channel,
2354 identity.root.as_path().display(),
2355 identity.harness,
2356 identity.session
2357 );
2358 }
2359 let has_pending_bind = pending_binds
2360 .values()
2361 .any(|pending| pending.bind_root_id == identity.root);
2362 if should_quiesce_removed_root(
2363 &identity.root,
2364 root_channels,
2365 has_pending_bind,
2366 replacement_root,
2367 ) {
2368 quiesce_unbound_root(&identity.root, live_roots, executor);
2369 }
2370 } else {
2371 if migrated > 0 {
2372 log::debug!(
2373 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2374 channel.channel
2375 );
2376 }
2377 log::debug!("subc attach: unbound route {} torn down", channel.channel);
2378 }
2379 Ok(())
2380}
2381
2382async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2383 if lifecycle_probe.is_none() {
2384 return;
2385 }
2386 let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2387 .ok()
2388 .and_then(|raw| raw.parse::<u64>().ok())
2389 else {
2390 return;
2391 };
2392 tokio::time::sleep(Duration::from_millis(delay)).await;
2393}
2394
2395fn remember_session_identity(
2396 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2397 identity: &RouteIdentity,
2398) {
2399 let key = (identity.root.clone(), identity.session.clone());
2400 if matches!(identity.trust, BindTrust::Untrusted)
2401 && session_identity
2402 .get(&key)
2403 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2404 {
2405 return;
2406 }
2407
2408 session_identity.insert(
2413 key,
2414 RetainedSessionIdentity {
2415 harness: identity.harness.clone(),
2416 trust: identity.trust,
2417 },
2418 );
2419}
2420
2421fn replay_key_for_session(
2422 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2423 root: &ProjectRootId,
2424 session: &str,
2425) -> Option<(push::ReplayKey, BindTrust)> {
2426 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2427 Some((
2428 push::ReplayKey {
2429 root: root.clone(),
2430 harness: retained.harness.clone(),
2431 session: session.to_string(),
2432 },
2433 retained.trust,
2434 ))
2435}
2436pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2439
2440#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2441enum ModuleLoopExit {
2442 Graceful,
2445 ConnectionLost,
2448 SkipSearchFlush,
2453}
2454
2455pub fn run_subc_mode(
2460 connection_file_path: &Path,
2461 ctx: Arc<AppContext>,
2462 executor: Arc<Executor>,
2463 dispatch: DispatchFn,
2464 user_config_path: Option<PathBuf>,
2465) -> Result<(), SubcError> {
2466 run_subc_mode_inner(
2470 connection_file_path,
2471 ctx,
2472 executor,
2473 dispatch,
2474 user_config_path,
2475 false,
2476 MAX_FRAME_BODY_LEN as usize,
2477 None,
2478 )
2479}
2480
2481fn run_subc_mode_inner(
2482 connection_file_path: &Path,
2483 ctx: Arc<AppContext>,
2484 executor: Arc<Executor>,
2485 dispatch: DispatchFn,
2486 user_config_path: Option<PathBuf>,
2487 allow_native_passthrough: bool,
2488 tool_response_body_limit: usize,
2489 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2490) -> Result<(), SubcError> {
2491 let runtime = tokio::runtime::Builder::new_current_thread()
2492 .enable_all()
2493 .build()
2494 .map_err(SubcError::Runtime)?;
2495
2496 let executor_for_loop = Arc::clone(&executor);
2497 let loop_result = runtime.block_on(async move {
2498 let shared_app = ctx.app();
2499 drop(ctx);
2500 let stream =
2501 connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2502 log::info!(
2503 "subc attach: authenticated to daemon via {}",
2504 connection_file_path.display()
2505 );
2506 let (read_half, write_half) = tokio::io::split(stream);
2507 run_module_loop(
2508 read_half,
2509 write_half,
2510 connection_file_path,
2511 shared_app,
2512 executor_for_loop,
2513 dispatch,
2514 user_config_path,
2515 allow_native_passthrough,
2516 tool_response_body_limit,
2517 lifecycle_probe,
2518 )
2519 .await
2520 });
2521
2522 let actor_contexts = executor.actor_contexts();
2523 if matches!(
2524 loop_result,
2525 Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2526 ) {
2527 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2530 }
2531 for actor_ctx in &actor_contexts {
2532 actor_ctx.lsp().shutdown_all();
2533 actor_ctx.bash_background().detach();
2534 }
2535
2536 match loop_result {
2537 Ok(exit) => module_loop_exit_result(exit),
2538 Err(error) => Err(error),
2539 }
2540}
2541
2542fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2546 match exit {
2547 ModuleLoopExit::Graceful => Ok(()),
2548 ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2549 ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2550 }
2551}
2552
2553fn note_fatal_panic_response(response: &Response) -> bool {
2557 let fatal = response_is_fatal_panic(response);
2558 if fatal {
2559 log::error!(
2560 "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2561 response.id,
2562 response
2563 .data
2564 .get("message")
2565 .and_then(Value::as_str)
2566 .unwrap_or("(no message)")
2567 );
2568 }
2569 fatal
2570}
2571
2572fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2573 for actor_ctx in actor_contexts {
2574 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2575 }
2576 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2577}
2578
2579#[doc(hidden)]
2584pub fn run_subc_mode_for_test(
2585 connection_file_path: &Path,
2586 ctx: Arc<AppContext>,
2587 executor: Arc<Executor>,
2588 dispatch: DispatchFn,
2589 user_config_path: Option<PathBuf>,
2590) -> Result<(), SubcError> {
2591 run_subc_mode_inner(
2592 connection_file_path,
2593 ctx,
2594 executor,
2595 dispatch,
2596 user_config_path,
2597 true,
2598 MAX_FRAME_BODY_LEN as usize,
2599 None,
2600 )
2601}
2602
2603#[doc(hidden)]
2605pub fn run_subc_mode_for_test_with_lifecycle_probe(
2606 connection_file_path: &Path,
2607 ctx: Arc<AppContext>,
2608 executor: Arc<Executor>,
2609 dispatch: DispatchFn,
2610 user_config_path: Option<PathBuf>,
2611 lifecycle_probe: SubcTestLifecycleProbe,
2612) -> Result<(), SubcError> {
2613 run_subc_mode_inner(
2614 connection_file_path,
2615 ctx,
2616 executor,
2617 dispatch,
2618 user_config_path,
2619 true,
2620 MAX_FRAME_BODY_LEN as usize,
2621 Some(lifecycle_probe),
2622 )
2623}
2624
2625#[doc(hidden)]
2628pub fn run_subc_mode_for_test_with_response_body_limit(
2629 connection_file_path: &Path,
2630 ctx: Arc<AppContext>,
2631 executor: Arc<Executor>,
2632 dispatch: DispatchFn,
2633 user_config_path: Option<PathBuf>,
2634 tool_response_body_limit: usize,
2635) -> Result<(), SubcError> {
2636 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2637 run_subc_mode_inner(
2638 connection_file_path,
2639 ctx,
2640 executor,
2641 dispatch,
2642 user_config_path,
2643 true,
2644 tool_response_body_limit,
2645 None,
2646 )
2647}
2648
2649#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2650enum AttachErrorClass {
2651 Transient,
2652 Permanent,
2653}
2654
2655impl fmt::Display for AttachErrorClass {
2656 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2657 match self {
2658 Self::Transient => f.write_str("transient"),
2659 Self::Permanent => f.write_str("permanent"),
2660 }
2661 }
2662}
2663
2664#[derive(Clone, Copy)]
2665struct AttachRetryPolicy {
2666 budget: Duration,
2667 initial_backoff: Duration,
2668 max_backoff: Duration,
2669 jitter_percent: u64,
2670}
2671
2672const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2673 budget: ATTACH_RETRY_BUDGET,
2674 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2675 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2676 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2677};
2678
2679fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2682 let transient = match error {
2683 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2684 SubcError::Auth { source, .. } => match source {
2685 subc_transport::AuthError::Timeout { .. }
2686 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2687 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2688 _ => false,
2689 },
2690 _ => false,
2691 };
2692 if transient {
2693 AttachErrorClass::Transient
2694 } else {
2695 AttachErrorClass::Permanent
2696 }
2697}
2698
2699fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2700 matches!(
2701 kind,
2702 io::ErrorKind::ConnectionRefused
2703 | io::ErrorKind::TimedOut
2704 | io::ErrorKind::ConnectionReset
2705 | io::ErrorKind::ConnectionAborted
2706 | io::ErrorKind::BrokenPipe
2707 | io::ErrorKind::UnexpectedEof
2708 )
2709}
2710
2711async fn connect_and_authenticate(
2715 connection_file_path: &Path,
2716 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2717) -> Result<TcpStream, SubcError> {
2718 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2719 .await
2720}
2721
2722async fn connect_and_authenticate_with_policy(
2723 connection_file_path: &Path,
2724 policy: AttachRetryPolicy,
2725 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2726) -> Result<TcpStream, SubcError> {
2727 let started_at = Instant::now();
2728 let deadline = started_at + policy.budget;
2729 let mut attempt = 0_u32;
2730 let mut backoff = policy.initial_backoff;
2731 let mut history = Vec::new();
2732
2733 loop {
2734 attempt = attempt.saturating_add(1);
2735 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2736 Ok(stream) => return Ok(stream),
2737 Err(error) => error,
2738 };
2739 let class = classify_attach_error(&error);
2740 let will_retry = class != AttachErrorClass::Permanent;
2741 if let Some(probe) = lifecycle_probe {
2742 probe.attach_decision(attempt, will_retry);
2743 }
2744 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2745 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2746
2747 if !will_retry {
2748 log_attach_final_failure(started_at.elapsed(), &history);
2749 return Err(error);
2750 }
2751
2752 let remaining = deadline.saturating_duration_since(Instant::now());
2753 if remaining.is_zero() {
2754 log_attach_final_failure(started_at.elapsed(), &history);
2755 return Err(error);
2756 }
2757
2758 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2759 log::info!(
2760 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2761 );
2762 tokio::time::sleep(delay).await;
2763
2764 if Instant::now() >= deadline {
2765 log_attach_final_failure(started_at.elapsed(), &history);
2766 return Err(error);
2767 }
2768 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2769 }
2770}
2771
2772fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2773 let jitter_percent = jitter_percent.min(100);
2774 if jitter_percent == 0 {
2775 return base;
2776 }
2777
2778 let mut random_bytes = [0_u8; 8];
2779 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2780 u64::from_le_bytes(random_bytes)
2781 } else {
2782 let timestamp = std::time::SystemTime::now()
2783 .duration_since(std::time::UNIX_EPOCH)
2784 .unwrap_or_default()
2785 .subsec_nanos();
2786 u64::from(timestamp) ^ u64::from(attempt)
2787 };
2788 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2789 let multiplier_percent = 100 - jitter_percent + random % span;
2790 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2791 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2792}
2793
2794fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2795 log::error!(
2796 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2797 history.len(),
2798 history.join(" | ")
2799 );
2800}
2801
2802async fn connect_and_authenticate_once(
2803 connection_file_path: &Path,
2804 deadline: Instant,
2805) -> Result<TcpStream, SubcError> {
2806 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2809 SubcError::ConnectionFile {
2810 path: connection_file_path.to_path_buf(),
2811 source,
2812 }
2813 })?;
2814
2815 let endpoint = conn
2816 .endpoints
2817 .first()
2818 .ok_or_else(|| SubcError::NoEndpoint {
2819 path: connection_file_path.to_path_buf(),
2820 })?;
2821 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2822 let ip = endpoint
2823 .host
2824 .parse::<IpAddr>()
2825 .map_err(|_| SubcError::InvalidEndpoint {
2826 path: connection_file_path.to_path_buf(),
2827 endpoint: endpoint_label.clone(),
2828 })?;
2829 let addr = SocketAddr::new(ip, endpoint.port);
2830
2831 let connect_budget = deadline.saturating_duration_since(Instant::now());
2832 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2833 .await
2834 .map_err(|_| SubcError::Connect {
2835 endpoint: endpoint_label.clone(),
2836 source: io::Error::new(
2837 io::ErrorKind::TimedOut,
2838 "initial subc attach retry budget elapsed during TCP connect",
2839 ),
2840 })?
2841 .map_err(|source| SubcError::Connect {
2842 endpoint: endpoint_label.clone(),
2843 source,
2844 })?;
2845 stream
2846 .set_nodelay(true)
2847 .map_err(|source| SubcError::Connect {
2848 endpoint: endpoint_label.clone(),
2849 source,
2850 })?;
2851
2852 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2853 authenticate_client(&mut stream, &conn, auth_budget)
2854 .await
2855 .map_err(|source| SubcError::Auth {
2856 endpoint: endpoint_label,
2857 source,
2858 })?;
2859
2860 Ok(stream)
2861}
2862
2863#[allow(clippy::too_many_arguments)]
2864async fn process_route_bind_completion(
2865 writer_tx: &WriterSender,
2866 completion: RouteBindCompletion,
2867 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2868 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2869 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2870 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2871 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2872 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2873 installed_route_epochs: &mut HashMap<u16, u32>,
2874 executor: &Arc<Executor>,
2875 standing_actor: &standing::StandingActor,
2876 shutdown: &Arc<Notify>,
2877 metrics: &Arc<DispatchPathMetrics>,
2878 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2879) -> Result<(), SubcError> {
2880 decrement_counted_channel(&metrics.control_completion_queued);
2881 handle_route_bind_completion(
2882 writer_tx,
2883 completion,
2884 routes,
2885 root_channels,
2886 session_identity,
2887 push_buffer,
2888 live_roots,
2889 pending_binds,
2890 installed_route_epochs,
2891 executor,
2892 standing_actor,
2893 shutdown,
2894 metrics,
2895 lifecycle_probe,
2896 )
2897 .await
2898}
2899
2900#[allow(clippy::too_many_arguments)]
2901async fn drain_pending_route_bind_completions(
2902 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2903 writer_tx: &WriterSender,
2904 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2905 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2906 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2907 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2908 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2909 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2910 installed_route_epochs: &mut HashMap<u16, u32>,
2911 executor: &Arc<Executor>,
2912 standing_actor: &standing::StandingActor,
2913 shutdown: &Arc<Notify>,
2914 metrics: &Arc<DispatchPathMetrics>,
2915 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2916) -> Result<usize, SubcError> {
2917 let mut drained = 0;
2918 while let Ok(completion) = control_completion_rx.try_recv() {
2919 process_route_bind_completion(
2920 writer_tx,
2921 completion,
2922 routes,
2923 root_channels,
2924 session_identity,
2925 push_buffer,
2926 live_roots,
2927 pending_binds,
2928 installed_route_epochs,
2929 executor,
2930 standing_actor,
2931 shutdown,
2932 metrics,
2933 lifecycle_probe,
2934 )
2935 .await?;
2936 drained += 1;
2937 }
2938 Ok(drained)
2939}
2940
2941async fn run_module_loop<R, W>(
2945 mut read: R,
2946 mut write: W,
2947 connection_file_path: &Path,
2948 shared_app: Arc<App>,
2949 executor: Arc<Executor>,
2950 dispatch: DispatchFn,
2951 user_config_path: Option<PathBuf>,
2952 allow_native_passthrough: bool,
2953 tool_response_body_limit: usize,
2954 lifecycle_probe: Option<SubcTestLifecycleProbe>,
2955) -> Result<ModuleLoopExit, SubcError>
2956where
2957 R: AsyncRead + Unpin + Send + 'static,
2958 W: AsyncWrite + Unpin + Send + 'static,
2959{
2960 let hello = ModuleHelloBody {
2965 manifest: build_manifest(),
2966 protocol_ver: PROTOCOL_VERSION,
2967 control_ops: control_ops(),
2968 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2969 };
2970 let hello_frame = Frame::build(
2971 FrameType::Hello,
2972 control_flags(),
2973 0,
2974 0,
2975 HELLO_CORR,
2976 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2977 )
2978 .map_err(SubcError::FrameBuild)?;
2979 write_frame(&mut write, &hello_frame)
2980 .await
2981 .map_err(SubcError::FrameIo)?;
2982
2983 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2985 None => return Err(SubcError::ClosedBeforeHelloAck),
2986 Some(frame) => match frame.header.ty {
2987 FrameType::HelloAck => {
2988 log::info!("subc attach: registered (HelloAck received)");
2989 }
2990 FrameType::Error => {
2991 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2992 return Err(SubcError::HelloRejected { body });
2993 }
2994 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2995 },
2996 }
2997
2998 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2999 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3000 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3001 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3008 let reader_task = spawn_reader_task(read, reader_tx);
3009 let shutdown = Arc::new(Notify::new());
3010 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3019 let mut next_maintenance_at = next_drain_at;
3020 let standing_actor =
3021 standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3022 standing_actor.reconcile_at_startup();
3025 let mut next_standing_pass_at = tokio::time::Instant::now();
3026 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3027 let (bash_deferred_tx, mut bash_deferred_rx) =
3028 mpsc::channel::<bash::BashDeferredCompletion>(256);
3029 let (deferred_response_tx, mut deferred_response_rx) =
3030 mpsc::unbounded_channel::<PendingSubcResponse>();
3031 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3032 let (control_completion_tx, mut control_completion_rx) =
3033 mpsc::channel::<RouteBindCompletion>(256);
3034 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3035 let lossy_overflow = Arc::new(push::LossyOverflow::default());
3036 let lossy_seq = Arc::new(AtomicU64::new(0));
3037 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3038 let (fleet_status_client, fleet_status_task) =
3039 spawn_fleet_status_dial(connection_file_path, 64);
3040 let push_senders = PushSenders {
3041 lossy_tx,
3042 reliable_tx,
3043 lossy_overflow: Arc::clone(&lossy_overflow),
3044 lossy_seq,
3045 fleet_status_client: fleet_status_client.clone(),
3046 };
3047 let connection_cancel = PersistentCancelSignal::new();
3048 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3049 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3050 let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3051 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3052 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3053 let mut bg_wake_pending = BgWakePending::new();
3054 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3055 let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3056 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3057 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3058 HashMap::new();
3059 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3060 let mut retry_buffer: RetryBuffer = HashMap::new();
3061 let mut reclaimed_routes = ReclaimedRoutes::default();
3062 let mut completed_tasks = push::CompletedTaskIds::default();
3063 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3064 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3065 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3066 let mut next_bash_ask_corr: u64 = 1;
3067 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3068 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3069 let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3070 let mut pending_responses = PendingSubcResponses::default();
3071 let health_rollup_cache = Arc::new(HealthRollupCache::new());
3072 let health_rollup_worker = HealthRollupWorker::start(
3073 Arc::clone(&health_rollup_cache),
3074 Arc::clone(&executor),
3075 Arc::clone(&shared_app),
3076 );
3077
3078 let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3079 shared_app.set_open_route_count(routes.len() + management_routes.len());
3080 crate::logging::perf_tick(Some(&executor));
3081 dispatch_path_metrics.mark_frame_loop_tick();
3082 let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3083 for resolved in ready_inspects {
3084 if let Err(error) = deliver_resolved_subc_response(
3085 &writer_tx,
3086 resolved,
3087 &routes,
3088 &mut live_roots,
3089 executor.as_ref(),
3090 &active_tool_calls,
3091 &shutdown,
3092 &dispatch_path_metrics,
3093 tool_response_body_limit,
3094 )
3095 .await
3096 {
3097 break 'module_loop Err(error);
3098 }
3099 }
3100 if let Err(error) = expire_pending_bash_asks(
3101 &writer_tx,
3102 &mut pending_bash_asks,
3103 &routes,
3104 &mut live_roots,
3105 &mut route_bash_cancels,
3106 &shutdown,
3107 &dispatch_path_metrics,
3108 )
3109 .await
3110 {
3111 break Err(error);
3112 }
3113
3114 match drain_pending_route_bind_completions(
3118 &mut control_completion_rx,
3119 &writer_tx,
3120 &mut routes,
3121 &mut root_channels,
3122 &mut session_identity,
3123 &mut push_buffer,
3124 &mut live_roots,
3125 &mut pending_binds,
3126 &mut installed_route_epochs,
3127 &executor,
3128 &standing_actor,
3129 &shutdown,
3130 &dispatch_path_metrics,
3131 lifecycle_probe.as_ref(),
3132 )
3133 .await
3134 {
3135 Ok(drained) => {
3136 if drained > 0 {
3137 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3138 health_rollup_worker.request_refresh();
3139 }
3140 }
3141 Err(error) => break Err(error),
3142 }
3143
3144 if tokio::time::Instant::now() >= next_drain_at {
3145 push::emit_bg_event_wakes(
3146 &writer_tx,
3147 &dispatch_path_metrics,
3148 &bg_subs,
3149 &mut bg_wake_pending,
3150 );
3151 dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3152 warn_slow_pending_binds(&mut pending_binds, &executor);
3153 warn_slow_running_interactive_jobs(&executor);
3154 if let Err(error) = expire_overdue_route_binds(
3155 &writer_tx,
3156 &executor,
3157 &mut pending_binds,
3158 &mut installed_route_epochs,
3159 &dispatch_path_metrics,
3160 )
3161 .await
3162 {
3163 break Err(error);
3164 }
3165
3166 let retried = push::drain_retry_buffers_for_bound_routes(
3167 &writer_tx,
3168 &dispatch_path_metrics,
3169 &routes,
3170 &mut retry_buffer,
3171 );
3172 if retried > 0 {
3173 log::debug!(
3174 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3175 );
3176 }
3177
3178 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3179 }
3180
3181 let overflow_batch = lossy_overflow.drain();
3187 if !overflow_batch.is_empty() {
3188 let (_, deferred) = push::drain_reliable_push_turn(
3189 &writer_tx,
3190 &dispatch_path_metrics,
3191 &routes,
3192 &root_channels,
3193 &session_identity,
3194 &mut retry_buffer,
3195 &mut push_buffer,
3196 &mut completed_tasks,
3197 &bg_sub_by_session,
3198 &mut bg_wake_pending,
3199 &mut bg_wake_epoch,
3200 &mut reliable_rx,
3201 None,
3202 lifecycle_probe.as_ref(),
3203 );
3204 if deferred {
3205 tokio::task::yield_now().await;
3206 }
3207
3208 let mut batch = Vec::new();
3209 while let Ok(item) = lossy_rx.try_recv() {
3210 batch.push(item);
3211 }
3212 batch.extend(overflow_batch);
3213 push::process_lossy_push_envelope_batch(
3214 &writer_tx,
3215 &dispatch_path_metrics,
3216 &routes,
3217 &root_channels,
3218 &completed_tasks,
3219 batch,
3220 );
3221 }
3222
3223 tokio::select! {
3224 biased;
3225 Some(completion) = control_completion_rx.recv() => {
3226 if let Err(error) = process_route_bind_completion(
3227 &writer_tx,
3228 completion,
3229 &mut routes,
3230 &mut root_channels,
3231 &mut session_identity,
3232 &mut push_buffer,
3233 &mut live_roots,
3234 &mut pending_binds,
3235 &mut installed_route_epochs,
3236 &executor,
3237 &standing_actor,
3238 &shutdown,
3239 &dispatch_path_metrics,
3240 lifecycle_probe.as_ref(),
3241 )
3242 .await
3243 {
3244 break Err(error);
3245 }
3246 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3247 health_rollup_worker.request_refresh();
3248 }
3249 _ = shutdown.notified() => {
3250 log::warn!("subc attach: fatal executor response requested teardown");
3251 break Ok(ModuleLoopExit::SkipSearchFlush);
3252 }
3253 maybe_frame = reader_rx.recv() => {
3254 let frame = match maybe_frame {
3255 None => {
3256 log::warn!(
3257 "subc attach: daemon connection ended without Goodbye; exiting for restart"
3258 );
3259 break Ok(ModuleLoopExit::ConnectionLost);
3260 }
3261 Some(Err(error)) => break Err(error),
3262 Some(Ok(frame)) => frame,
3263 };
3264 let phase_trace = frame.phase_trace;
3265 let frame = frame.frame;
3266
3267 if !ingress_route_should_be_processed(
3268 &installed_route_epochs,
3269 &reclaimed_routes,
3270 &frame,
3271 ) {
3272 log::debug!(
3273 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3274 frame.header.ty,
3275 frame.header.channel,
3276 frame.header.epoch
3277 );
3278 continue;
3279 }
3280
3281 match frame.header.ty {
3282 FrameType::Ping if frame.header.channel == 0 => {
3283 let pong = match Frame::build_with_version(
3284 frame.header.ver,
3285 FrameType::Pong,
3286 frame.header.flags,
3287 0,
3288 0,
3289 frame.header.corr,
3290 Vec::new(),
3291 ) {
3292 Ok(pong) => pong,
3293 Err(error) => break Err(SubcError::FrameBuild(error)),
3294 };
3295 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3296 break Err(error);
3297 }
3298 }
3299 FrameType::Goodbye if frame.header.channel == 0 => {
3300 log::info!("subc attach: received channel-0 Goodbye");
3301 break Ok(ModuleLoopExit::Graceful);
3302 }
3303 FrameType::Goodbye => {
3304 let channel = route_key(frame.header.channel, frame.header.epoch);
3305 if let Err(error) = teardown_installed_route(
3306 &writer_tx,
3307 &dispatch_path_metrics,
3308 &executor,
3309 channel,
3310 "Goodbye",
3311 None,
3312 &mut installed_route_epochs,
3313 &mut routes,
3314 &mut management_routes,
3315 &mut root_channels,
3316 &mut bg_subs,
3317 &mut bg_sub_by_session,
3318 &mut bg_wake_pending,
3319 &mut pending_bash_asks,
3320 &mut live_roots,
3321 &mut route_bash_cancels,
3322 &active_tool_calls,
3323 &mut pending_responses,
3324 &mut pending_binds,
3325 &mut retry_buffer,
3326 &mut push_buffer,
3327 &shutdown,
3328 tool_response_body_limit,
3329 lifecycle_probe.as_ref(),
3330 )
3331 .await
3332 {
3333 break Err(error);
3334 }
3335 }
3336 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3337 if let Err(error) = handle_bash_elicitation_reply(
3338 &writer_tx,
3339 &frame,
3340 &mut pending_bash_asks,
3341 &routes,
3342 &mut live_roots,
3343 &executor,
3344 &shutdown,
3345 &bash_deferred_tx,
3346 &bash_poll_touch_tx,
3347 &dispatch_path_metrics,
3348 &mut route_bash_cancels,
3349 dispatch,
3350 )
3351 .await
3352 {
3353 break Err(error);
3354 }
3355 }
3356 FrameType::Request if frame.header.channel == 0 => {
3357 if let Err(error) = handle_control_request(
3358 &writer_tx,
3359 &frame,
3360 &shared_app,
3361 &executor,
3362 &mut live_roots,
3363 &mut pending_binds,
3364 &mut installed_route_epochs,
3365 &mut routes,
3366 &mut management_routes,
3367 &mut root_channels,
3368 &mut bg_subs,
3369 &mut bg_sub_by_session,
3370 &mut bg_wake_pending,
3371 &mut pending_bash_asks,
3372 &mut route_bash_cancels,
3373 &active_tool_calls,
3374 &mut pending_responses,
3375 &mut retry_buffer,
3376 &mut push_buffer,
3377 &shutdown,
3378 &control_completion_tx,
3379 &dispatch_path_metrics,
3380 lifecycle_probe.as_ref(),
3381 &health_rollup_cache,
3382 &push_senders,
3383 dispatch,
3384 user_config_path.as_deref(),
3385 tool_response_body_limit,
3386 )
3387 .await
3388 {
3389 break Err(error);
3390 }
3391 }
3392 FrameType::Request => {
3393 let route = route_key(frame.header.channel, frame.header.epoch);
3394 let result = if management_routes.contains(&route) {
3395 handle_management_request(
3396 &writer_tx,
3397 &frame,
3398 &shared_app,
3399 &executor,
3400 &live_roots,
3401 &root_channels,
3402 &health_rollup_cache,
3403 &dispatch_path_metrics,
3404 )
3405 .await
3406 } else {
3407 handle_tool_call(
3408 &writer_tx,
3409 &frame,
3410 phase_trace,
3411 &routes,
3412 &pending_binds,
3413 &mut live_roots,
3414 &executor,
3415 &active_tool_calls,
3416 &pending_deferred_setups,
3417 &shutdown,
3418 &connection_cancel,
3419 &bash_deferred_tx,
3420 &bash_poll_touch_tx,
3421 &dispatch_path_metrics,
3422 &mut route_bash_cancels,
3423 &mut pending_bash_asks,
3424 &mut next_bash_ask_corr,
3425 &mut bg_subs,
3426 &mut bg_sub_by_session,
3427 &mut bg_wake_pending,
3428 &mut bg_wake_epoch,
3429 dispatch,
3430 &deferred_response_tx,
3431 allow_native_passthrough,
3432 tool_response_body_limit,
3433 )
3434 .await
3435 };
3436 if let Err(error) = result {
3437 break Err(error);
3438 }
3439 }
3440 FrameType::Cancel => {
3441 let channel = route_key(frame.header.channel, frame.header.epoch);
3442 cancel_active_tool_call(
3443 &active_tool_calls,
3444 executor.as_ref(),
3445 channel,
3446 frame.header.corr,
3447 "Cancel frame",
3448 );
3449 pending_responses.cancel_request(channel, frame.header.corr);
3450 if bg_subs.contains_key(&channel) {
3451 if let Err(error) = end_bg_subscription(
3452 &writer_tx,
3453 &dispatch_path_metrics,
3454 &mut bg_subs,
3455 &mut bg_sub_by_session,
3456 &mut bg_wake_pending,
3457 channel,
3458 routes.get(&channel),
3459 "cancel",
3460 )
3461 .await
3462 {
3463 break Err(error);
3464 }
3465 }
3466 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3467 &writer_tx,
3468 &mut pending_bash_asks,
3469 channel,
3470 frame.header.corr,
3471 &routes,
3472 &mut live_roots,
3473 &mut route_bash_cancels,
3474 &shutdown,
3475 &dispatch_path_metrics,
3476 )
3477 .await
3478 {
3479 break Err(error);
3480 }
3481 }
3482 _ => {}
3486 }
3487 }
3488 Some(pending) = deferred_response_rx.recv() => {
3489 if routes.contains_key(&pending.route)
3490 && active_tool_call_is_registered(
3491 &active_tool_calls,
3492 pending.route,
3493 pending.corr,
3494 )
3495 {
3496 pending_responses.register(pending);
3497 } else {
3498 if let Some(cancellation) = &pending.pending.cancellation {
3499 cancellation.request_cancel();
3500 }
3501 finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3502 }
3503 }
3504 Some((root_id, frame)) = reliable_rx.recv() => {
3505 let (_, deferred) = push::drain_reliable_push_turn(
3509 &writer_tx,
3510 &dispatch_path_metrics,
3511 &routes,
3512 &root_channels,
3513 &session_identity,
3514 &mut retry_buffer,
3515 &mut push_buffer,
3516 &mut completed_tasks,
3517 &bg_sub_by_session,
3518 &mut bg_wake_pending,
3519 &mut bg_wake_epoch,
3520 &mut reliable_rx,
3521 Some((root_id, frame)),
3522 lifecycle_probe.as_ref(),
3523 );
3524 if deferred {
3525 tokio::task::yield_now().await;
3526 }
3527 }
3528 Some((order, root_id, frame)) = lossy_rx.recv() => {
3529 let (_, deferred) = push::drain_reliable_push_turn(
3533 &writer_tx,
3534 &dispatch_path_metrics,
3535 &routes,
3536 &root_channels,
3537 &session_identity,
3538 &mut retry_buffer,
3539 &mut push_buffer,
3540 &mut completed_tasks,
3541 &bg_sub_by_session,
3542 &mut bg_wake_pending,
3543 &mut bg_wake_epoch,
3544 &mut reliable_rx,
3545 None,
3546 lifecycle_probe.as_ref(),
3547 );
3548 if deferred {
3549 tokio::task::yield_now().await;
3550 }
3551
3552 let mut batch = vec![(order, root_id, frame)];
3559 while let Ok(item) = lossy_rx.try_recv() {
3560 batch.push(item);
3561 }
3562 batch.extend(lossy_overflow.drain());
3563 push::process_lossy_push_envelope_batch(
3564 &writer_tx,
3565 &dispatch_path_metrics,
3566 &routes,
3567 &root_channels,
3568 &completed_tasks,
3569 batch,
3570 );
3571 }
3572 Some(done) = bash_deferred_rx.recv() => {
3573 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3574 if let Err(error) = bash::handle_bash_deferred_completion(
3575 &writer_tx,
3576 done,
3577 &routes,
3578 &mut live_roots,
3579 &mut route_bash_cancels,
3580 &shutdown,
3581 &dispatch_path_metrics,
3582 )
3583 .await
3584 {
3585 break Err(error);
3586 }
3587 }
3588 Some(root_id) = bash_poll_touch_rx.recv() => {
3589 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3590 if let Some(meta) = live_roots.get_mut(&root_id) {
3591 meta.note_activity();
3592 }
3593 }
3594 Some(completion) = maintenance_rx.recv() => {
3595 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3596 let root_id = completion.root_id.clone();
3597 let response = completion.response;
3598 let response_is_fatal = response_is_fatal_panic(&response);
3599 let bind_pending = pending_binds
3600 .values()
3601 .any(|pending| pending.bind_root_id == root_id);
3602 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3603 let defer_requeue = meta.unbound_quiesced || bind_pending;
3604 note_maintenance_completion(
3605 meta,
3606 completion.requeue_kind,
3607 response_is_fatal,
3608 defer_requeue,
3609 );
3610 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3611 } else {
3612 false
3613 };
3614 if requiesce {
3615 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3616 }
3617 push::clear_stale_bg_wakes_for_empty_sessions(
3618 &root_id,
3619 &completion.empty_bg_sessions,
3620 &bg_sub_by_session,
3621 &mut bg_wake_pending,
3622 &bg_wake_epoch,
3623 );
3624 if let Some(keys) = completion.unacked_bg_keys {
3625 bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3626 }
3627 record_bg_runtime_from_snapshots(
3628 &dispatch_path_metrics,
3629 bg_subs.len(),
3630 bg_wake_pending.len(),
3631 &bg_unacked_keys_by_root,
3632 );
3633 if response_is_fatal {
3634 if let Some(meta) = live_roots.get_mut(&root_id) {
3635 meta.maintenance_poisoned = true;
3636 }
3637 log::warn!(
3638 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3639 );
3640 }
3641 }
3642 _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3643 }
3646 _ = tokio::time::sleep_until(next_drain_at) => {
3647 }
3650 _ = tokio::time::sleep_until(next_maintenance_at) => {
3651 crate::logging::maybe_sweep_logs();
3656 crate::db::compression_events::maybe_spawn_retention(shared_app.db());
3657 let reaped_lsp_children = shared_app
3658 .lsp_child_registry()
3659 .reap_children_with_gone_cwd_or_reclaimed_root();
3660 if reaped_lsp_children > 0 {
3661 log::warn!(
3662 "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3663 );
3664 }
3665 let now = Instant::now();
3666 reap_idle_lsp_servers(now, &live_roots, &executor);
3667 let reap = reap_idle_roots(
3668 now,
3669 &mut live_roots,
3670 &pending_binds,
3671 &root_channels,
3672 &executor,
3673 &dispatch_path_metrics,
3674 );
3675 for root_id in &reap.forgotten_deleted_roots {
3676 bg_unacked_keys_by_root.remove(root_id);
3677 purge_deleted_root_residents(
3678 root_id,
3679 &mut routes,
3680 &mut root_channels,
3681 &mut installed_route_epochs,
3682 &mut route_bash_cancels,
3683 &active_tool_calls,
3684 executor.as_ref(),
3685 &mut retry_buffer,
3686 &mut reclaimed_routes,
3687 &mut session_identity,
3688 &mut push_buffer,
3689 &mut bg_subs,
3690 &mut bg_sub_by_session,
3691 &mut bg_wake_pending,
3692 &mut bg_wake_epoch,
3693 &mut pending_bash_asks,
3694 &dispatch_path_metrics,
3695 );
3696 }
3697 if reap.evicted > 0 {
3698 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3699 }
3700 record_bg_runtime_from_snapshots(
3701 &dispatch_path_metrics,
3702 bg_subs.len(),
3703 bg_wake_pending.len(),
3704 &bg_unacked_keys_by_root,
3705 );
3706 submit_due_maintenance_jobs(
3707 &executor,
3708 &mut live_roots,
3709 &pending_binds,
3710 &bg_sub_by_session,
3711 &bg_wake_pending,
3712 &bg_wake_epoch,
3713 &maintenance_tx,
3714 &dispatch_path_metrics,
3715 );
3716 if tokio::time::Instant::now() >= next_standing_pass_at {
3717 standing_actor.tick();
3718 next_standing_pass_at = tokio::time::Instant::now()
3719 + standing::STANDING_MAINTENANCE_INTERVAL;
3720 }
3721 #[cfg(any(target_os = "macos", target_os = "linux"))]
3728 {
3729 let now_std = std::time::Instant::now();
3730 let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3731 }
3732 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3733 }
3734 }
3735 };
3736
3737 shared_app.set_open_route_count(0);
3738 health_rollup_worker.shutdown();
3739
3740 connection_cancel.cancel();
3741 cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3742 let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3743 while pending_deferred_setups.load(Ordering::SeqCst) != 0
3744 && tokio::time::Instant::now() < setup_drain_deadline
3745 {
3746 tokio::select! {
3747 biased;
3748 Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3749 _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3750 }
3751 }
3752 if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3753 log::warn!(
3754 "subc attach: timed out waiting for deferred response setup registration during shutdown"
3755 );
3756 }
3757 while let Ok(pending) = deferred_response_rx.try_recv() {
3758 pending_responses.register(pending);
3759 }
3760 for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3761 if let Err(error) = deliver_resolved_subc_response(
3762 &writer_tx,
3763 resolved,
3764 &routes,
3765 &mut live_roots,
3766 executor.as_ref(),
3767 &active_tool_calls,
3768 &shutdown,
3769 &dispatch_path_metrics,
3770 tool_response_body_limit,
3771 )
3772 .await
3773 {
3774 log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3775 }
3776 }
3777 quiesce_connection_roots(
3780 &mut live_roots,
3781 &mut pending_binds,
3782 &mut routes,
3783 &mut root_channels,
3784 &mut installed_route_epochs,
3785 &mut route_bash_cancels,
3786 &active_tool_calls,
3787 &executor,
3788 );
3789
3790 fleet_status_client.set_route_live(false);
3791 fleet_status_task.abort();
3792 let _ = fleet_status_task.await;
3793
3794 let mut loop_result = loop_result;
3795 if !pending_bash_asks.is_empty() {
3796 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3797 if let Err(error) = settle_all_pending_bash_asks(
3798 &writer_tx,
3799 &mut pending_bash_asks,
3800 &no_routes,
3801 &mut live_roots,
3802 &mut route_bash_cancels,
3803 &shutdown,
3804 &dispatch_path_metrics,
3805 )
3806 .await
3807 {
3808 loop_result = loop_result.and(Err(error));
3809 }
3810 }
3811
3812 reader_task.abort();
3815 drop(writer_tx);
3816 let writer_result = finish_writer_task(writer_task).await;
3817 loop_result.and_then(|exit| writer_result.map(|_| exit))
3818}
3819
3820fn spawn_writer_task<W>(
3821 mut write: W,
3822 mut rx: mpsc::Receiver<WriterFrame>,
3823 metrics: Arc<DispatchPathMetrics>,
3824) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3825where
3826 W: AsyncWrite + Unpin + Send + 'static,
3827{
3828 tokio::spawn(async move {
3829 let mut write_buffer = Vec::new();
3830 while let Some(mut queued) = rx.recv().await {
3831 let measure = queued.tool_response_trace.is_some();
3832 let dequeued = measure.then(Instant::now);
3833 metrics.writer_active.store(true, Ordering::Relaxed);
3834 decrement_counted_channel(&metrics.writer_queued);
3835 let write_timing = write_frame_contiguous(
3836 &mut write,
3837 queued.frame(),
3838 queued.body(),
3839 &mut write_buffer,
3840 measure,
3841 )
3842 .await;
3843 metrics.writer_active.store(false, Ordering::Relaxed);
3844 let write_timing = write_timing?;
3845
3846 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3847 (queued.tool_response_trace.take(), dequeued, write_timing)
3848 {
3849 if let Some(completed) = trace.finish(
3850 dequeued,
3851 write_timing.write_started,
3852 write_timing.write_finished,
3853 write_timing.frame_bytes,
3854 ) {
3855 log_ctx::with_session(Some(completed.session), || {
3856 crate::logging::note_tool_call_trace(
3857 &completed.name,
3858 &completed.root,
3859 completed.channel,
3860 completed.corr,
3861 completed.phases,
3862 );
3863 });
3864 }
3865 }
3866 }
3867 Ok(())
3868 })
3869}
3870
3871struct FrameWriteTiming {
3872 write_started: Instant,
3873 write_finished: Instant,
3874 frame_bytes: usize,
3875}
3876
3877async fn write_frame_contiguous<W>(
3881 writer: &mut W,
3882 frame: &Frame,
3883 body: &[u8],
3884 buffer: &mut Vec<u8>,
3885 measure: bool,
3886) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3887where
3888 W: AsyncWrite + Unpin,
3889{
3890 if frame.header.len as usize != body.len() {
3891 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3892 header_len: frame.header.len,
3893 body_len: body.len(),
3894 });
3895 }
3896
3897 let header = frame.header.encode();
3898 buffer.clear();
3899 buffer.reserve(header.len() + body.len());
3900 buffer.extend_from_slice(&header);
3901 buffer.extend_from_slice(body);
3902 let write_started = measure.then(Instant::now);
3903 writer
3904 .write_all(buffer)
3905 .await
3906 .map_err(subc_transport::FrameIoError::Io)?;
3907 Ok(write_started.map(|write_started| FrameWriteTiming {
3908 write_started,
3909 write_finished: Instant::now(),
3910 frame_bytes: buffer.len(),
3911 }))
3912}
3913
3914fn spawn_reader_task<R>(
3915 mut read: R,
3916 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3917) -> JoinHandle<()>
3918where
3919 R: AsyncRead + Unpin + Send + 'static,
3920{
3921 tokio::spawn(async move {
3922 loop {
3923 match read_frame(&mut read).await {
3924 Ok(Some(frame)) => {
3925 let decoded = DecodedFrame {
3926 frame,
3927 phase_trace: PhaseTrace::new(Instant::now()),
3928 };
3929 if tx.send(Ok(decoded)).await.is_err() {
3930 return;
3931 }
3932 }
3933 Ok(None) => {
3934 return;
3936 }
3937 Err(error) => {
3938 if let subc_transport::FrameIoError::Io(io_error) = &error {
3945 if matches!(
3946 io_error.kind(),
3947 std::io::ErrorKind::ConnectionReset
3948 | std::io::ErrorKind::ConnectionAborted
3949 ) {
3950 log::info!(
3951 "subc attach: connection reset by daemon; treating as close"
3952 );
3953 return;
3954 }
3955 }
3956 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3957 return;
3958 }
3959 }
3960 }
3961 })
3962}
3963
3964async fn finish_writer_task(
3965 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3966) -> Result<(), SubcError> {
3967 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3968 Ok(Ok(Ok(()))) => Ok(()),
3969 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3970 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3971 Err(_) => {
3972 writer_task.abort();
3973 Ok(())
3974 }
3975 }
3976}
3977
3978fn register_actor_for_bind(
3979 shared_app: &Arc<App>,
3980 executor: &Arc<Executor>,
3981 push_senders: &PushSenders,
3982 bind_root_id: &ProjectRootId,
3983 route_channel: u16,
3984 root_was_live: bool,
3985) -> bool {
3986 if executor.actor_registered(bind_root_id) {
3987 log::debug!(
3988 "subc attach: reusing actor for route {} root {}",
3989 route_channel,
3990 bind_root_id.as_path().display()
3991 );
3992 return false;
3993 }
3994
3995 if root_was_live {
3996 log::warn!(
3997 "subc attach: recreating missing actor for live root {} on route {}",
3998 bind_root_id.as_path().display(),
3999 route_channel
4000 );
4001 }
4002
4003 let actor_ctx = Arc::new(AppContext::from_app(
4004 Arc::clone(shared_app),
4005 Config::default(),
4006 ));
4007 install_bash_compressor(&actor_ctx);
4008 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4009 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4010 push_senders.clone(),
4011 bind_root_id.clone(),
4012 )));
4013 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4014 drop(actor_ctx);
4015 if inserted {
4016 log::debug!(
4020 "subc attach: registered actor for route {} root {}",
4021 route_channel,
4022 bind_root_id.as_path().display()
4023 );
4024 } else {
4025 log::debug!(
4026 "subc attach: actor appeared while binding route {} root {}; reusing it",
4027 route_channel,
4028 bind_root_id.as_path().display()
4029 );
4030 }
4031 inserted
4032}
4033
4034fn rollback_pending_bind_actor(
4035 executor: &Arc<Executor>,
4036 live_roots: &HashMap<ProjectRootId, RootMeta>,
4037 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4038 root_id: &ProjectRootId,
4039 inserted_new_actor: bool,
4040) {
4041 if !inserted_new_actor || live_roots.contains_key(root_id) {
4042 return;
4043 }
4044
4045 if let Some((route, pending)) = pending_binds
4046 .iter_mut()
4047 .find(|(_, pending)| &pending.bind_root_id == root_id)
4048 {
4049 pending.inserted_new_actor = true;
4050 log::debug!(
4051 "subc attach: transferred rollback ownership for root {} to pending route {}",
4052 root_id.as_path().display(),
4053 route
4054 );
4055 return;
4056 }
4057
4058 executor.remove_actor(root_id);
4059}
4060
4061fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4062 match response.data.get("code").and_then(|code| code.as_str()) {
4063 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4068 Some("cache_key_probe_failed") => "cache_key_probe_failed",
4072 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4076 _ => "config_divergence",
4077 }
4078}
4079
4080fn queue_post_bind_configure_and_completion_maintenance(
4081 root_id: &ProjectRootId,
4082 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4083) {
4084 let Some(meta) = live_roots.get_mut(root_id) else {
4085 return;
4086 };
4087 if meta.maintenance_poisoned || meta.maintenance_pending {
4088 return;
4089 }
4090
4091 meta.maintenance_pending = true;
4092 meta.maintenance_queued_kinds
4093 .push_back(MaintenanceDrainKind::ConfigureTail);
4094 meta.maintenance_queued_kinds
4095 .push_back(MaintenanceDrainKind::CompletionDrains);
4096}
4097
4098#[allow(clippy::too_many_arguments)]
4099async fn handle_route_bind_completion(
4100 tx: &WriterSender,
4101 completion: RouteBindCompletion,
4102 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4103 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4104 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4105 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4106 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4107 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4108 installed_route_epochs: &mut HashMap<u16, u32>,
4109 executor: &Arc<Executor>,
4110 standing_actor: &standing::StandingActor,
4111 shutdown: &Arc<Notify>,
4112 metrics: &Arc<DispatchPathMetrics>,
4113 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4114) -> Result<(), SubcError> {
4115 let route_id = completion.route;
4116 let Some(pending) = pending_binds.remove(&route_id) else {
4117 log::warn!(
4118 "subc attach: dropping RouteBind completion for non-pending route {}",
4119 completion.route
4120 );
4121 rollback_pending_bind_actor(
4122 executor,
4123 live_roots,
4124 pending_binds,
4125 &completion.bind_root_id,
4126 completion.inserted_new_actor,
4127 );
4128 let has_pending_bind = pending_binds
4129 .values()
4130 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4131 if !root_channels
4132 .get(&completion.bind_root_id)
4133 .is_some_and(|channels| !channels.is_empty())
4134 && !has_pending_bind
4135 {
4136 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4137 }
4138 remove_installed_route(installed_route_epochs, route_id);
4139 return Ok(());
4140 };
4141
4142 if pending.bind_root_id != completion.bind_root_id {
4143 log::warn!(
4144 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4145 completion.route,
4146 pending.bind_root_id.as_path().display(),
4147 completion.bind_root_id.as_path().display()
4148 );
4149 }
4150
4151 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4152 if pending.cancelled {
4153 rollback_pending_bind_actor(
4154 executor,
4155 live_roots,
4156 pending_binds,
4157 &completion.bind_root_id,
4158 inserted_new_actor,
4159 );
4160 let has_pending_bind = pending_binds
4161 .values()
4162 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4163 if !root_channels
4164 .get(&completion.bind_root_id)
4165 .is_some_and(|channels| !channels.is_empty())
4166 && !has_pending_bind
4167 {
4168 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4169 }
4170 log::debug!(
4171 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4172 completion.route,
4173 completion.bind_root_id.as_path().display()
4174 );
4175 remove_installed_route(installed_route_epochs, route_id);
4176 return Ok(());
4177 }
4178
4179 let failure = if !completion.configure_response.success {
4180 Some((
4181 &completion.configure_response,
4182 "configure failed during route bind",
4183 ))
4184 } else {
4185 None
4186 };
4187
4188 if let Some((response, fallback)) = failure {
4189 rollback_pending_bind_actor(
4190 executor,
4191 live_roots,
4192 pending_binds,
4193 &completion.bind_root_id,
4194 inserted_new_actor,
4195 );
4196 let has_pending_bind = pending_binds
4197 .values()
4198 .any(|pending| pending.bind_root_id == completion.bind_root_id);
4199 if !root_channels
4200 .get(&completion.bind_root_id)
4201 .is_some_and(|channels| !channels.is_empty())
4202 && !has_pending_bind
4203 {
4204 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4205 }
4206 let message = response_message(response, fallback);
4207 let fatal = response_is_fatal_panic(response);
4208 let error_code = route_bind_error_code_for_configure_response(response);
4209 send_route_bind_error_parts(
4210 tx,
4211 completion.ver,
4212 completion.corr,
4213 completion.flags,
4214 error_code,
4215 &message,
4216 metrics,
4217 )
4218 .await?;
4219 remove_installed_route(installed_route_epochs, route_id);
4220 if fatal {
4221 signal_fatal_teardown(
4222 tx,
4223 Some(completion.route),
4224 completion.ver,
4225 completion.corr,
4226 shutdown,
4227 metrics,
4228 )
4229 .await;
4230 }
4231 return Ok(());
4232 }
4233
4234 remember_session_identity(session_identity, &completion.identity);
4235 let replay_key = push::ReplayKey::from_identity(&completion.identity);
4236 let bind_trust = completion.identity.trust;
4237 insert_route_channel(routes, root_channels, route_id, completion.identity);
4238 sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4239 let restore_watcher = live_roots
4240 .get(&completion.bind_root_id)
4241 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4242 live_roots
4243 .entry(completion.bind_root_id.clone())
4244 .and_modify(|meta| {
4245 meta.reactivate_bound();
4246 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4247 meta.maintenance_poisoned = false;
4248 })
4249 .or_insert_with(|| RootMeta::new(Instant::now()));
4250 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4251 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4252 meta.maintenance_poisoned = false;
4253 }
4254 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4255 standing_actor.begin_session_bind(&ctx);
4258 ctx.mark_subc_bound();
4259 if restore_watcher {
4260 crate::commands::configure::ensure_project_watcher(&ctx);
4261 }
4262 }
4263
4264 let ack =
4265 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4266 let response = Frame::build_with_version(
4267 completion.ver,
4268 FrameType::Response,
4269 control_flags(),
4270 0,
4271 0,
4272 completion.corr,
4273 ack,
4274 )
4275 .map_err(SubcError::FrameBuild)?;
4276 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4277 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4278 let replayed = push::replay_buffered_push_frames(
4279 tx,
4280 metrics,
4281 route_id,
4282 push_buffer,
4283 &replay_key,
4284 bind_trust,
4285 lifecycle_probe,
4286 );
4287 if replayed > 0 {
4288 log::debug!(
4289 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4290 replayed,
4291 completion.route,
4292 replay_key.root.as_path().display(),
4293 replay_key.harness,
4294 replay_key.session
4295 );
4296 }
4297 log::info!(
4298 "subc attach: route {} bound to root {}",
4299 completion.route,
4300 completion.bind_root_id.as_path().display()
4301 );
4302 Ok(())
4303}
4304
4305async fn expire_overdue_route_binds(
4306 tx: &WriterSender,
4307 executor: &Arc<Executor>,
4308 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4309 installed_route_epochs: &mut HashMap<u16, u32>,
4310 metrics: &DispatchPathMetrics,
4311) -> Result<(), SubcError> {
4312 let now = Instant::now();
4313 let expired: Vec<_> = pending_binds
4314 .iter()
4315 .filter_map(|(route, pending)| {
4316 let age = now.saturating_duration_since(pending.started_at);
4317 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4318 (
4319 *route,
4320 pending.corr,
4321 pending.ver,
4322 pending.flags,
4323 pending.bind_root_id.clone(),
4324 pending.configure_request_id.clone(),
4325 age,
4326 )
4327 })
4328 })
4329 .collect();
4330
4331 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4332 if let Some(pending) = pending_binds.get_mut(&route) {
4333 pending.cancelled = true;
4334 pending.deadline_reported = true;
4335 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4336 log::debug!(
4337 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4338 );
4339 }
4340 remove_installed_route(installed_route_epochs, route);
4341 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4342 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4343 send_route_bind_error_parts(
4344 tx,
4345 ver,
4346 corr,
4347 flags,
4348 "actor_not_ready",
4349 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4350 metrics,
4351 )
4352 .await?;
4353 log::warn!(
4354 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4355 route,
4356 root_id.as_path().display(),
4357 deadline_ms,
4358 configure_request_id
4359 );
4360 }
4361
4362 Ok(())
4363}
4364
4365fn record_bg_runtime_from_snapshots(
4366 metrics: &DispatchPathMetrics,
4367 subscriptions: usize,
4368 wake_pending: usize,
4369 unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4370) {
4371 let unacked_total = unacked_keys_by_root
4372 .values()
4373 .flatten()
4374 .collect::<HashSet<_>>()
4375 .len();
4376 metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4377}
4378
4379async fn send_cached_health_response(
4380 tx: &WriterSender,
4381 frame: &Frame,
4382 shared_app: &App,
4383 executor: &Executor,
4384 pending_binds: &HashMap<RouteChannel, PendingBind>,
4385 metrics: &DispatchPathMetrics,
4386 health_rollup_cache: &HealthRollupCache,
4387) -> Result<(), SubcError> {
4388 let report = build_health_report(
4389 health_rollup_cache,
4390 executor,
4391 pending_binds,
4392 metrics,
4393 shared_app,
4394 );
4395 let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4396 let response = Frame::build_with_version(
4397 frame.header.ver,
4398 FrameType::Response,
4399 frame.header.flags,
4400 0,
4401 0,
4402 frame.header.corr,
4403 body,
4404 )
4405 .map_err(SubcError::FrameBuild)?;
4406 send_frame(tx, metrics, response).await
4407}
4408
4409#[allow(clippy::too_many_arguments)]
4413async fn handle_control_request(
4414 tx: &WriterSender,
4415 frame: &Frame,
4416 shared_app: &Arc<App>,
4417 executor: &Arc<Executor>,
4418 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4419 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4420 installed_route_epochs: &mut HashMap<u16, u32>,
4421 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4422 management_routes: &mut HashSet<RouteChannel>,
4423 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4424 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4425 bg_sub_by_session: &mut BgSubsBySession,
4426 bg_wake_pending: &mut BgWakePending,
4427 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4428 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4429 active_tool_calls: &ActiveToolCalls,
4430 pending_responses: &mut PendingSubcResponses,
4431 retry_buffer: &mut RetryBuffer,
4432 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4433 shutdown: &Arc<Notify>,
4434 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4435 metrics: &Arc<DispatchPathMetrics>,
4436 lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4437 health_rollup_cache: &HealthRollupCache,
4438 push_senders: &PushSenders,
4439 dispatch: DispatchFn,
4440 user_config_path: Option<&Path>,
4441 tool_response_body_limit: usize,
4442) -> Result<(), SubcError> {
4443 let request =
4444 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4445 match request {
4446 ModuleControlRequest::RouteBind {
4447 route_channel,
4448 epoch,
4449 target,
4450 identity,
4451 principal,
4452 consumer_capabilities,
4453 admission_facts: _,
4454 } => {
4455 let route_id = route_key(route_channel, epoch);
4456 if epoch == 0 {
4457 return send_route_bind_error(
4458 tx,
4459 frame,
4460 "config_divergence",
4461 "route bind uses an invalid channel generation",
4462 metrics,
4463 )
4464 .await;
4465 }
4466
4467 let bind_trust = trust_for_bind(&identity.harness, &principal);
4468 if let RouteTarget::ManagementSurface { module_id } = &target {
4469 if module_id != "aft" {
4470 return send_route_bind_error(
4471 tx,
4472 frame,
4473 "route_refused",
4474 "management route target is not AFT",
4475 metrics,
4476 )
4477 .await;
4478 }
4479 if !matches!(bind_trust, BindTrust::FirstParty) {
4480 return send_route_bind_error(
4481 tx,
4482 frame,
4483 "route_refused",
4484 "AFT management routes require a first-party principal",
4485 metrics,
4486 )
4487 .await;
4488 }
4489 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4490 if installed_epoch >= epoch {
4491 return send_route_bind_error(
4492 tx,
4493 frame,
4494 "config_divergence",
4495 "route bind generation is not newer than the installed generation",
4496 metrics,
4497 )
4498 .await;
4499 }
4500 teardown_installed_route(
4501 tx,
4502 metrics,
4503 executor,
4504 route_key(route_channel, installed_epoch),
4505 "higher-epoch RouteBind",
4506 None,
4507 installed_route_epochs,
4508 routes,
4509 management_routes,
4510 root_channels,
4511 bg_subs,
4512 bg_sub_by_session,
4513 bg_wake_pending,
4514 pending_bash_asks,
4515 live_roots,
4516 route_bash_cancels,
4517 active_tool_calls,
4518 pending_responses,
4519 pending_binds,
4520 retry_buffer,
4521 push_buffer,
4522 shutdown,
4523 tool_response_body_limit,
4524 lifecycle_probe,
4525 )
4526 .await?;
4527 }
4528 if pending_binds.contains_key(&route_id) {
4529 return send_route_bind_error(
4530 tx,
4531 frame,
4532 "config_divergence",
4533 "route bind is already pending for channel",
4534 metrics,
4535 )
4536 .await;
4537 }
4538
4539 installed_route_epochs.insert(route_channel, epoch);
4540 management_routes.insert(route_id);
4541 return send_route_bind_ack(
4542 tx,
4543 frame.header.ver,
4544 frame.header.corr,
4545 frame.header.flags,
4546 metrics,
4547 )
4548 .await;
4549 }
4550 if matches!(&target, RouteTarget::InternalService { .. }) {
4551 return send_route_bind_error(
4552 tx,
4553 frame,
4554 "route_refused",
4555 "AFT does not provide an internal-service route",
4556 metrics,
4557 )
4558 .await;
4559 }
4560
4561 let mut bind_root_id = None;
4562 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4563 if installed_epoch >= epoch {
4564 return send_route_bind_error(
4565 tx,
4566 frame,
4567 "config_divergence",
4568 "route bind generation is not newer than the installed generation",
4569 metrics,
4570 )
4571 .await;
4572 }
4573
4574 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4575 Ok(root_id) => root_id,
4576 Err(error) => {
4577 return send_route_bind_error(
4578 tx,
4579 frame,
4580 "config_divergence",
4581 &format!("invalid route project root: {error}"),
4582 metrics,
4583 )
4584 .await;
4585 }
4586 };
4587 teardown_installed_route(
4588 tx,
4589 metrics,
4590 executor,
4591 route_key(route_channel, installed_epoch),
4592 "higher-epoch RouteBind",
4593 Some(&replacement_root),
4594 installed_route_epochs,
4595 routes,
4596 management_routes,
4597 root_channels,
4598 bg_subs,
4599 bg_sub_by_session,
4600 bg_wake_pending,
4601 pending_bash_asks,
4602 live_roots,
4603 route_bash_cancels,
4604 active_tool_calls,
4605 pending_responses,
4606 pending_binds,
4607 retry_buffer,
4608 push_buffer,
4609 shutdown,
4610 tool_response_body_limit,
4611 lifecycle_probe,
4612 )
4613 .await?;
4614 bind_root_id = Some(replacement_root);
4615 }
4616 if pending_binds.contains_key(&route_id) {
4617 return send_route_bind_error(
4618 tx,
4619 frame,
4620 "config_divergence",
4621 "route bind is already pending for channel",
4622 metrics,
4623 )
4624 .await;
4625 }
4626 let bind_root_id = match bind_root_id {
4627 Some(root_id) => root_id,
4628 None => match ProjectRootId::from_path(&identity.project_root) {
4629 Ok(root_id) => root_id,
4630 Err(error) => {
4631 return send_route_bind_error(
4632 tx,
4633 frame,
4634 "config_divergence",
4635 &format!("invalid route project root: {error}"),
4636 metrics,
4637 )
4638 .await;
4639 }
4640 },
4641 };
4642
4643 let request_id = format!("subc-bind-{route_channel}");
4646 let bind_project_root = identity.project_root.clone();
4647 let bind_harness = identity.harness.clone();
4648 let bind_session = identity.session.clone();
4649 let bind_principal_id = principal_id(&principal);
4650 let consumer_elicitation_capable = consumer_capabilities
4655 .as_ref()
4656 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4657 log::info!(
4658 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4659 route_channel,
4660 bind_harness,
4661 principal_label(&principal),
4662 bind_trust.label(),
4663 consumer_elicitation_capable
4664 );
4665
4666 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4671 user_config_path,
4672 Path::new(&bind_project_root),
4673 );
4674 let config_tiers: Vec<Value> = local_tiers
4675 .iter()
4676 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4677 .collect();
4678 let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4681 let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4682 &local_tiers,
4683 active_harness.as_ref(),
4684 )
4685 .config
4686 .diagnostics_on_edit;
4687 let configure_json = json!({
4688 "id": request_id,
4689 "command": "configure",
4690 "project_root": bind_project_root,
4691 "harness": bind_harness,
4692 "session_id": bind_session.clone(),
4693 "config": config_tiers,
4694 });
4695 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4696 Ok(req) => req,
4697 Err(error) => {
4698 return send_route_bind_error(
4699 tx,
4700 frame,
4701 "config_divergence",
4702 &format!("failed to build configure request: {error}"),
4703 metrics,
4704 )
4705 .await;
4706 }
4707 };
4708
4709 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4710 root: bind_root_id.clone(),
4711 project_root: PathBuf::from(&bind_project_root),
4712 harness: bind_harness.clone(),
4713 session: bind_session.clone(),
4714 trust: bind_trust,
4715 spawn_principal: AuthenticatedPrincipal::RouteBind {
4716 trust: bind_trust.sandbox_trust(),
4717 route_channel,
4718 route_epoch: epoch,
4719 project_root: PathBuf::from(&bind_project_root),
4720 harness: bind_harness.clone(),
4721 session_id: bind_session.clone(),
4722 principal_id: bind_principal_id,
4723 },
4724 consumer_elicitation_capable,
4725 }));
4726 let configure_session = route_identity.session.clone();
4727 let root_was_live = live_roots.contains_key(&bind_root_id);
4728 let inserted_new_actor = register_actor_for_bind(
4729 shared_app,
4730 executor,
4731 push_senders,
4732 &bind_root_id,
4733 route_channel,
4734 root_was_live,
4735 );
4736
4737 sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4738 let configure_request_id = configure_req.id.clone();
4739 installed_route_epochs.insert(route_channel, epoch);
4740 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4741 meta.maintenance_queued_kinds.clear();
4742 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4743 }
4744 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4745 bind_root_id.clone(),
4746 Lane::Mutating,
4747 configure_request_id.clone(),
4748 Box::new(move |ctx| {
4749 log_ctx::with_session(Some(configure_session.clone()), || {
4750 dispatch(configure_req, ctx)
4751 })
4752 }),
4753 );
4754 pending_binds.insert(
4755 route_id,
4756 PendingBind {
4757 bind_root_id: bind_root_id.clone(),
4758 inserted_new_actor,
4759 cancelled: false,
4760 configure_request_id: configure_request_id.clone(),
4761 started_at: Instant::now(),
4762 warned_half_deadline: false,
4763 deadline_reported: false,
4764 corr: frame.header.corr,
4765 ver: frame.header.ver,
4766 flags: frame.header.flags,
4767 cancellation: configure_cancellation,
4768 },
4769 );
4770
4771 let completion_tx = control_completion_tx.clone();
4772 let completion_identity = route_identity;
4773 let completion_root = bind_root_id.clone();
4774 let completion_route_channel = route_channel;
4775 let completion_ver = frame.header.ver;
4776 let completion_corr = frame.header.corr;
4777 let completion_flags = frame.header.flags;
4778 let completion_metrics = Arc::clone(metrics);
4779 tokio::spawn(async move {
4780 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4781 let configure_response =
4782 await_executor_response(configure_rx, configure_request_id.clone()).await;
4783 let completion = RouteBindCompletion {
4788 route: route_key(completion_route_channel, epoch),
4789 identity: completion_identity,
4790 bind_root_id: completion_root,
4791 inserted_new_actor,
4792 configure_response,
4793 diagnostics_on_edit,
4794 ver: completion_ver,
4795 corr: completion_corr,
4796 flags: completion_flags,
4797 };
4798 if send_counted_channel(
4799 &completion_tx,
4800 &completion_metrics.control_completion_queued,
4801 completion,
4802 )
4803 .await
4804 .is_err()
4805 {
4806 log::debug!(
4807 "subc attach: dropped RouteBind completion for route {} after loop exit",
4808 completion_route_channel
4809 );
4810 }
4811 });
4812
4813 Ok(())
4816 }
4817 ModuleControlRequest::HealthCheck {} => {
4818 send_cached_health_response(
4819 tx,
4820 frame,
4821 shared_app,
4822 executor,
4823 pending_binds,
4824 metrics,
4825 health_rollup_cache,
4826 )
4827 .await
4828 }
4829 }
4830}
4831
4832async fn handle_management_request(
4833 tx: &WriterSender,
4834 frame: &Frame,
4835 shared_app: &App,
4836 executor: &Executor,
4837 live_roots: &HashMap<ProjectRootId, RootMeta>,
4838 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4839 health_rollup_cache: &HealthRollupCache,
4840 metrics: &DispatchPathMetrics,
4841) -> Result<(), SubcError> {
4842 let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4843 let operation = decoded
4844 .as_ref()
4845 .and_then(|value| value.get("op"))
4846 .and_then(Value::as_str);
4847 let Some(operation) = operation else {
4848 let error = build_error_frame(
4849 frame.header.ver,
4850 frame.header.channel,
4851 frame.header.epoch,
4852 frame.header.corr,
4853 frame.header.flags,
4854 "unknown_management_op",
4855 "management routes accept only declared operation envelopes",
4856 )?;
4857 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4858 };
4859
4860 let result = match operation {
4861 crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4862 "management-memory-census",
4863 memory_census_with_lifecycle(
4864 health_rollup_cache,
4865 shared_app,
4866 executor,
4867 live_roots,
4868 root_channels,
4869 ),
4870 ),
4871 crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4872 let params = decoded
4873 .as_ref()
4874 .and_then(|value| value.get("params"))
4875 .cloned()
4876 .unwrap_or_else(|| json!({}));
4877 let Some(params) = params.as_object() else {
4878 return send_management_response(
4879 tx,
4880 frame,
4881 operation,
4882 Response::error(
4883 "management-health-digest",
4884 "invalid_request",
4885 "health.digest params must be an object",
4886 ),
4887 metrics,
4888 )
4889 .await;
4890 };
4891 let root = params
4892 .get("project_root")
4893 .or_else(|| params.get("root"))
4894 .and_then(Value::as_str);
4895 let Some(root) = root else {
4896 return send_management_response(
4897 tx,
4898 frame,
4899 operation,
4900 Response::error(
4901 "management-health-digest",
4902 "invalid_request",
4903 "health.digest requires params.project_root",
4904 ),
4905 metrics,
4906 )
4907 .await;
4908 };
4909
4910 let mut request = params.clone();
4911 request.insert("id".to_string(), json!("management-health-digest"));
4912 request.insert(
4913 "command".to_string(),
4914 json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4915 );
4916 let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4917 .map_err(SubcError::Json)?;
4918 match ProjectRootId::from_path(Path::new(root))
4919 .ok()
4920 .and_then(|root_id| executor.actor_context(&root_id))
4921 {
4922 Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4923 None => crate::commands::health_digest::root_not_bound_response(&request, root),
4924 }
4925 }
4926 _ => {
4927 let error = build_error_frame(
4928 frame.header.ver,
4929 frame.header.channel,
4930 frame.header.epoch,
4931 frame.header.corr,
4932 frame.header.flags,
4933 "unknown_management_op",
4934 &format!("management operation {operation:?} is not declared by AFT"),
4935 )?;
4936 return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4937 }
4938 };
4939
4940 send_management_response(tx, frame, operation, result, metrics).await
4941}
4942
4943fn memory_census_with_lifecycle(
4944 health_rollup_cache: &HealthRollupCache,
4945 shared_app: &App,
4946 executor: &Executor,
4947 live_roots: &HashMap<ProjectRootId, RootMeta>,
4948 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4949) -> Value {
4950 let mut census = health_rollup_cache.memory_census();
4951 if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4952 let lifecycle = shared_app.lifecycle_census_snapshot();
4953 for (root_id, meta) in live_roots {
4954 let root = root_id.as_path().display().to_string();
4955 let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4956 let age_ms = Instant::now()
4957 .saturating_duration_since(meta.last_touched)
4958 .as_millis()
4959 .min(u128::from(u64::MAX)) as u64;
4960 let ttl_ms = root_idle_ttl(executor, root_id)
4961 .as_millis()
4962 .min(u128::from(u64::MAX)) as u64;
4963 let lsp = lifecycle
4964 .lsp
4965 .children_by_root
4966 .iter()
4967 .find(|child| child.root == root);
4968 if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4969 let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4970 row.insert("root_id".to_string(), json!(root));
4971 row.insert("bound_routes".to_string(), json!(bound_routes));
4972 row.insert("last_request_age_ms".to_string(), json!(age_ms));
4973 row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4974 row.insert(
4975 "lsp_idle_ttl_ms".to_string(),
4976 json!(executor
4977 .actor_context(root_id)
4978 .map(|ctx| ctx
4979 .config()
4980 .idle
4981 .lsp_ttl()
4982 .as_millis()
4983 .min(u128::from(u64::MAX)) as u64)
4984 .unwrap_or(0)),
4985 );
4986 row.insert(
4987 "evictable_in_ms".to_string(),
4988 crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4989 .map_or(Value::Null, |value| json!(value)),
4990 );
4991 row.insert(
4992 "evictable_bytes".to_string(),
4993 if bound_routes == 0 {
4994 evictable_bytes
4995 } else {
4996 json!(0)
4997 },
4998 );
4999 row.insert(
5000 "lsp_children".to_string(),
5001 json!({
5002 "count": lsp.map_or(0, |child| child.count),
5003 "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5004 }),
5005 );
5006 }
5007 }
5008 }
5009 census
5010}
5011
5012async fn send_management_response(
5013 tx: &WriterSender,
5014 request: &Frame,
5015 operation: &str,
5016 result: Response,
5017 metrics: &DispatchPathMetrics,
5018) -> Result<(), SubcError> {
5019 let status = if result.success { "ok" } else { "error" };
5020 let body = json!({ "op": operation, "status": status, "data": result.data });
5021 let response = Frame::build_with_version(
5022 request.header.ver,
5023 FrameType::Response,
5024 request.header.flags,
5025 request.header.channel,
5026 request.header.epoch,
5027 request.header.corr,
5028 serde_json::to_vec(&body).map_err(SubcError::Json)?,
5029 )
5030 .map_err(SubcError::FrameBuild)?;
5031 send_reliable_writer_frame(tx, metrics, response, "management response").await
5032}
5033
5034fn install_bash_compressor(ctx: &AppContext) {
5035 let filter_registry_handle = ctx.shared_filter_registry();
5037 let compress_flag = ctx.bash_compress_flag();
5038 ctx.bash_background().set_compressor_with_exit_code(
5039 move |command: &str, output: String, exit_code: Option<i32>| {
5040 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5041 return crate::compress::CompressionResult::new(output);
5042 }
5043 let registry_guard = match filter_registry_handle.read() {
5044 Ok(g) => g,
5045 Err(poisoned) => poisoned.into_inner(),
5046 };
5047 crate::compress::compress_with_registry_exit_code(
5048 command,
5049 &output,
5050 exit_code,
5051 ®istry_guard,
5052 )
5053 },
5054 );
5055}
5056
5057async fn send_route_bind_ack(
5058 tx: &WriterSender,
5059 ver: u8,
5060 corr: u64,
5061 flags: Flags,
5062 metrics: &DispatchPathMetrics,
5063) -> Result<(), SubcError> {
5064 let body =
5065 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5066 let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5067 .map_err(SubcError::FrameBuild)?;
5068 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5069}
5070
5071async fn send_route_bind_error(
5072 tx: &WriterSender,
5073 frame: &Frame,
5074 code: &str,
5075 message: &str,
5076 metrics: &DispatchPathMetrics,
5077) -> Result<(), SubcError> {
5078 send_route_bind_error_parts(
5079 tx,
5080 frame.header.ver,
5081 frame.header.corr,
5082 frame.header.flags,
5083 code,
5084 message,
5085 metrics,
5086 )
5087 .await
5088}
5089
5090async fn send_route_bind_error_parts(
5091 tx: &WriterSender,
5092 ver: u8,
5093 corr: u64,
5094 flags: Flags,
5095 code: &str,
5096 message: &str,
5097 metrics: &DispatchPathMetrics,
5098) -> Result<(), SubcError> {
5099 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5100 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5101 log_route_bind_rejection(code, message);
5102 Ok(())
5103}
5104
5105fn log_route_bind_rejection(code: &str, message: &str) {
5113 const WINDOW: Duration = Duration::from_secs(60);
5114 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5115 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5116 let mut map = match map.try_lock() {
5117 Ok(map) => map,
5118 Err(_) => {
5120 log::warn!("subc attach: route bind rejected ({code}): {message}");
5121 return;
5122 }
5123 };
5124 let now = Instant::now();
5125 if map.len() > 512 {
5129 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5130 }
5131 match map.get_mut(message) {
5132 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5133 *suppressed += 1;
5134 }
5135 Some((window_start, suppressed)) => {
5136 if *suppressed > 0 {
5137 log::warn!(
5138 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5139 *suppressed
5140 );
5141 } else {
5142 log::warn!("subc attach: route bind rejected ({code}): {message}");
5143 }
5144 *window_start = now;
5145 *suppressed = 0;
5146 }
5147 None => {
5148 log::warn!("subc attach: route bind rejected ({code}): {message}");
5149 map.insert(message.to_string(), (now, 0));
5150 }
5151 }
5152}
5153
5154async fn handle_tool_call(
5159 tx: &WriterSender,
5160 frame: &Frame,
5161 mut phase_trace: PhaseTrace,
5162 routes: &HashMap<RouteChannel, RouteIdentity>,
5163 pending_binds: &HashMap<RouteChannel, PendingBind>,
5164 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5165 executor: &Arc<Executor>,
5166 active_tool_calls: &ActiveToolCalls,
5167 pending_deferred_setups: &Arc<AtomicUsize>,
5168 shutdown: &Arc<Notify>,
5169 connection_cancel: &PersistentCancelSignal,
5170 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5171 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5172 metrics: &Arc<DispatchPathMetrics>,
5173 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5174 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5175 next_bash_ask_corr: &mut u64,
5176 bg_subs: &mut HashMap<RouteChannel, BgSub>,
5177 bg_sub_by_session: &mut BgSubsBySession,
5178 bg_wake_pending: &mut BgWakePending,
5179 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5180 dispatch: DispatchFn,
5181 deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5182 allow_native_passthrough: bool,
5183 tool_response_body_limit: usize,
5184) -> Result<(), SubcError> {
5185 let route_id = route_key(frame.header.channel, frame.header.epoch);
5186 if pending_binds.contains_key(&route_id) {
5187 let error = build_error_frame(
5188 frame.header.ver,
5189 frame.header.channel,
5190 frame.header.epoch,
5191 frame.header.corr,
5192 frame.header.flags,
5193 "route_not_bound",
5194 "route is not bound before tool call",
5195 )?;
5196 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5197 }
5198
5199 let Some(identity) = routes.get(&route_id).cloned() else {
5200 let error = build_error_frame(
5201 frame.header.ver,
5202 frame.header.channel,
5203 frame.header.epoch,
5204 frame.header.corr,
5205 frame.header.flags,
5206 "route_not_bound",
5207 "route is not bound before tool call",
5208 )?;
5209 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5210 };
5211 let restore_watcher = live_roots
5212 .get(&identity.root)
5213 .is_some_and(|meta| meta.idle_artifacts_evicted);
5214 if let Some(meta) = live_roots.get_mut(&identity.root) {
5215 meta.reactivate_bound();
5216 }
5217 if restore_watcher {
5218 if let Some(ctx) = executor.actor_context(&identity.root) {
5219 crate::commands::configure::ensure_project_watcher(&ctx);
5220 }
5221 }
5222
5223 let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5224 Ok(request) => request,
5225 Err(error) => {
5226 let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5227 let Some(operation) = management_envelope
5228 .as_ref()
5229 .and_then(|value| value.get("op"))
5230 .and_then(Value::as_str)
5231 else {
5232 return Err(SubcError::Json(error));
5233 };
5234 RouteRequest::ToolCall(ToolCallRequest {
5235 name: operation.to_string(),
5236 arguments: management_envelope
5237 .and_then(|value| value.get("params").cloned())
5238 .unwrap_or_else(|| json!({})),
5239 edit_slot_survives: None,
5240 preview: false,
5241 })
5242 }
5243 };
5244 if matches!(
5245 route_request,
5246 RouteRequest::BgEvents(BgEventsRequest {
5247 op: BgEventsOp::BgEvents
5248 })
5249 ) {
5250 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5251 metrics.record_bg_subscription_ended(
5252 &old_sub.root,
5253 &old_sub.session,
5254 route_id,
5255 "resubscribe",
5256 );
5257 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5258 }
5259 if !identity.trust.allows_bash_observation() {
5260 bg_subs.remove(&route_id);
5261 bg_wake_pending.remove(&route_id);
5262 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5263 let denied_sub = BgSub {
5264 corr: frame.header.corr,
5265 ver: frame.header.ver,
5266 flags: frame.header.flags,
5267 root: identity.root.clone(),
5268 session: identity.session.clone(),
5269 };
5270 metrics.record_bg_subscription_ended(
5271 &identity.root,
5272 &identity.session,
5273 route_id,
5274 "subscribe-denied",
5275 );
5276 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5277 return Ok(());
5278 }
5279 bg_subs.insert(
5280 route_id,
5281 BgSub {
5282 corr: frame.header.corr,
5283 ver: frame.header.ver,
5284 flags: frame.header.flags,
5285 root: identity.root.clone(),
5286 session: identity.session.clone(),
5287 },
5288 );
5289 insert_bg_subscription_index(
5290 bg_sub_by_session,
5291 identity.root.clone(),
5292 identity.session.clone(),
5293 route_id,
5294 );
5295 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5296 push::arm_bg_wake(
5297 identity.root.clone(),
5298 identity.session.clone(),
5299 route_id,
5300 bg_wake_pending,
5301 bg_wake_epoch,
5302 metrics,
5303 );
5304 return Ok(());
5305 }
5306
5307 let RouteRequest::ToolCall(call) = route_request else {
5308 unreachable!("background event subscription returned above")
5309 };
5310 let bare_name = call.name;
5311 let arguments = strip_agent_preview_arg_owned(call.arguments);
5312 let format_context = crate::subc_format::FormatContext::from_tool_call(
5313 &bare_name,
5314 &arguments,
5315 identity.project_root.as_path(),
5316 );
5317
5318 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5319 let bind_trust = identity.trust;
5320 let diagnostics_on_edit = live_roots
5321 .get(&identity.root)
5322 .map(|meta| meta.diagnostics_on_edit)
5323 .unwrap_or(false);
5324
5325 let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5326 && arguments
5327 .get("sandbox")
5328 .or_else(|| {
5329 arguments
5330 .get("params")
5331 .and_then(|params| params.get("sandbox"))
5332 })
5333 .and_then(Value::as_str)
5334 == Some("host");
5335 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5336 let response = Response::error(
5337 request_id.clone(),
5338 "sandbox_escalation_denied",
5339 "sandbox host escalation is unavailable to untrusted principals",
5340 );
5341 let text = crate::subc_format::format_response_with_context(
5342 &bare_name,
5343 &response,
5344 &format_context,
5345 );
5346 let result = ToolCallResult { text, response };
5347 let response_frame = build_tool_response_frame_with_limit(
5348 frame.header.ver,
5349 route_id,
5350 frame.header.corr,
5351 frame.header.flags,
5352 &result,
5353 bind_trust,
5354 tool_response_body_limit,
5355 )?;
5356 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5357 }
5358
5359 if matches!(bind_trust, BindTrust::Untrusted)
5360 && is_bash_family_tool(&bare_name)
5361 && (!matches!(bare_name.as_str(), "bash" | "powershell")
5362 || !identity.consumer_elicitation_capable)
5363 {
5364 let response = bash::bash_denied_untrusted_response(request_id.clone());
5365 let text = crate::subc_format::format_response_with_context(
5366 &bare_name,
5367 &response,
5368 &format_context,
5369 );
5370 let result = ToolCallResult { text, response };
5371 let response_frame = build_tool_response_frame_with_limit(
5372 frame.header.ver,
5373 route_id,
5374 frame.header.corr,
5375 frame.header.flags,
5376 &result,
5377 bind_trust,
5378 tool_response_body_limit,
5379 )?;
5380 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5381 }
5382
5383 if !is_subc_agent_core_tool(&bare_name)
5391 && !is_subc_native_plumbing_tool(&bare_name)
5392 && !allow_native_passthrough
5393 {
5394 log::warn!(
5395 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5396 bare_name,
5397 frame.header.channel
5398 );
5399 let response = Response::error(
5400 request_id.clone(),
5401 "unknown_tool",
5402 format!("tool {:?} is not in the AFT tool manifest", bare_name),
5403 );
5404 let text = crate::subc_format::format_response_with_context(
5405 &bare_name,
5406 &response,
5407 &format_context,
5408 );
5409 let result = ToolCallResult { text, response };
5410 let response_frame = build_tool_response_frame_with_limit(
5411 frame.header.ver,
5412 route_id,
5413 frame.header.corr,
5414 frame.header.flags,
5415 &result,
5416 bind_trust,
5417 tool_response_body_limit,
5418 )?;
5419 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5420 }
5421
5422 if matches!(bare_name.as_str(), "bash" | "powershell") {
5423 if matches!(bind_trust, BindTrust::Untrusted) {
5424 let plan = match bash::prepare_bash_elicitation_plan(
5425 &arguments,
5426 identity.project_root.as_path(),
5427 ) {
5428 Ok(plan) => plan,
5429 Err(error) => {
5430 let response = Response::error(request_id.clone(), error.code, error.message);
5431 let text = crate::subc_format::format_response_with_context(
5432 &bare_name,
5433 &response,
5434 &format_context,
5435 );
5436 let result = ToolCallResult { text, response };
5437 let response_frame = build_tool_response_frame_with_limit(
5438 frame.header.ver,
5439 route_id,
5440 frame.header.corr,
5441 frame.header.flags,
5442 &result,
5443 bind_trust,
5444 tool_response_body_limit,
5445 )?;
5446 return send_reliable_writer_frame(
5447 tx,
5448 metrics,
5449 response_frame,
5450 "tool response",
5451 )
5452 .await;
5453 }
5454 };
5455
5456 let reverse_corr =
5457 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5458 let ask_frame = build_bash_elicitation_request_frame(
5459 frame.header.ver,
5460 route_id,
5461 reverse_corr,
5462 frame.header.flags,
5463 &plan.command,
5464 &plan.asks,
5465 )?;
5466
5467 let meta = live_roots
5468 .entry(identity.root.clone())
5469 .or_insert_with(|| RootMeta::new(Instant::now()));
5470 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5471 meta.reactivate_bound();
5472
5473 let route_cancel =
5474 route_bash_cancels
5475 .entry(route_id)
5476 .or_insert_with(|| bash::RouteBashCancel {
5477 token: PersistentCancelSignal::new(),
5478 active_waits: 0,
5479 });
5480 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5481 let cancel = bash::BashWaitCancel {
5482 connection: connection_cancel.clone(),
5483 route: route_cancel.token.clone(),
5484 };
5485 pending_bash_asks.insert(
5486 ReverseCorrKey {
5487 route: route_id,
5488 corr: reverse_corr,
5489 },
5490 PendingBashAsk {
5491 route: route_id,
5492 tool_corr: frame.header.corr,
5493 tool_flags: frame.header.flags,
5494 tool_ver: frame.header.ver,
5495 root: identity.root.clone(),
5496 project_root: identity.project_root.clone(),
5497 session_id: identity.session.clone(),
5498 spawn_principal: identity.spawn_principal.clone(),
5499 edit_slot_survives: call.edit_slot_survives,
5500 request_id,
5501 arguments,
5502 format_context,
5503 cancel,
5504 grants: plan.grants,
5505 expires_at: Instant::now() + bash_elicitation_timeout(),
5506 },
5507 );
5508 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5509 .await;
5510 }
5511
5512 let meta = live_roots
5513 .entry(identity.root.clone())
5514 .or_insert_with(|| RootMeta::new(Instant::now()));
5515 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5516 meta.reactivate_bound();
5517
5518 let route_cancel =
5519 route_bash_cancels
5520 .entry(route_id)
5521 .or_insert_with(|| bash::RouteBashCancel {
5522 token: PersistentCancelSignal::new(),
5523 active_waits: 0,
5524 });
5525 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5526 let cancel = bash::BashWaitCancel {
5527 connection: connection_cancel.clone(),
5528 route: route_cancel.token.clone(),
5529 };
5530
5531 bash::submit_deferred_bash(
5532 executor,
5533 bash_deferred_tx,
5534 bash_poll_touch_tx,
5535 metrics,
5536 dispatch,
5537 identity.root.clone(),
5538 identity.project_root.clone(),
5539 identity.session.clone(),
5540 request_id,
5541 route_id,
5542 frame.header.corr,
5543 frame.header.flags,
5544 frame.header.ver,
5545 arguments,
5546 format_context,
5547 cancel,
5548 bind_trust,
5549 identity.spawn_principal.clone(),
5550 call.edit_slot_survives,
5551 None,
5552 );
5553 return Ok(());
5554 }
5555
5556 let lane = command_lane(&bare_name);
5557 let tool_call_context = ToolCallContext {
5558 project_root: identity.project_root.clone(),
5559 session_id: Some(identity.session.clone()),
5560 request_id: request_id.clone(),
5561 diagnostics_on_edit,
5562 preview: call.preview,
5563 edit_slot_survives: call.edit_slot_survives,
5564 report_registration_downgrade: true,
5565 };
5566
5567 let uses_deferred_response_seam = bare_name == "inspect"
5568 || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5569 if uses_deferred_response_seam {
5570 let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5571 let response = Response::error(
5572 &request_id,
5573 "actor_not_registered",
5574 "executor actor is not registered",
5575 );
5576 let text = crate::subc_format::format_response_with_context(
5577 &bare_name,
5578 &response,
5579 &format_context,
5580 );
5581 let result = ToolCallResult { text, response };
5582 let response_frame = build_tool_response_frame_with_limit(
5583 frame.header.ver,
5584 route_id,
5585 frame.header.corr,
5586 frame.header.flags,
5587 &result,
5588 bind_trust,
5589 tool_response_body_limit,
5590 )?;
5591 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5592 };
5593 let identity_for_run = identity.clone();
5594 let request_id_for_force = request_id.clone();
5595 let format_context_for_run = format_context.clone();
5596 let bare_name_for_run = bare_name.clone();
5597 let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5598 phase_trace.mark_executor_submitted();
5599 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5600 phase_trace.mark_job_admitted();
5601 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5602 let run = || match prepare_tool_call(
5603 &bare_name_for_run,
5604 arguments,
5605 &format_context_for_run,
5606 &tool_call_context,
5607 ctx,
5608 Some(&mut phase_trace),
5609 ) {
5610 Err(result) => {
5611 let response = result.response;
5612 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5613 text: result.text,
5614 phase_trace,
5615 });
5616 response
5617 }
5618 Ok(prepared) => {
5619 let outcome = if bare_name_for_run == "inspect" {
5620 crate::commands::inspect::handle_inspect_deferred_with_restriction(
5621 &prepared.request,
5622 Arc::clone(&deferred_ctx),
5623 matches!(bind_trust, BindTrust::Untrusted),
5624 )
5625 } else {
5626 crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5627 &prepared.request,
5628 Arc::clone(&deferred_ctx),
5629 matches!(bind_trust, BindTrust::Untrusted),
5630 )
5631 };
5632 match outcome {
5633 DispatchOutcome::Deferred(pending) => {
5634 let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5635 pending,
5636 surface_downgraded: prepared.surface_downgraded,
5637 phase_trace,
5638 });
5639 Response::success(
5640 request_id_for_force.clone(),
5641 json!({ "response_deferred": true }),
5642 )
5643 }
5644 DispatchOutcome::Immediate(response) => {
5645 phase_trace.mark_execute_done();
5646 let finalizer = |response: &mut Response| {
5647 crate::response_finalize::finalize_response_with_bg_completions(
5648 response,
5649 ctx,
5650 &identity_for_run.session,
5651 &bare_name_for_run,
5652 bind_trust.allows_bash_observation(),
5653 );
5654 };
5655 let result = finish_tool_call_response(
5656 &bare_name_for_run,
5657 &format_context_for_run,
5658 response,
5659 prepared.surface_downgraded,
5660 Some(&finalizer),
5661 Some(&mut phase_trace),
5662 );
5663 let response = result.response;
5664 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5665 text: result.text,
5666 phase_trace,
5667 });
5668 response
5669 }
5670 }
5671 }
5672 };
5673 if matches!(bind_trust, BindTrust::Untrusted) {
5674 ctx.with_force_restrict(&request_id_for_force, run)
5675 } else {
5676 run()
5677 }
5678 })
5679 });
5680 let deferred_setup_guard =
5681 PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5682 let rx = submit_active_tool_call(
5683 executor.as_ref(),
5684 active_tool_calls,
5685 route_id,
5686 frame.header.corr,
5687 identity.root.clone(),
5688 lane,
5689 request_id.clone(),
5690 RouteDetachPolicy::CancelOnDetach,
5691 job,
5692 );
5693
5694 let completion_tx = tx.clone();
5695 let completion_shutdown = Arc::clone(shutdown);
5696 let completion_metrics = Arc::clone(metrics);
5697 let active_tool_calls = Arc::clone(active_tool_calls);
5698 let deferred_response_tx = deferred_response_tx.clone();
5699 let route = route_id;
5700 let corr = frame.header.corr;
5701 let flags = frame.header.flags;
5702 let ver = frame.header.ver;
5703 let root = identity.root.clone();
5704 let session_id = identity.session.clone();
5705 tokio::spawn(async move {
5706 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5707 let _deferred_setup = deferred_setup_guard;
5708 let response = await_executor_response(rx, request_id.clone()).await;
5709 match setup_rx.await {
5710 Ok(DeferredSetupOutcome::Deferred {
5711 pending,
5712 surface_downgraded,
5713 phase_trace,
5714 }) => {
5715 let pending = PendingSubcResponse {
5716 route,
5717 corr,
5718 flags,
5719 ver,
5720 root,
5721 session_id,
5722 bare_name,
5723 format_context,
5724 bind_trust,
5725 pending,
5726 surface_downgraded,
5727 phase_trace,
5728 };
5729 if let Err(error) = deferred_response_tx.send(pending) {
5730 if let Some(cancellation) = &error.0.pending.cancellation {
5731 cancellation.request_cancel();
5732 }
5733 finish_active_tool_call(&active_tool_calls, route, corr);
5734 }
5735 }
5736 Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5737 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5738 return;
5739 }
5740 let result = ToolCallResult { text, response };
5741 let fatal = note_fatal_panic_response(&result.response);
5742 match build_tool_response_frame_with_limit(
5743 ver,
5744 route,
5745 corr,
5746 flags,
5747 &result,
5748 bind_trust,
5749 tool_response_body_limit,
5750 ) {
5751 Ok(response_frame) => {
5752 let trace = ToolResponseWriteTrace::new(
5753 phase_trace,
5754 bare_name.clone(),
5755 identity.project_root.clone(),
5756 identity.session.clone(),
5757 route.channel,
5758 corr,
5759 );
5760 if let Err(error) = send_traced_tool_response_frame(
5761 &completion_tx,
5762 &completion_metrics,
5763 response_frame,
5764 trace,
5765 )
5766 .await
5767 {
5768 log::warn!(
5769 "subc attach: failed to queue deferred-seam setup response: {error}"
5770 );
5771 }
5772 }
5773 Err(error) => {
5774 log::error!(
5775 "subc attach: failed to build deferred-seam setup response: {error}"
5776 );
5777 }
5778 }
5779 if fatal {
5780 signal_fatal_teardown(
5781 &completion_tx,
5782 Some(route),
5783 ver,
5784 corr,
5785 &completion_shutdown,
5786 &completion_metrics,
5787 )
5788 .await;
5789 }
5790 }
5791 Err(_) => {
5792 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5793 return;
5794 }
5795 let text = crate::subc_format::format_response_with_context(
5796 &bare_name,
5797 &response,
5798 &format_context,
5799 );
5800 let result = ToolCallResult { text, response };
5801 if let Ok(response_frame) = build_tool_response_frame_with_limit(
5802 ver,
5803 route,
5804 corr,
5805 flags,
5806 &result,
5807 bind_trust,
5808 tool_response_body_limit,
5809 ) {
5810 let _ = send_reliable_writer_frame(
5811 &completion_tx,
5812 &completion_metrics,
5813 response_frame,
5814 "deferred setup failure",
5815 )
5816 .await;
5817 }
5818 }
5819 }
5820 });
5821 return Ok(());
5822 }
5823
5824 let bare_name_for_frame = bare_name.clone();
5825 let identity_for_run = identity.clone();
5826 let completion_session = identity.session.clone();
5827 let completion_root = identity.project_root.clone();
5828 let request_id_for_force = request_id.clone();
5829 let format_context_for_frame = format_context.clone();
5830 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5831 phase_trace.mark_executor_submitted();
5832 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5833 phase_trace.mark_job_admitted();
5834 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5835 let run = || {
5836 let finalizer = |response: &mut Response| {
5837 crate::response_finalize::finalize_response_with_bg_completions(
5838 response,
5839 ctx,
5840 &identity_for_run.session,
5841 &bare_name,
5842 bind_trust.allows_bash_observation(),
5843 );
5844 };
5845 match run_tool_call(
5846 &bare_name,
5847 arguments,
5848 &format_context,
5849 &tool_call_context,
5850 ctx,
5851 &dispatch,
5852 Some(&finalizer),
5853 Some(&mut phase_trace),
5854 ) {
5855 ToolCallOutcome::Unary(result) => {
5856 let response = result.response;
5857 let _ = tool_call_tx.send(ToolCallCompletion {
5858 text: result.text,
5859 phase_trace,
5860 });
5861 response
5862 }
5863 }
5864 };
5865 if matches!(bind_trust, BindTrust::Untrusted) {
5866 ctx.with_force_restrict(&request_id_for_force, run)
5867 } else {
5868 run()
5869 }
5870 })
5871 });
5872 let rx = submit_active_tool_call(
5873 executor.as_ref(),
5874 active_tool_calls,
5875 route_id,
5876 frame.header.corr,
5877 identity.root.clone(),
5878 lane,
5879 request_id.clone(),
5880 RouteDetachPolicy::RetainForReplay,
5881 job,
5882 );
5883 let completion_tx = tx.clone();
5884 let completion_shutdown = Arc::clone(shutdown);
5885 let route = route_id;
5886 let corr = frame.header.corr;
5887 let flags = frame.header.flags;
5888 let ver = frame.header.ver;
5889 let completion_metrics = Arc::clone(metrics);
5890 let active_tool_calls = Arc::clone(active_tool_calls);
5891 tokio::spawn(async move {
5892 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5893 let response = await_executor_response(rx, request_id.clone()).await;
5894 let (text, phase_trace) = match tool_call_rx.await {
5895 Ok(completion) => (completion.text, Some(completion.phase_trace)),
5896 Err(_) => (
5897 crate::subc_format::format_response_with_context(
5898 &bare_name_for_frame,
5899 &response,
5900 &format_context_for_frame,
5901 ),
5902 None,
5903 ),
5904 };
5905 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5906 return;
5907 }
5908 let result = ToolCallResult { text, response };
5909 let fatal = note_fatal_panic_response(&result.response);
5910 match build_tool_response_frame_with_limit(
5911 ver,
5912 route,
5913 corr,
5914 flags,
5915 &result,
5916 bind_trust,
5917 tool_response_body_limit,
5918 ) {
5919 Ok(response_frame) => {
5920 let send_result = if let Some(phase_trace) = phase_trace {
5921 let trace = ToolResponseWriteTrace::new(
5922 phase_trace,
5923 bare_name_for_frame,
5924 completion_root,
5925 completion_session,
5926 route.channel,
5927 corr,
5928 );
5929 send_traced_tool_response_frame(
5930 &completion_tx,
5931 &completion_metrics,
5932 response_frame,
5933 trace,
5934 )
5935 .await
5936 } else {
5937 send_reliable_writer_frame(
5938 &completion_tx,
5939 &completion_metrics,
5940 response_frame,
5941 "tool response",
5942 )
5943 .await
5944 };
5945 if let Err(error) = send_result {
5946 log::warn!("subc attach: failed to queue tool response frame: {error}");
5947 }
5948 }
5949 Err(error) => {
5950 log::error!("subc attach: failed to build tool response frame: {error}");
5951 }
5952 }
5953 if fatal {
5954 signal_fatal_teardown(
5955 &completion_tx,
5956 Some(route),
5957 ver,
5958 corr,
5959 &completion_shutdown,
5960 &completion_metrics,
5961 )
5962 .await;
5963 }
5964 });
5965 Ok(())
5966}
5967
5968fn submit_maintenance_job(
5969 executor: &Arc<Executor>,
5970 root_id: ProjectRootId,
5971 kind: MaintenanceDrainKind,
5972 bg_sessions_to_check: Vec<(String, u64)>,
5973 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5974 metrics: &Arc<DispatchPathMetrics>,
5975) {
5976 let request_id = format!(
5977 "subc-maintenance-drain-{}-{}",
5978 kind.label(),
5979 root_id.as_path().to_string_lossy()
5980 );
5981 let response_id = request_id.clone();
5982 let completion_root_id = root_id.clone();
5983 let maintenance_generation = executor
5984 .actor_context(&root_id)
5985 .map(|ctx| ctx.configure_generation())
5986 .unwrap_or(0);
5987 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5988 let lane = Lane::MaintenanceCommit;
5992 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5993 let outcome = match kind {
5994 MaintenanceDrainKind::Watcher => {
5995 let drained = runtime_drain::drain_watcher_events_bounded(
5996 ctx,
5997 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5998 );
5999 MaintenanceJobOutcome {
6000 empty_bg_sessions: Vec::new(),
6001 unacked_bg_keys: None,
6002 requeue_kind: drained.has_more.then_some(kind),
6003 }
6004 }
6005 MaintenanceDrainKind::Lsp => {
6006 let drained = runtime_drain::drain_lsp_events_bounded(
6007 ctx,
6008 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6009 );
6010 MaintenanceJobOutcome {
6011 empty_bg_sessions: Vec::new(),
6012 unacked_bg_keys: None,
6013 requeue_kind: drained.has_more.then_some(kind),
6014 }
6015 }
6016 MaintenanceDrainKind::ConfigureTail => {
6017 runtime_drain::drain_deferred_configure_maintenance(ctx);
6018 runtime_drain::drain_configure_warning_events(ctx);
6019 MaintenanceJobOutcome::default()
6020 }
6021 MaintenanceDrainKind::CompletionDrains => {
6022 runtime_drain::drain_search_index_events(ctx);
6023 runtime_drain::drain_callgraph_store_events(ctx);
6024 runtime_drain::drain_semantic_index_events(ctx);
6025 runtime_drain::drain_semantic_refresh_events(ctx);
6026 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6027 let empty_bg_sessions = bg_sessions_to_check
6028 .into_iter()
6029 .filter(|(session, _)| {
6030 !ctx.bash_background().has_unacked_wakes_for_session(session)
6031 })
6032 .collect();
6033 MaintenanceJobOutcome {
6034 empty_bg_sessions,
6035 unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6036 requeue_kind: None,
6037 }
6038 }
6039 };
6040 let requeued = outcome.requeue_kind.is_some();
6041 let _ = outcome_tx.send(outcome);
6042 Response::success(
6043 response_id,
6044 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6045 )
6046 });
6047 let rx = match kind {
6048 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6049 root_id,
6050 lane,
6051 request_id.clone(),
6052 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6053 job,
6054 ),
6055 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6056 root_id,
6057 lane,
6058 request_id.clone(),
6059 crate::executor::MaintenanceCoalesceKey::LspDrain,
6060 job,
6061 ),
6062 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6063 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6064 }
6065 };
6066 let completion_tx = completion_tx.clone();
6067 let completion_metrics = Arc::clone(metrics);
6068 tokio::spawn(async move {
6069 let _response_task = ResponseTaskGuard::new(&completion_metrics);
6070 let response = await_executor_response(rx, request_id).await;
6071 let outcome = outcome_rx.await.unwrap_or_default();
6072 let _ = send_counted_channel(
6073 &completion_tx,
6074 &completion_metrics.maintenance_queued,
6075 MaintenanceCompletion {
6076 root_id: completion_root_id,
6077 kind,
6078 response,
6079 empty_bg_sessions: outcome.empty_bg_sessions,
6080 unacked_bg_keys: outcome.unacked_bg_keys,
6081 requeue_kind: outcome.requeue_kind,
6082 },
6083 )
6084 .await;
6085 });
6086}
6087
6088async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6089 rx.await
6090 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6091}
6092
6093async fn deliver_resolved_subc_response(
6094 tx: &WriterSender,
6095 mut resolved: ResolvedSubcResponse,
6096 routes: &HashMap<RouteChannel, RouteIdentity>,
6097 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6098 executor: &Executor,
6099 active_tool_calls: &ActiveToolCalls,
6100 shutdown: &Arc<Notify>,
6101 metrics: &DispatchPathMetrics,
6102 tool_response_body_limit: usize,
6103) -> Result<(), SubcError> {
6104 let entry = &mut resolved.entry;
6105 finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6106 if let Some(meta) = live_roots.get_mut(&entry.root) {
6107 meta.note_activity();
6108 }
6109
6110 let Some(identity) = routes.get(&entry.route) else {
6111 log::debug!(
6112 "subc attach: dropping deferred {} response {} for unbound route {}",
6113 entry.bare_name,
6114 entry.pending.request_id,
6115 entry.route
6116 );
6117 return Ok(());
6118 };
6119 let Some(ctx) = executor.actor_context(&entry.root) else {
6120 return Ok(());
6121 };
6122 entry.phase_trace.mark_execute_done();
6123 let finalizer = |response: &mut Response| {
6124 crate::response_finalize::finalize_response_with_bg_completions(
6125 response,
6126 &ctx,
6127 &entry.session_id,
6128 &entry.bare_name,
6129 entry.bind_trust.allows_bash_observation(),
6130 );
6131 };
6132 let result = finish_tool_call_response(
6133 &entry.bare_name,
6134 &entry.format_context,
6135 resolved.response,
6136 entry.surface_downgraded,
6137 Some(&finalizer),
6138 Some(&mut entry.phase_trace),
6139 );
6140 let fatal = note_fatal_panic_response(&result.response);
6141 let response_frame = build_tool_response_frame_with_limit(
6142 entry.ver,
6143 entry.route,
6144 entry.corr,
6145 entry.flags,
6146 &result,
6147 identity.trust,
6148 tool_response_body_limit,
6149 )?;
6150 let trace = ToolResponseWriteTrace::new(
6151 std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6152 entry.bare_name.clone(),
6153 identity.project_root.clone(),
6154 entry.session_id.clone(),
6155 entry.route.channel,
6156 entry.corr,
6157 );
6158 send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6159 if fatal {
6160 signal_fatal_teardown(
6161 tx,
6162 Some(entry.route),
6163 entry.ver,
6164 entry.corr,
6165 shutdown,
6166 metrics,
6167 )
6168 .await;
6169 }
6170 Ok(())
6171}
6172
6173async fn signal_fatal_teardown(
6174 tx: &WriterSender,
6175 route: Option<RouteChannel>,
6176 ver: u8,
6177 corr: u64,
6178 shutdown: &Arc<Notify>,
6179 metrics: &DispatchPathMetrics,
6180) {
6181 if let Some(route) = route {
6182 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6183 if let Err(error) = send_frame(tx, metrics, frame).await {
6184 log::warn!(
6185 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6186 );
6187 }
6188 }
6189 }
6190 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6191 if let Err(error) = send_frame(tx, metrics, frame).await {
6192 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6193 }
6194 }
6195 shutdown.notify_one();
6196}
6197#[derive(Debug, Deserialize)]
6198#[serde(untagged)]
6199enum RouteRequest {
6200 BgEvents(BgEventsRequest),
6201 ToolCall(ToolCallRequest),
6202}
6203
6204#[derive(Debug, Deserialize)]
6205struct BgEventsRequest {
6206 op: BgEventsOp,
6207}
6208
6209#[derive(Debug, Deserialize)]
6210#[serde(rename_all = "snake_case")]
6211enum BgEventsOp {
6212 BgEvents,
6213}
6214
6215#[derive(Debug, Deserialize)]
6216struct ToolCallRequest {
6217 name: String,
6218 #[serde(default)]
6219 arguments: Value,
6220 #[serde(default)]
6222 edit_slot_survives: Option<bool>,
6223 #[serde(default)]
6228 preview: bool,
6229}
6230
6231#[cfg(test)]
6232pub(crate) mod test_support {
6233 use super::*;
6234 use crate::bash_background::BgTaskStatus;
6235 use crate::protocol::{
6236 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6237 ProgressFrame, StatusChangedFrame,
6238 };
6239 use serde_json::json;
6240
6241 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6242 let dir = tempfile::Builder::new()
6243 .prefix(name)
6244 .tempdir()
6245 .expect("temp root");
6246 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6247 (dir, root)
6248 }
6249
6250 pub(super) fn test_ctx() -> Arc<AppContext> {
6251 Arc::new(AppContext::new(
6252 Box::new(crate::parser::TreeSitterProvider::new()),
6253 crate::config::Config::default(),
6254 ))
6255 }
6256
6257 fn inspect_context(root: &Path) -> Arc<AppContext> {
6258 inspect_context_with_timeout(root, None)
6259 }
6260
6261 fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6262 let mut config = crate::config::Config::default();
6263 config.project_root = Some(root.to_path_buf());
6264 if let Some(timeout_ms) = timeout_ms {
6265 config.inspect.diagnostics_timeout_ms = timeout_ms;
6266 }
6267 let ctx = Arc::new(AppContext::new(
6268 Box::new(crate::parser::TreeSitterProvider::new()),
6269 config,
6270 ));
6271 ctx.set_harness(crate::harness::Harness::Opencode);
6272 ctx
6273 }
6274
6275 fn inspect_request(id: &str) -> RawRequest {
6276 serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6277 }
6278
6279 fn submit_deferred_inspect_setup(
6280 executor: &Arc<Executor>,
6281 root: &ProjectRootId,
6282 ctx: &Arc<AppContext>,
6283 request_id: &str,
6284 ) -> (PendingResponse, JobCancellation) {
6285 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6286 let request = inspect_request(request_id);
6287 let inspect_ctx = Arc::clone(ctx);
6288 let (_rx, cancellation) = executor.submit_cancellable_async(
6289 root.clone(),
6290 Lane::SerialLspStatus,
6291 request_id.to_string(),
6292 Box::new(move |_| {
6293 let DispatchOutcome::Deferred(pending) =
6294 crate::commands::inspect::handle_inspect_deferred_with_restriction(
6295 &request,
6296 inspect_ctx,
6297 true,
6298 )
6299 else {
6300 panic!("inspect setup must defer")
6301 };
6302 pending_tx.send(pending).expect("send pending inspect");
6303 Response::success("inspect-setup", json!({}))
6304 }),
6305 );
6306 let pending = pending_rx
6307 .recv_timeout(Duration::from_secs(1))
6308 .expect("inspect setup leaves the executor lane");
6309 let deadline = Instant::now() + Duration::from_secs(1);
6310 while !executor.actor_is_idle(root) {
6311 assert!(
6312 Instant::now() < deadline,
6313 "inspect setup kept lane counters live"
6314 );
6315 std::thread::sleep(Duration::from_millis(5));
6316 }
6317 (pending, cancellation)
6318 }
6319
6320 fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6321 let deadline = Instant::now() + Duration::from_secs(60);
6322 loop {
6323 if let Some(response) = (pending.poll)(ctx) {
6324 return response;
6325 }
6326 assert!(Instant::now() < deadline, "inspect terminal timed out");
6327 std::thread::sleep(Duration::from_millis(5));
6328 }
6329 }
6330
6331 fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6332 let source_dir = root.join("src");
6333 std::fs::create_dir_all(&source_dir).expect("create source dir");
6334 std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6335 .expect("write Cargo manifest");
6336 let source = source_dir.join("main.rs");
6337 std::fs::write(&source, "fn main() {}\n").expect("write source");
6338 let binary = root.join("cold-navigation-server");
6339 std::fs::write(&binary, b"fixture").expect("write server placeholder");
6340
6341 let ctx = inspect_context(root);
6342 ctx.lsp()
6343 .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6344 (ctx, source)
6345 }
6346
6347 fn navigation_request(id: &str, source: &Path) -> RawRequest {
6348 serde_json::from_value(json!({
6349 "id": id,
6350 "command": "lsp_hover",
6351 "file": source.display().to_string(),
6352 "line": 1,
6353 "character": 1,
6354 }))
6355 .expect("navigation request")
6356 }
6357
6358 fn submit_deferred_navigation_setup(
6359 executor: &Arc<Executor>,
6360 root: &ProjectRootId,
6361 ctx: &Arc<AppContext>,
6362 source: &Path,
6363 request_id: &str,
6364 ) -> (PendingResponse, JobCancellation) {
6365 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6366 let request = navigation_request(request_id, source);
6367 let navigation_ctx = Arc::clone(ctx);
6368 let (_rx, cancellation) = executor.submit_cancellable_async(
6369 root.clone(),
6370 Lane::SerialLspStatus,
6371 request_id.to_string(),
6372 Box::new(move |_| {
6373 let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6374 handle_lsp_navigation_deferred_with_restriction(
6375 &request,
6376 navigation_ctx,
6377 true,
6378 )
6379 else {
6380 panic!("cold navigation setup must defer")
6381 };
6382 pending_tx.send(pending).expect("send pending navigation");
6383 Response::success("navigation-setup", json!({}))
6384 }),
6385 );
6386 let pending = pending_rx
6387 .recv_timeout(Duration::from_secs(1))
6388 .expect("navigation setup leaves the executor lane");
6389 let deadline = Instant::now() + Duration::from_secs(1);
6390 while !executor.actor_is_idle(root) {
6391 assert!(
6392 Instant::now() < deadline,
6393 "navigation setup kept lane counters live"
6394 );
6395 std::thread::sleep(Duration::from_millis(5));
6396 }
6397 (pending, cancellation)
6398 }
6399
6400 fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6401 let deadline = Instant::now() + Duration::from_secs(1);
6402 loop {
6403 if let Some(response) = (pending.poll)(ctx) {
6404 return response;
6405 }
6406 assert!(Instant::now() < deadline, "navigation terminal timed out");
6407 std::thread::sleep(Duration::from_millis(5));
6408 }
6409 }
6410
6411 #[test]
6412 fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6413 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6414 let executor = Arc::new(Executor::new());
6415 let (dir, root) = test_root("deferred-inspect-storm");
6416 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6417 let ctx = inspect_context(dir.path());
6418 executor.register_actor(root.clone(), Arc::clone(&ctx));
6419 let (started_rx, release_tx) =
6420 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6421 let (mut pending, _cancellation) =
6422 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6423 started_rx
6424 .recv_timeout(Duration::from_secs(1))
6425 .expect("deferred inspect body starts");
6426
6427 for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6428 let response = executor.submit(
6429 root.clone(),
6430 Lane::Mutating,
6431 request_id.to_string(),
6432 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6433 );
6434 assert!(
6435 response
6436 .recv_timeout(Duration::from_secs(1))
6437 .expect("writer admits while inspect remains deferred")
6438 .success
6439 );
6440 }
6441 assert_eq!(
6442 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6443 1,
6444 "writer admissions must not finish the detached inspect"
6445 );
6446
6447 release_tx.send(()).expect("release inspect body");
6448 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6449 assert!(
6450 terminal.data.get("inspect_terminal").is_some(),
6451 "inspect must still produce its terminal: {:?}",
6452 terminal.data
6453 );
6454 }
6455
6456 #[test]
6457 fn deferred_inspect_honors_the_shared_request_deadline() {
6458 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6459 let executor = Arc::new(Executor::new());
6460 let (dir, root) = test_root("deferred-inspect-deadline");
6461 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6462 let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6463 executor.register_actor(root.clone(), Arc::clone(&ctx));
6464 let (started_rx, _release_tx) =
6465 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6466 let started = Instant::now();
6467 let (mut pending, _cancellation) =
6468 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6469 started_rx
6470 .recv_timeout(Duration::from_secs(1))
6471 .expect("deferred inspect body starts");
6472
6473 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6474 assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6475 assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6476 assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6477 assert!(
6478 started.elapsed() < Duration::from_secs(8),
6479 "deferred inspect missed its terminal reserve: {:?}",
6480 started.elapsed()
6481 );
6482 }
6483
6484 #[test]
6485 fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6486 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6487 let executor = Arc::new(Executor::new());
6488 let (dir, root) = test_root("deferred-cold-navigation");
6489 let (ctx, source) = cold_navigation_context(dir.path());
6490 executor.register_actor(root.clone(), Arc::clone(&ctx));
6491 let (started_rx, _release_tx) =
6492 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6493 let (mut pending, cancellation) = submit_deferred_navigation_setup(
6494 &executor,
6495 &root,
6496 &ctx,
6497 &source,
6498 "subc-cold-navigation",
6499 );
6500 started_rx
6501 .recv_timeout(Duration::from_secs(1))
6502 .expect("detached navigation reaches its cold-start gate");
6503
6504 for (lane, request_id) in [
6505 (Lane::PureRead, "subc-navigation-read"),
6506 (Lane::Mutating, "subc-navigation-write"),
6507 ] {
6508 let response = executor.submit(
6509 root.clone(),
6510 lane,
6511 request_id.to_string(),
6512 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6513 );
6514 assert!(
6515 response
6516 .recv_timeout(Duration::from_secs(1))
6517 .expect("unrelated work admits while navigation remains deferred")
6518 .success
6519 );
6520 }
6521 assert_eq!(
6522 crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6523 1,
6524 "lane admissions must not finish the detached navigation"
6525 );
6526
6527 cancellation.request_cancel();
6528 let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6529 assert!(!terminal.success);
6530 let deadline = Instant::now() + Duration::from_secs(1);
6531 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6532 assert!(
6533 Instant::now() < deadline,
6534 "cancelled navigation worker did not settle"
6535 );
6536 std::thread::sleep(Duration::from_millis(5));
6537 }
6538 assert!(executor.actor_is_idle(&root));
6539 }
6540
6541 #[test]
6542 fn cancelling_pending_navigation_removes_entry_without_reply() {
6543 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6544 let executor = Arc::new(Executor::new());
6545 let (dir, root) = test_root("cancelled-pending-navigation");
6546 let (ctx, source) = cold_navigation_context(dir.path());
6547 executor.register_actor(root.clone(), Arc::clone(&ctx));
6548 let (started_rx, _release_tx) =
6549 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6550 let (pending, _cancellation) = submit_deferred_navigation_setup(
6551 &executor,
6552 &root,
6553 &ctx,
6554 &source,
6555 "subc-cancel-navigation",
6556 );
6557 started_rx
6558 .recv_timeout(Duration::from_secs(1))
6559 .expect("detached navigation reaches its cancellation gate");
6560
6561 let route = RouteChannel {
6562 channel: 17,
6563 epoch: 1,
6564 };
6565 let mut registry = PendingSubcResponses::default();
6566 registry.register(PendingSubcResponse {
6567 route,
6568 corr: 71,
6569 flags: Flags::new(false, Priority::Passive, false),
6570 ver: PROTOCOL_VERSION,
6571 root: root.clone(),
6572 session_id: "navigation-cancel-session".to_string(),
6573 bare_name: "lsp_hover".to_string(),
6574 format_context: crate::subc_format::FormatContext::from_tool_call(
6575 "lsp_hover",
6576 &json!({}),
6577 dir.path(),
6578 ),
6579 bind_trust: BindTrust::FirstParty,
6580 pending,
6581 surface_downgraded: false,
6582 phase_trace: PhaseTrace::new(Instant::now()),
6583 });
6584
6585 assert!(registry.cancel_request(route, 71));
6586 assert!(registry.is_empty());
6587 let deadline = Instant::now() + Duration::from_secs(1);
6588 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6589 assert!(
6590 Instant::now() < deadline,
6591 "pending cancellation did not settle the detached worker"
6592 );
6593 std::thread::sleep(Duration::from_millis(5));
6594 }
6595 assert!(
6596 registry.poll_ready(executor.as_ref()).is_empty(),
6597 "a cancelled navigation must not leak a reply"
6598 );
6599 }
6600
6601 #[test]
6602 fn same_root_deferred_inspects_are_single_flight() {
6603 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6604 let executor = Arc::new(Executor::new());
6605 let (dir, root) = test_root("single-flight-deferred-inspect");
6606 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6607 let ctx = inspect_context(dir.path());
6608 executor.register_actor(root.clone(), Arc::clone(&ctx));
6609 let (started_rx, release_tx) =
6610 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6611 let (mut first, _first_cancellation) =
6612 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6613 started_rx
6614 .recv_timeout(Duration::from_secs(1))
6615 .expect("first inspect owns the root flight");
6616 let (mut second, second_cancellation) =
6617 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6618
6619 assert_eq!(
6620 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6621 1,
6622 "only one detached body may run for a root"
6623 );
6624 assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6625 second_cancellation.request_cancel();
6626 let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6627 assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6628 assert_eq!(
6629 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6630 1,
6631 "cancelling the queued request must not release the active flight"
6632 );
6633
6634 release_tx.send(()).expect("release first inspect");
6635 let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6636 assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6637 assert_eq!(
6638 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6639 0
6640 );
6641 }
6642
6643 #[test]
6644 fn route_abandonment_cancels_detached_inspect_thread() {
6645 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6646 let executor = Arc::new(Executor::new());
6647 let (dir, root) = test_root("cancelled-deferred-inspect");
6648 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6649 let ctx = inspect_context(dir.path());
6650 executor.register_actor(root.clone(), Arc::clone(&ctx));
6651 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6652 let route = RouteChannel {
6653 channel: 7,
6654 epoch: 1,
6655 };
6656 let (started_rx, _release_tx) =
6657 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6658 let (mut pending, cancellation) =
6659 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6660 active.lock().expect("active tool call map").insert(
6661 (route, 41),
6662 ActiveToolCall {
6663 root_id: root.clone(),
6664 cancellation,
6665 detach_policy: RouteDetachPolicy::CancelOnDetach,
6666 },
6667 );
6668 started_rx
6669 .recv_timeout(Duration::from_secs(1))
6670 .expect("detached inspect reaches cancellation gate");
6671 assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6672
6673 assert!(cancel_active_tool_call(
6674 &active,
6675 executor.as_ref(),
6676 route,
6677 41,
6678 "test route abandonment"
6679 ));
6680 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6681 assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6682 assert_eq!(
6683 crate::commands::inspect::deferred_inspect_root_count_for_test(),
6684 0
6685 );
6686 assert!(executor.actor_is_idle(&root));
6687 assert!(active.lock().expect("active tool call map").is_empty());
6688 let restriction_deadline = Instant::now() + Duration::from_secs(1);
6689 while ctx.request_force_restrict("subc-inspect-abandoned") {
6690 assert!(
6691 Instant::now() < restriction_deadline,
6692 "detached force-restrict guard leaked"
6693 );
6694 std::thread::sleep(Duration::from_millis(5));
6695 }
6696 }
6697
6698 #[test]
6699 fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6700 let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6705 pool_size: 3,
6706 read_cap: 2,
6707 actor_cap: 2,
6708 heavy_permits: 1,
6709 drr_quantum: 1,
6710 }));
6711 let (_dir, root) = test_root("cancelled-interactive-search");
6712 executor.register_actor(root.clone(), test_ctx());
6713 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6714 let route = RouteChannel {
6715 channel: 9,
6716 epoch: 1,
6717 };
6718
6719 let disabled_iterations = Arc::new(AtomicUsize::new(0));
6720 let disabled_probe = Arc::clone(&disabled_iterations);
6721 let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6722 let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6723 root.clone(),
6724 Lane::PureRead,
6725 "untracked-search".to_string(),
6726 Box::new(move |_| {
6727 disabled_started_tx
6728 .send(())
6729 .expect("signal untracked search");
6730 let deadline = Instant::now() + Duration::from_secs(5);
6731 while !crate::commands::semantic_search::search_cancellation_requested() {
6732 if Instant::now() >= deadline {
6733 return Response::error(
6734 "untracked-search",
6735 "test_timeout",
6736 "untracked search did not receive cancellation",
6737 );
6738 }
6739 disabled_probe.fetch_add(1, Ordering::Relaxed);
6740 std::thread::yield_now();
6741 }
6742 Response::error(
6743 "untracked-search",
6744 "request_cancelled",
6745 "cancelled at search checkpoint",
6746 )
6747 }),
6748 );
6749 disabled_started_rx
6750 .recv_timeout(Duration::from_secs(1))
6751 .expect("untracked search starts");
6752 assert_eq!(
6753 apply_route_work_disposition(
6754 &active,
6755 executor.as_ref(),
6756 route,
6757 RouteWorkDisposition::Abandon,
6758 "disabled cancellation wiring",
6759 ),
6760 0
6761 );
6762 let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6763 std::thread::sleep(Duration::from_millis(10));
6764 assert!(
6765 disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6766 "without route registration the search keeps computing"
6767 );
6768 disabled_cancellation.request_cancel();
6769 let disabled_response = disabled_rx
6770 .blocking_recv()
6771 .expect("untracked search settles after explicit cleanup");
6772 assert_eq!(disabled_response.data["code"], "request_cancelled");
6773
6774 let tracked_iterations = Arc::new(AtomicUsize::new(0));
6775 let tracked_probe = Arc::clone(&tracked_iterations);
6776 let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6777 let tracked_rx = submit_active_tool_call(
6778 executor.as_ref(),
6779 &active,
6780 route,
6781 42,
6782 root.clone(),
6783 Lane::PureRead,
6784 "tracked-search".to_string(),
6785 RouteDetachPolicy::RetainForReplay,
6786 Box::new(move |_| {
6787 tracked_started_tx.send(()).expect("signal tracked search");
6788 let deadline = Instant::now() + Duration::from_secs(5);
6789 while !crate::commands::semantic_search::search_cancellation_requested() {
6790 if Instant::now() >= deadline {
6791 return Response::error(
6792 "tracked-search",
6793 "test_timeout",
6794 "tracked search did not receive cancellation",
6795 );
6796 }
6797 tracked_probe.fetch_add(1, Ordering::Relaxed);
6798 std::thread::yield_now();
6799 }
6800 Response::error(
6801 "tracked-search",
6802 "request_cancelled",
6803 "cancelled at search checkpoint",
6804 )
6805 }),
6806 );
6807 tracked_started_rx
6808 .recv_timeout(Duration::from_secs(1))
6809 .expect("tracked search starts");
6810
6811 let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6812 let terminal_rx = submit_active_tool_call(
6813 executor.as_ref(),
6814 &active,
6815 route,
6816 43,
6817 root.clone(),
6818 Lane::PureRead,
6819 "teardown-terminal".to_string(),
6820 RouteDetachPolicy::CancelOnDetach,
6821 Box::new(move |_| {
6822 terminal_started_tx
6823 .send(())
6824 .expect("signal teardown-terminal call");
6825 let deadline = Instant::now() + Duration::from_secs(5);
6826 while !crate::executor::current_job_cancelled() {
6827 if Instant::now() >= deadline {
6828 return Response::error(
6829 "teardown-terminal",
6830 "test_timeout",
6831 "terminal call did not receive cancellation",
6832 );
6833 }
6834 std::thread::yield_now();
6835 }
6836 Response::error(
6837 "teardown-terminal",
6838 "request_cancelled",
6839 "cancelled for terminal-emitting teardown",
6840 )
6841 }),
6842 );
6843 terminal_started_rx
6844 .recv_timeout(Duration::from_secs(1))
6845 .expect("teardown-terminal call starts");
6846 assert_eq!(
6847 apply_route_work_disposition(
6848 &active,
6849 executor.as_ref(),
6850 route,
6851 RouteWorkDisposition::RetainForReplay,
6852 "test route detach",
6853 ),
6854 1,
6855 "only the replayable search remains active after route detach"
6856 );
6857 let terminal_response = terminal_rx
6858 .blocking_recv()
6859 .expect("teardown-terminal call stops at cancellation checkpoint");
6860 assert_eq!(terminal_response.data["code"], "request_cancelled");
6861 let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6862 std::thread::sleep(Duration::from_millis(10));
6863 assert!(
6864 tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6865 "route detach must retain work whose response can replay after rebind"
6866 );
6867 assert_eq!(
6868 apply_route_work_disposition(
6869 &active,
6870 executor.as_ref(),
6871 route,
6872 RouteWorkDisposition::Abandon,
6873 "test session purge",
6874 ),
6875 1
6876 );
6877 let tracked_response = tracked_rx
6878 .blocking_recv()
6879 .expect("tracked search stops at cancellation checkpoint");
6880 assert_eq!(tracked_response.data["code"], "request_cancelled");
6881 assert!(active.lock().expect("active tool calls").is_empty());
6882
6883 let deadline = Instant::now() + Duration::from_secs(1);
6884 while !executor.actor_is_idle(&root) {
6885 assert!(
6886 Instant::now() < deadline,
6887 "cancelled search must release the PureRead lane"
6888 );
6889 std::thread::sleep(Duration::from_millis(2));
6890 }
6891 }
6892
6893 #[test]
6894 fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6895 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6896 let executor = Arc::new(Executor::new());
6897 let (dir, root) = test_root("shutdown-deferred-inspect");
6898 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6899 let ctx = inspect_context(dir.path());
6900 executor.register_actor(root.clone(), Arc::clone(&ctx));
6901 let (started_rx, _release_tx) =
6902 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6903 let (pending, cancellation) =
6904 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6905 let route = RouteChannel {
6906 channel: 8,
6907 epoch: 1,
6908 };
6909 started_rx
6910 .recv_timeout(Duration::from_secs(1))
6911 .expect("detached inspect reaches shutdown gate");
6912 let mut registry = PendingSubcResponses::default();
6913 registry.register(PendingSubcResponse {
6914 route,
6915 corr: 42,
6916 flags: Flags::new(false, Priority::Passive, false),
6917 ver: PROTOCOL_VERSION,
6918 root: root.clone(),
6919 session_id: "shutdown-session".to_string(),
6920 bare_name: "inspect".to_string(),
6921 format_context: crate::subc_format::FormatContext::from_tool_call(
6922 "inspect",
6923 &json!({}),
6924 dir.path(),
6925 ),
6926 bind_trust: BindTrust::FirstParty,
6927 pending,
6928 surface_downgraded: false,
6929 phase_trace: PhaseTrace::new(Instant::now()),
6930 });
6931 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6932 (route, 42),
6933 ActiveToolCall {
6934 root_id: root.clone(),
6935 cancellation,
6936 detach_policy: RouteDetachPolicy::CancelOnDetach,
6937 },
6938 )])));
6939
6940 let resolved = registry.drain_on_shutdown(executor.as_ref());
6941 assert!(registry.is_empty());
6942 assert_eq!(resolved.len(), 1);
6943 assert_eq!(
6944 resolved[0].response.data["failure_reason"],
6945 "daemon_shutdown"
6946 );
6947 finish_active_tool_call(&active, route, 42);
6948 let deadline = Instant::now() + Duration::from_secs(1);
6949 while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6950 assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6951 std::thread::sleep(Duration::from_millis(5));
6952 }
6953 assert!(active.lock().expect("active calls").is_empty());
6954 assert!(executor.actor_is_idle(&root));
6955 }
6956
6957 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6958 let deadline = Instant::now() + Duration::from_secs(30);
6959 loop {
6960 let observed = ctx.watcher_registry_count();
6961 if observed == expected {
6962 return;
6963 }
6964 assert!(
6965 Instant::now() < deadline,
6966 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6967 );
6968 std::thread::sleep(Duration::from_millis(50));
6969 }
6970 }
6971
6972 pub(super) fn reap_until_forgotten(
6982 root: &ProjectRootId,
6983 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6984 pending_binds: &HashMap<RouteChannel, PendingBind>,
6985 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6986 executor: &Arc<Executor>,
6987 metrics: &DispatchPathMetrics,
6988 ) -> IdleReapOutcome {
6989 let deadline = Instant::now() + Duration::from_secs(30);
6990 loop {
6991 let outcome = reap_idle_roots(
6992 Instant::now(),
6993 live_roots,
6994 pending_binds,
6995 root_channels,
6996 executor,
6997 metrics,
6998 );
6999 if outcome.forgotten_deleted_roots.contains(root) {
7000 return outcome;
7001 }
7002 assert!(
7003 Instant::now() < deadline,
7004 "deleted root was never forgotten: {root:?}"
7005 );
7006 std::thread::sleep(Duration::from_millis(10));
7007 }
7008 }
7009
7010 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7011 let deadline = Instant::now() + Duration::from_secs(30);
7012 loop {
7013 let observed = app.actor_root_count();
7014 if observed == expected {
7015 return;
7016 }
7017 assert!(
7018 Instant::now() < deadline,
7019 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7020 );
7021 std::thread::sleep(Duration::from_millis(50));
7022 }
7023 }
7024
7025 pub(super) fn status_frame(seq: u64) -> PushFrame {
7026 status_frame_with_session(seq, None)
7027 }
7028
7029 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7030 PushFrame::StatusChanged(StatusChangedFrame {
7031 frame_type: "status_changed",
7032 session_id: session_id.map(str::to_string),
7033 snapshot: json!({ "seq": seq }),
7034 })
7035 }
7036
7037 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7038 completion_frame_with_session(task_id, "session-1")
7039 }
7040
7041 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7042 PushFrame::BashCompleted(BashCompletedFrame {
7043 frame_type: "bash_completed",
7044 task_id: task_id.to_string(),
7045 session_id: session_id.to_string(),
7046 status: BgTaskStatus::Completed,
7047 exit_code: Some(0),
7048 command: format!("echo {task_id}"),
7049 output_preview: String::new(),
7050 bash_output_list_envelope: None,
7051 output_truncated: false,
7052 original_tokens: None,
7053 compressed_tokens: None,
7054 tokens_skipped: false,
7055 status_reason: None,
7056 live_descendants: Some(Vec::new()),
7057 live_descendants_omitted: 0,
7058 live_descendants_summary: None,
7059 })
7060 }
7061
7062 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7063 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7064 }
7065
7066 pub(super) fn long_running_frame_with_session(
7067 task_id: &str,
7068 session_id: &str,
7069 elapsed_ms: u64,
7070 ) -> PushFrame {
7071 PushFrame::BashLongRunning(BashLongRunningFrame {
7072 frame_type: "bash_long_running",
7073 task_id: task_id.to_string(),
7074 session_id: session_id.to_string(),
7075 command: format!("sleep {elapsed_ms}"),
7076 elapsed_ms,
7077 })
7078 }
7079
7080 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7081 PushFrame::BashPatternMatch(BashPatternMatchFrame {
7082 frame_type: "bash_pattern_match",
7083 task_id: "task-pattern".to_string(),
7084 session_id: session_id.to_string(),
7085 watch_id: "watch-1".to_string(),
7086 match_text: "needle".to_string(),
7087 match_offset: 7,
7088 context: "haystack needle".to_string(),
7089 once: true,
7090 reason: "pattern_match",
7091 })
7092 }
7093
7094 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7095 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7096 frame_type: "configure_warnings",
7097 session_id: session_id.map(str::to_string),
7098 project_root: "/tmp/subc-test".to_string(),
7099 warnings: Vec::new(),
7100 })
7101 }
7102
7103 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7104 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7105 }
7106
7107 pub(super) fn route_identity_with_trust(
7108 root: &ProjectRootId,
7109 session_id: &str,
7110 trust: BindTrust,
7111 ) -> RouteIdentity {
7112 RouteIdentity(Arc::new(RouteIdentityData {
7113 root: root.clone(),
7114 project_root: root.as_path().to_path_buf(),
7115 harness: "opencode".to_string(),
7116 session: session_id.to_string(),
7117 trust,
7118 spawn_principal: AuthenticatedPrincipal::RouteBind {
7119 trust: trust.sandbox_trust(),
7120 route_channel: 0,
7121 route_epoch: 0,
7122 project_root: root.as_path().to_path_buf(),
7123 harness: "opencode".to_string(),
7124 session_id: session_id.to_string(),
7125 principal_id: Some(match trust {
7126 BindTrust::FirstParty => "direct".to_string(),
7127 BindTrust::Untrusted => "unverified".to_string(),
7128 }),
7129 },
7130 consumer_elicitation_capable: false,
7131 }))
7132 }
7133
7134 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7135 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7136 }
7137
7138 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7139 match frame {
7140 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7141 _ => None,
7142 }
7143 }
7144
7145 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7146 match frame {
7147 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7148 _ => None,
7149 }
7150 }
7151
7152 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7153 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7154 body.get("task_id")
7155 .and_then(serde_json::Value::as_str)
7156 .map(str::to_string)
7157 }
7158}
7159
7160#[cfg(test)]
7161mod tests {
7162 use super::test_support::{
7163 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7164 wait_for_actor_root_count, wait_for_watcher_count,
7165 };
7166 use super::*;
7167 use crate::bash_background::BgTaskStatus;
7168
7169 #[test]
7173 fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7174 assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7175 assert!(matches!(
7176 module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7177 Err(SubcError::ConnectionLost)
7178 ));
7179 assert!(matches!(
7180 module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7181 Err(SubcError::ActorFatal)
7182 ));
7183 }
7184
7185 #[test]
7186 fn fatal_panic_responses_are_detected_and_noted() {
7187 let panic = Response::error(
7188 "req-fatal",
7189 "actor_fatal",
7190 "start byte index 7 is not a char boundary",
7191 );
7192 assert!(note_fatal_panic_response(&panic));
7193 let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7194 assert!(!note_fatal_panic_response(&ordinary));
7195 }
7196
7197 fn attach_error(kind: io::ErrorKind) -> SubcError {
7198 SubcError::Connect {
7199 endpoint: "127.0.0.1:1".to_string(),
7200 source: io::Error::new(kind, "constructed attach failure"),
7201 }
7202 }
7203
7204 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7205 SubcError::Auth {
7206 endpoint: "127.0.0.1:1".to_string(),
7207 source: subc_transport::AuthError::Io {
7208 stage: subc_transport::AuthStage::ServerProof,
7209 source: io::Error::new(kind, "constructed auth failure"),
7210 },
7211 }
7212 }
7213
7214 fn cpu_hunt_process_cpu_us() -> u64 {
7215 #[cfg(unix)]
7216 {
7217 let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7218 if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7220 let usage = unsafe { usage.assume_init() };
7221 return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7222 + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7223 }
7224 }
7225 0
7226 }
7227
7228 #[test]
7229 fn route_bind_does_not_recompute_fleet_health() {
7230 let runtime = tokio::runtime::Builder::new_current_thread()
7231 .enable_all()
7232 .build()
7233 .expect("bind runtime");
7234 runtime.block_on(async {
7235 let (dir, root) = test_root("bind-health-work-count");
7236 let app = App::default_shared();
7237 let executor = Arc::new(Executor::new());
7238 let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7239 let mut fixture_dirs = Vec::new();
7240 if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7243 let copy = std::fs::canonicalize(copy).expect("copied store exists");
7244 let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7245 .parent()
7246 .expect("crates directory")
7247 .parent()
7248 .expect("checkout directory")
7249 .canonicalize()
7250 .expect("canonical checkout");
7251 assert!(copy.starts_with(project.join("target")));
7252 for index in 0..36 {
7253 let actor = if index == 0 {
7254 Arc::clone(&ctx)
7255 } else {
7256 Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7257 };
7258 actor.update_config(|config| config.project_root = Some(project.clone()));
7259 *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7260 crate::callgraph_store::CallGraphStore::open_readonly(
7261 copy.clone(),
7262 project.clone(),
7263 )
7264 .expect("open copied graph")
7265 .expect("copied graph ready"),
7266 ));
7267 if index > 0 {
7268 let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7269 assert!(executor.register_actor(id, actor));
7270 fixture_dirs.push(fixture);
7271 }
7272 }
7273 }
7274 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7275 let cache = HealthRollupCache::new();
7276 if !fixture_dirs.is_empty() {
7277 let started = Instant::now();
7278 let cpu = cpu_hunt_process_cpu_us();
7279 cache.refresh(&executor, &app);
7280 eprintln!(
7281 "cpu_hunt health_rollup wall_us={} cpu_us={}",
7282 started.elapsed().as_micros(),
7283 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7284 );
7285 let started = Instant::now();
7286 let cpu = cpu_hunt_process_cpu_us();
7287 std::hint::black_box(ctx.build_status_snapshot());
7288 eprintln!(
7289 "cpu_hunt status wall_us={} cpu_us={}",
7290 started.elapsed().as_micros(),
7291 cpu_hunt_process_cpu_us().saturating_sub(cpu)
7292 );
7293 }
7294 let refreshes_before = cache.refresh_count_for_test();
7295 let metrics = Arc::new(DispatchPathMetrics::new());
7296 let (writer_tx, _writer_rx) = mpsc::channel(8);
7297 let (completion_tx, mut completion_rx) = mpsc::channel(8);
7298 let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7299 let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7300 let senders = PushSenders {
7301 lossy_tx,
7302 reliable_tx,
7303 lossy_overflow: Arc::new(push::LossyOverflow::default()),
7304 lossy_seq: Arc::new(AtomicU64::new(0)),
7305 fleet_status_client: FleetStatusClient::channel(1).0,
7306 };
7307 let request = ModuleControlRequest::RouteBind {
7308 route_channel: 1,
7309 epoch: 1,
7310 target: RouteTarget::ToolProvider {
7311 module_id: "aft".to_string(),
7312 },
7313 identity: subc_protocol::BindIdentity {
7314 project_root: root.as_path().to_path_buf(),
7315 harness: "opencode".to_string(),
7316 session: "bind-health-work-count".to_string(),
7317 },
7318 principal: Some(subc_protocol::Principal::Direct),
7319 consumer_capabilities: None,
7320 admission_facts: Default::default(),
7321 };
7322 let frame = Frame::build_with_version(
7323 PROTOCOL_VERSION,
7324 FrameType::Request,
7325 control_flags(),
7326 0,
7327 0,
7328 1,
7329 serde_json::to_vec(&request).expect("bind body"),
7330 )
7331 .expect("bind frame");
7332 let mut pending_binds = HashMap::new();
7333 let started = Instant::now();
7334 let cpu = cpu_hunt_process_cpu_us();
7335 handle_control_request(
7336 &writer_tx,
7337 &frame,
7338 &app,
7339 &executor,
7340 &mut HashMap::new(),
7341 &mut pending_binds,
7342 &mut HashMap::new(),
7343 &mut HashMap::new(),
7344 &mut HashSet::new(),
7345 &mut HashMap::new(),
7346 &mut HashMap::new(),
7347 &mut HashMap::new(),
7348 &mut HashMap::new(),
7349 &mut HashMap::new(),
7350 &mut HashMap::new(),
7351 &Arc::new(StdMutex::new(HashMap::new())),
7352 &mut PendingSubcResponses::default(),
7353 &mut RetryBuffer::new(),
7354 &mut HashMap::new(),
7355 &Arc::new(Notify::new()),
7356 &completion_tx,
7357 &metrics,
7358 None,
7359 &cache,
7360 &senders,
7361 |request, _| Response::success(request.id, json!({})),
7362 Some(&dir.path().join("absent-user-config.json")),
7363 usize::MAX,
7364 )
7365 .await
7366 .expect("admit route bind");
7367 eprintln!(
7368 "route_bind_health admission_us={} cpu_us={} refreshes={}",
7369 started.elapsed().as_micros(),
7370 cpu_hunt_process_cpu_us().saturating_sub(cpu),
7371 cache.refresh_count_for_test() - refreshes_before
7372 );
7373 let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7374 .await
7375 .expect("configure completes")
7376 .expect("completion delivered");
7377 assert!(completion.configure_response.success);
7378 assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7379 assert_eq!(
7380 cache.refresh_count_for_test() - refreshes_before,
7381 0,
7382 "route admission must not perform a fleet-wide health census"
7383 );
7384 });
7385 }
7386
7387 #[test]
7388 fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7389 let (dir, root) = test_root("health-does-not-lock-bash-db");
7390 let executor = Arc::new(Executor::new());
7391 let ctx = test_ctx();
7392 ctx.set_harness(crate::harness::Harness::Opencode);
7393 let db = Arc::new(StdMutex::new(
7394 crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7395 ));
7396 ctx.bash_background().set_db_pool(Arc::clone(&db));
7397 executor.register_actor(root, ctx);
7398
7399 let guard = db.lock().expect("hold bash-background DB mutex");
7400 let app = App::default_shared();
7401 let metrics = Arc::new(DispatchPathMetrics::new());
7402 let health_rollup_cache = Arc::new(HealthRollupCache::new());
7403 let frame = Frame::build_with_version(
7404 PROTOCOL_VERSION,
7405 FrameType::Request,
7406 control_flags(),
7407 0,
7408 0,
7409 77,
7410 Vec::new(),
7411 )
7412 .expect("health request frame");
7413 let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7414 let (done_tx, done_rx) = std::sync::mpsc::channel();
7415 let join = std::thread::spawn(move || {
7416 let runtime = tokio::runtime::Builder::new_current_thread()
7417 .enable_all()
7418 .build()
7419 .expect("health test runtime");
7420 let result = runtime.block_on(send_cached_health_response(
7421 &writer_tx,
7422 &frame,
7423 &app,
7424 &executor,
7425 &HashMap::new(),
7426 &metrics,
7427 &health_rollup_cache,
7428 ));
7429 done_tx.send(result).expect("report health result");
7430 });
7431
7432 let result = done_rx
7433 .recv_timeout(Duration::from_millis(500))
7434 .expect("channel-0 health blocked on bash-background DB mutex");
7435 result.expect("send cached health response");
7436 drop(guard);
7437 join.join().expect("health thread");
7438 }
7439
7440 #[test]
7441 fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7442 let (_dir_a, root_a) = test_root("health-metric-root-a");
7443 let (_dir_b, root_b) = test_root("health-metric-root-b");
7444 let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7445 let snapshots = HashMap::from([
7446 (root_a, HashSet::from([duplicate_key.to_string()])),
7447 (root_b, HashSet::from([duplicate_key.to_string()])),
7448 ]);
7449 let metrics = DispatchPathMetrics::new();
7450
7451 record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7452
7453 assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7454 }
7455
7456 #[test]
7457 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7458 let transient_errors = vec![
7459 attach_error(io::ErrorKind::ConnectionRefused),
7460 attach_error(io::ErrorKind::TimedOut),
7461 attach_error(io::ErrorKind::ConnectionReset),
7462 auth_io_error(io::ErrorKind::ConnectionAborted),
7463 auth_io_error(io::ErrorKind::BrokenPipe),
7464 SubcError::Auth {
7465 endpoint: "127.0.0.1:1".to_string(),
7466 source: subc_transport::AuthError::UnexpectedEof {
7467 stage: subc_transport::AuthStage::ServerProof,
7468 expected: 4,
7469 actual: 0,
7470 },
7471 },
7472 SubcError::Auth {
7473 endpoint: "127.0.0.1:1".to_string(),
7474 source: subc_transport::AuthError::Timeout {
7475 stage: subc_transport::AuthStage::ServerProof,
7476 deadline: AUTH_DEADLINE,
7477 },
7478 },
7479 ];
7480 for error in &transient_errors {
7481 assert_eq!(
7482 classify_attach_error(error),
7483 AttachErrorClass::Transient,
7484 "expected transient: {error}"
7485 );
7486 }
7487
7488 let permanent_errors = vec![
7489 attach_error(io::ErrorKind::PermissionDenied),
7490 auth_io_error(io::ErrorKind::InvalidData),
7491 SubcError::Auth {
7492 endpoint: "127.0.0.1:1".to_string(),
7493 source: subc_transport::AuthError::InvalidServerProof,
7494 },
7495 SubcError::Auth {
7496 endpoint: "127.0.0.1:1".to_string(),
7497 source: subc_transport::AuthError::DaemonIdMismatch,
7498 },
7499 SubcError::ConnectionFile {
7500 path: PathBuf::from("subc-connection.json"),
7501 source: subc_transport::ConnectionFileError::Invalid {
7502 reason: "constructed invalid file".to_string(),
7503 },
7504 },
7505 SubcError::NoEndpoint {
7506 path: PathBuf::from("subc-connection.json"),
7507 },
7508 SubcError::InvalidEndpoint {
7509 path: PathBuf::from("subc-connection.json"),
7510 endpoint: "not-an-ip:1234".to_string(),
7511 },
7512 ];
7513 for error in &permanent_errors {
7514 assert_eq!(
7515 classify_attach_error(error),
7516 AttachErrorClass::Permanent,
7517 "expected permanent: {error}"
7518 );
7519 }
7520 }
7521
7522 #[test]
7523 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7524 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7525 let conn_path = conn_dir.path().join("subc-connection.json");
7526 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7527 listener
7528 .set_nonblocking(true)
7529 .expect("set listener nonblocking");
7530 let port = listener.local_addr().expect("listener addr").port();
7531 connection_file::write_atomic(
7532 &conn_path,
7533 &connection_file::ConnectionInfo {
7534 schema: connection_file::SCHEMA_VERSION,
7535 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7536 endpoints: vec![connection_file::Endpoint {
7537 host: "127.0.0.1".to_string(),
7538 port,
7539 }],
7540 key: vec![0x42; subc_transport::KEY_LEN],
7541 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7542 pid: std::process::id(),
7543 daemon_ver: "subc-test".to_string(),
7544 },
7545 )
7546 .expect("write connection file");
7547
7548 let runtime = tokio::runtime::Builder::new_current_thread()
7549 .enable_all()
7550 .build()
7551 .expect("test runtime");
7552 let result = runtime.block_on(connect_and_authenticate_with_policy(
7553 &conn_path,
7554 AttachRetryPolicy {
7555 budget: Duration::from_secs(1),
7556 initial_backoff: Duration::from_millis(5),
7557 max_backoff: Duration::from_millis(10),
7558 jitter_percent: 0,
7559 },
7560 None,
7561 ));
7562 assert!(matches!(
7563 result,
7564 Err(SubcError::ConnectionFile {
7565 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7566 ..
7567 })
7568 ));
7569 assert!(matches!(
7570 listener.accept(),
7571 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7572 ));
7573 }
7574
7575 #[test]
7576 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7577 let conn_dir = tempfile::tempdir().expect("connection tempdir");
7578 let conn_path = conn_dir.path().join("subc-connection.json");
7579 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7580 let port = listener.local_addr().expect("reserved addr").port();
7581 drop(listener);
7582 connection_file::write_atomic(
7583 &conn_path,
7584 &connection_file::ConnectionInfo {
7585 schema: connection_file::SCHEMA_VERSION,
7586 wire_version: Some(PROTOCOL_VERSION),
7587 endpoints: vec![connection_file::Endpoint {
7588 host: "127.0.0.1".to_string(),
7589 port,
7590 }],
7591 key: vec![0x42; subc_transport::KEY_LEN],
7592 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7593 pid: std::process::id(),
7594 daemon_ver: "subc-test".to_string(),
7595 },
7596 )
7597 .expect("write connection file");
7598
7599 let policy = AttachRetryPolicy {
7600 budget: Duration::from_millis(40),
7601 initial_backoff: Duration::from_millis(5),
7602 max_backoff: Duration::from_millis(10),
7603 jitter_percent: 0,
7604 };
7605 let runtime = tokio::runtime::Builder::new_current_thread()
7606 .enable_all()
7607 .build()
7608 .expect("test runtime");
7609 let started_at = Instant::now();
7610 let result = runtime.block_on(connect_and_authenticate_with_policy(
7611 &conn_path, policy, None,
7612 ));
7613 let elapsed = started_at.elapsed();
7614 let error = match result {
7615 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7616 Err(error) => error,
7617 };
7618
7619 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7620 assert!(
7621 elapsed >= Duration::from_millis(35),
7622 "retry budget ended too early: {elapsed:?}"
7623 );
7624 assert!(
7625 elapsed < Duration::from_secs(1),
7626 "retry budget was not bounded: {elapsed:?}"
7627 );
7628 }
7629
7630 fn due_maintenance_jobs_without_actor_context(
7631 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7632 budget: usize,
7633 pending_bind_roots: &HashSet<ProjectRootId>,
7634 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7635 due_maintenance_jobs(
7636 live_roots,
7637 None,
7638 &HashMap::new(),
7639 &BgWakePending::new(),
7640 budget,
7641 pending_bind_roots,
7642 )
7643 }
7644
7645 fn actor_ctx_with_dirty_search_index(
7646 root: &Path,
7647 storage: &Path,
7648 file_name: &str,
7649 old_contents: &str,
7650 new_contents: &str,
7651 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7652 let file = root.join(file_name);
7653 std::fs::write(&file, old_contents).expect("write source");
7654 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7655 let ctx = Arc::new(AppContext::new(
7656 Box::new(crate::parser::TreeSitterProvider::new()),
7657 Config {
7658 project_root: Some(root.to_path_buf()),
7659 storage_dir: Some(storage.to_path_buf()),
7660 ..Config::default()
7661 },
7662 ));
7663 ctx.set_canonical_cache_root(canonical_root.clone());
7664
7665 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7666 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7667 let git_head = index.stored_git_head().map(str::to_owned);
7668 index.write_to_disk(&cache_dir, git_head.as_deref());
7669
7670 std::fs::write(&file, new_contents).expect("edit source");
7671 index.update_file(&file);
7672 *ctx.search_index()
7673 .write()
7674 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7675 (ctx, canonical_root, cache_dir)
7676 }
7677
7678 #[test]
7679 fn graceful_shutdown_flushes_every_actor_search_index() {
7680 let storage = tempfile::tempdir().expect("storage tempdir");
7681 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7682 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7683 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7684 root1_dir.path(),
7685 storage.path(),
7686 "alpha.txt",
7687 "old actor one token\n",
7688 "new actor one token\n",
7689 );
7690 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7691 root2_dir.path(),
7692 storage.path(),
7693 "beta.txt",
7694 "old actor two token\n",
7695 "new actor two token\n",
7696 );
7697
7698 let executor = Executor::new();
7699 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7700 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7701
7702 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7703
7704 let mut restored1 =
7705 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7706 .expect("load flushed root one index");
7707 restored1.ready = true;
7708 assert_eq!(
7709 restored1
7710 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7711 .matches
7712 .len(),
7713 1,
7714 "graceful subc shutdown should flush the first root's trigram delta"
7715 );
7716
7717 let mut restored2 =
7718 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7719 .expect("load flushed root two index");
7720 restored2.ready = true;
7721 assert_eq!(
7722 restored2
7723 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7724 .matches
7725 .len(),
7726 1,
7727 "graceful subc shutdown should flush every registered root"
7728 );
7729 }
7730
7731 #[test]
7732 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7733 let _ = env_logger::builder().is_test(true).try_init();
7734 let (root_dir, root) = test_root("idle-root-reaper");
7735 let storage = tempfile::tempdir().expect("storage tempdir");
7736 std::fs::write(
7737 root_dir.path().join("main.rs"),
7738 "fn entry() { leaf(); }\nfn leaf() {}\n",
7739 )
7740 .expect("source file");
7741 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7742 let app = App::default_shared();
7743 let ctx = Arc::new(AppContext::from_app(
7744 Arc::clone(&app),
7745 Config {
7746 project_root: Some(canonical_root.clone()),
7747 storage_dir: Some(storage.path().to_path_buf()),
7748 callgraph_store: true,
7749 search_index: true,
7750 ..Config::default()
7751 },
7752 ));
7753 ctx.set_canonical_cache_root(canonical_root.clone());
7754 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7755 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7756 assert!(ctx
7757 .ensure_callgraph_store()
7758 .expect("build callgraph store")
7759 .is_some());
7760
7761 let cache_dir =
7762 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7763 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7764 let git_head = index.stored_git_head().map(str::to_owned);
7765 index.write_to_disk(&cache_dir, git_head.as_deref());
7766 *ctx.search_index()
7767 .write()
7768 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7769 let seeded_generation =
7772 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7773 .expect("seeded artifact generation");
7774 crate::cache_freshness::record_verify_completed(
7775 &canonical_root,
7776 crate::cache_freshness::VerifyArtifact::Search,
7777 Some(seeded_generation),
7778 );
7779 assert!(
7780 matches!(
7781 crate::cache_freshness::warm_verify_plan(
7782 canonical_root.as_path(),
7783 crate::cache_freshness::VerifyArtifact::Search,
7784 Some(seeded_generation),
7785 ),
7786 crate::cache_freshness::WarmVerifyPlan::Skip
7787 ),
7788 "memo must be warm before eviction for the downgrade assertion to bite"
7789 );
7790
7791 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7792 let _dispatch_tx = dispatch_tx;
7793 let shutdown = Arc::new(AtomicBool::new(false));
7794 let thread_shutdown = Arc::clone(&shutdown);
7795 let join = std::thread::spawn(move || {
7796 while !thread_shutdown.load(Ordering::SeqCst) {
7797 std::thread::yield_now();
7798 }
7799 });
7800 ctx.install_watcher_runtime(
7801 dispatch_rx,
7802 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7803 );
7804 wait_for_watcher_count(&ctx, 1);
7805
7806 let executor = Arc::new(Executor::new());
7807 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7808 ctx.mark_subc_unbound();
7809 let mut live_roots = HashMap::new();
7810 let mut meta = RootMeta::new(Instant::now());
7811 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7812 meta.unbound_quiesced = true;
7813 live_roots.insert(root.clone(), meta);
7814
7815 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7816 assert!(message.contains("evicted idle root"));
7817 assert!(message.contains("freed ~"));
7818 assert!(message.contains("semantic"));
7819 assert!(!message.contains("semantic not estimated retained"));
7820 assert!(message.contains("trigram"));
7821 assert!(message.contains("retained: bash"));
7822 assert!(message.contains("parser_pool"));
7823
7824 assert_eq!(
7825 reap_idle_roots(
7826 Instant::now(),
7827 &mut live_roots,
7828 &HashMap::new(),
7829 &HashMap::new(),
7830 &executor,
7831 &DispatchPathMetrics::new(),
7832 )
7833 .evicted,
7834 1
7835 );
7836 assert!(ctx.search_index().read().unwrap().is_none());
7837 wait_for_watcher_count(&ctx, 0);
7838 assert!(
7843 matches!(
7844 crate::cache_freshness::warm_verify_plan(
7845 canonical_root.as_path(),
7846 crate::cache_freshness::VerifyArtifact::Search,
7847 Some(seeded_generation),
7848 ),
7849 crate::cache_freshness::WarmVerifyPlan::Strict
7850 ),
7851 "idle eviction must force strict re-verification"
7852 );
7853 assert!(
7854 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7855 );
7856 ctx.mark_subc_bound();
7857 assert!(ctx
7858 .ensure_callgraph_store()
7859 .expect("reopen callgraph store")
7860 .is_some());
7861 assert!(live_roots[&root].idle_artifacts_evicted);
7862 }
7863
7864 #[test]
7865 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7866 let (_root_dir, root) = test_root("idle-root-ttl-gate");
7867 let ctx = test_ctx();
7868 let executor = Arc::new(Executor::new());
7869 assert!(executor.register_actor(root.clone(), ctx));
7870 let ctx = executor.actor_context(&root).expect("actor context");
7871 ctx.mark_subc_unbound();
7872 let now = Instant::now();
7873 let mut meta = RootMeta::new(now);
7874 meta.unbound_quiesced = true;
7875 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7876
7877 assert_eq!(
7881 reap_idle_roots(
7882 now,
7883 &mut live_roots,
7884 &HashMap::new(),
7885 &HashMap::new(),
7886 &executor,
7887 &DispatchPathMetrics::new(),
7888 )
7889 .evicted,
7890 0
7891 );
7892 assert!(!live_roots[&root].idle_artifacts_evicted);
7893
7894 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7900 assert_eq!(
7901 reap_idle_roots(
7902 now + IDLE_ROOT_TTL,
7903 &mut live_roots,
7904 &HashMap::new(),
7905 &HashMap::new(),
7906 &executor,
7907 &DispatchPathMetrics::new(),
7908 )
7909 .evicted,
7910 1
7911 );
7912 assert!(live_roots[&root].idle_artifacts_evicted);
7913 assert!(
7914 ctx.take_pending_search_index_paths().is_empty(),
7915 "TTL eviction must dispose retained pending reconciliation paths"
7916 );
7917 }
7918
7919 #[test]
7920 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7921 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7922 let ctx = test_ctx();
7923 let executor = Arc::new(Executor::new());
7924 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7925 ctx.mark_subc_unbound();
7926
7927 let pending = root.as_path().join("edited-while-unbound.rs");
7933 ctx.add_pending_search_index_paths([pending.clone()]);
7934 let dirty_source = root.as_path().join("dirty.rs");
7935 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7936 let mut dirty = crate::search_index::SearchIndex::new();
7937 dirty.ready = true;
7938 dirty.update_file(&dirty_source);
7939 *ctx.search_index()
7940 .write()
7941 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7942 assert!(ctx.artifact_eviction_blocked());
7943
7944 let mut live_roots = HashMap::new();
7945 let mut meta = RootMeta::new(Instant::now());
7946 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7947 meta.unbound_quiesced = true;
7948 live_roots.insert(root.clone(), meta);
7949
7950 assert_eq!(
7951 reap_idle_roots(
7952 Instant::now(),
7953 &mut live_roots,
7954 &HashMap::new(),
7955 &HashMap::new(),
7956 &executor,
7957 &DispatchPathMetrics::new(),
7958 )
7959 .evicted,
7960 0,
7961 "the dirty index must still block this eviction"
7962 );
7963 assert_eq!(
7964 ctx.take_pending_search_index_paths(),
7965 vec![pending],
7966 "a blocked eviction must restore the taken pending paths"
7967 );
7968 }
7969
7970 #[test]
7971 fn idle_reap_with_bound_route_keeps_watcher_running() {
7972 let (_root_dir, root) = test_root("bound-root-reap-gate");
7973 let ctx = test_ctx();
7974 let executor = Arc::new(Executor::new());
7975 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7976
7977 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7978 let _dispatch_tx = dispatch_tx;
7979 let shutdown = Arc::new(AtomicBool::new(false));
7980 let thread_shutdown = Arc::clone(&shutdown);
7981 let join = std::thread::spawn(move || {
7982 while !thread_shutdown.load(Ordering::SeqCst) {
7983 std::thread::yield_now();
7984 }
7985 });
7986 ctx.install_watcher_runtime(
7987 dispatch_rx,
7988 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7989 );
7990
7991 let mut meta = RootMeta::new(Instant::now());
7992 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7993 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7994 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7995 assert_eq!(
7996 reap_idle_roots(
7997 Instant::now(),
7998 &mut live_roots,
7999 &HashMap::new(),
8000 &bound,
8001 &executor,
8002 &DispatchPathMetrics::new(),
8003 )
8004 .evicted,
8005 0
8006 );
8007 wait_for_watcher_count(&ctx, 1);
8008 ctx.stop_watcher_runtime_in_background();
8009 wait_for_watcher_count(&ctx, 0);
8010 }
8011
8012 #[test]
8013 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8014 let (root_dir, root) = test_root("deleted-bound-root-reap");
8015 let executor = Arc::new(Executor::new());
8016 assert!(executor.register_actor(root.clone(), test_ctx()));
8017 root_dir.close().expect("delete project root");
8018
8019 let route = route_key(19, 3);
8020 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8021 let cancel_signal = PersistentCancelSignal::new();
8022 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8023 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8024 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8025 let mut route_bash_cancels = HashMap::from([(
8026 route,
8027 bash::RouteBashCancel {
8028 token: cancel_signal.clone(),
8029 active_waits: 0,
8030 },
8031 )]);
8032 let metrics = DispatchPathMetrics::new();
8033
8034 let first = reap_idle_roots(
8035 Instant::now(),
8036 &mut live_roots,
8037 &HashMap::new(),
8038 &root_channels,
8039 &executor,
8040 &metrics,
8041 );
8042 assert!(first.forgotten_deleted_roots.is_empty());
8043 assert!(executor.actor_registered(&root));
8044
8045 let mut forgotten = Vec::new();
8046 for _ in 0..100 {
8047 let outcome = reap_idle_roots(
8048 Instant::now(),
8049 &mut live_roots,
8050 &HashMap::new(),
8051 &root_channels,
8052 &executor,
8053 &metrics,
8054 );
8055 if !outcome.forgotten_deleted_roots.is_empty() {
8056 forgotten = outcome.forgotten_deleted_roots;
8057 break;
8058 }
8059 std::thread::sleep(Duration::from_millis(10));
8060 }
8061 assert_eq!(forgotten, vec![root.clone()]);
8062 assert!(!executor.actor_registered(&root));
8063
8064 let mut retry_buffer = HashMap::new();
8065 let mut reclaimed_routes = ReclaimedRoutes::default();
8066 let mut session_identity = HashMap::new();
8067 let mut push_buffer = HashMap::new();
8068 let mut bg_subs = HashMap::from([(
8069 route,
8070 BgSub {
8071 corr: 77,
8072 ver: PROTOCOL_VERSION,
8073 flags: control_flags(),
8074 root: root.clone(),
8075 session: "deleted-route".to_string(),
8076 },
8077 )]);
8078 let mut bg_sub_by_session = HashMap::from([(
8079 (root.clone(), "deleted-route".to_string()),
8080 HashSet::from([route]),
8081 )]);
8082 let mut bg_wake_pending =
8083 BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8084 let mut bg_wake_epoch = HashMap::new();
8085 let mut pending_bash_asks = HashMap::new();
8086 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8087 health::take_bg_observability_logs_for_test();
8088 purge_deleted_root_residents(
8089 &root,
8090 &mut routes,
8091 &mut root_channels,
8092 &mut installed_route_epochs,
8093 &mut route_bash_cancels,
8094 &active_tool_calls,
8095 executor.as_ref(),
8096 &mut retry_buffer,
8097 &mut reclaimed_routes,
8098 &mut session_identity,
8099 &mut push_buffer,
8100 &mut bg_subs,
8101 &mut bg_sub_by_session,
8102 &mut bg_wake_pending,
8103 &mut bg_wake_epoch,
8104 &mut pending_bash_asks,
8105 &metrics,
8106 );
8107
8108 assert!(routes.is_empty());
8109 assert!(root_channels.is_empty());
8110 assert!(installed_route_epochs.is_empty());
8111 assert!(route_bash_cancels.is_empty());
8112 assert!(reclaimed_routes.contains(route));
8113 assert!(cancel_signal.is_cancelled());
8114 assert_eq!(
8115 health::take_bg_observability_logs_for_test(),
8116 vec![format!(
8117 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8118 root.as_path().display()
8119 )]
8120 );
8121 }
8122
8123 #[test]
8130 fn live_root_with_bound_route_is_never_reclaimed() {
8131 let (_root_dir, root) = test_root("live-bound-root-retained");
8132 let ctx = test_ctx();
8133 ctx.mark_subc_unbound();
8134 let executor = Arc::new(Executor::new());
8135 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8136
8137 let route = RouteChannel {
8138 channel: 7,
8139 epoch: 1,
8140 };
8141 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8142 meta.unbound_quiesced = true;
8143 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8144 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8145
8146 for _ in 0..3 {
8151 let outcome = reap_idle_roots(
8152 Instant::now(),
8153 &mut live_roots,
8154 &HashMap::new(),
8155 &root_channels,
8156 &executor,
8157 &DispatchPathMetrics::new(),
8158 );
8159 assert!(
8160 outcome.forgotten_deleted_roots.is_empty(),
8161 "a root whose directory exists must never be forgotten"
8162 );
8163 }
8164
8165 assert!(live_roots.contains_key(&root), "live root must be retained");
8166 assert!(
8167 executor.actor_registered(&root),
8168 "live root's actor must survive"
8169 );
8170 assert!(
8171 root.as_path().exists(),
8172 "test vehicle must keep the directory alive; otherwise this control proves nothing"
8173 );
8174 }
8175
8176 #[test]
8177 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8178 let (root_dir, root) = test_root("deleted-root-first-observation");
8179 let ctx = test_ctx();
8180 ctx.mark_subc_unbound();
8181 let executor = Arc::new(Executor::new());
8182 assert!(executor.register_actor(root.clone(), ctx));
8183 root_dir.close().expect("delete project root");
8184
8185 let mut meta = RootMeta::new(Instant::now());
8186 meta.unbound_quiesced = true;
8187 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8188 let outcome = reap_idle_roots(
8189 Instant::now(),
8190 &mut live_roots,
8191 &HashMap::new(),
8192 &HashMap::new(),
8193 &executor,
8194 &DispatchPathMetrics::new(),
8195 );
8196
8197 assert!(outcome.forgotten_deleted_roots.is_empty());
8198 assert!(live_roots.contains_key(&root));
8199 assert!(executor.actor_registered(&root));
8200 }
8201
8202 fn spawn_background_for_root(
8203 ctx: &AppContext,
8204 root: &ProjectRootId,
8205 storage: &tempfile::TempDir,
8206 session_id: &str,
8207 ) -> (String, u32) {
8208 let command = if cfg!(windows) {
8214 "ping -n 31 127.0.0.1 > nul"
8216 } else {
8217 "sleep 30"
8218 };
8219 let task_id = ctx
8220 .bash_background()
8221 .spawn(
8222 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8223 command,
8224 session_id.to_string(),
8225 storage.path().to_path_buf(),
8226 HashMap::new(),
8227 Some(Duration::from_secs(60)),
8228 storage.path().to_path_buf(),
8229 8,
8230 true,
8231 false,
8232 Some(root.as_path().to_path_buf()),
8233 )
8234 .expect("spawn background task");
8235 let snapshot = ctx
8236 .bash_background()
8237 .status(
8238 &task_id,
8239 session_id,
8240 Some(root.as_path()),
8241 Some(storage.path()),
8242 0,
8243 )
8244 .expect("background task status");
8245 (task_id, snapshot.child_pid.expect("background child pid"))
8246 }
8247
8248 fn wait_for_background_exit(pid: u32) {
8249 let deadline = Instant::now() + Duration::from_secs(5);
8250 while crate::bash_background::process::is_process_alive(pid) {
8251 assert!(
8252 Instant::now() < deadline,
8253 "background task process survived kill"
8254 );
8255 std::thread::sleep(Duration::from_millis(20));
8256 }
8257 }
8258
8259 #[test]
8260 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8261 let (root_dir, root) = test_root("deleted-root-background-task");
8262 let storage = tempfile::tempdir().expect("task storage");
8263 let ctx = test_ctx();
8264 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8265 assert!(crate::bash_background::process::is_process_alive(pid));
8266
8267 let executor = Arc::new(Executor::new());
8268 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8269 assert!(executor.actor_is_idle(&root));
8270 root_dir.close().expect("delete project root");
8271 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8272 let pending_binds = HashMap::new();
8273 let root_channels = HashMap::new();
8274 let metrics = DispatchPathMetrics::new();
8275
8276 let first = reap_idle_roots(
8277 Instant::now(),
8278 &mut live_roots,
8279 &pending_binds,
8280 &root_channels,
8281 &executor,
8282 &metrics,
8283 );
8284 assert!(first.forgotten_deleted_roots.is_empty());
8285 assert!(crate::bash_background::process::is_process_alive(pid));
8286
8287 let outcome = reap_until_forgotten(
8288 &root,
8289 &mut live_roots,
8290 &pending_binds,
8291 &root_channels,
8292 &executor,
8293 &metrics,
8294 );
8295 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8296 wait_for_background_exit(pid);
8297
8298 let snapshot = ctx
8299 .bash_background()
8300 .status(
8301 &task_id,
8302 "reclaim-session",
8303 Some(root.as_path()),
8304 Some(storage.path()),
8305 0,
8306 )
8307 .expect("reclaimed task status");
8308 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8309 assert_eq!(
8310 snapshot.info.status_reason.as_deref(),
8311 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8312 );
8313 assert_eq!(
8314 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8315 crate::bash_background::registry::ROOT_RECLAIMED_REASON
8316 );
8317 let completion = ctx
8318 .bash_background()
8319 .drain_completions_for_session(Some("reclaim-session"))
8320 .pop()
8321 .expect("reclaimed task completion");
8322 assert_eq!(
8323 completion.status_reason.as_deref(),
8324 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8325 );
8326 }
8327
8328 #[test]
8329 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8330 let (root_dir, root) = test_root("existing-root-background-task");
8331 let storage = tempfile::tempdir().expect("task storage");
8332 let ctx = test_ctx();
8333 ctx.mark_subc_unbound();
8334 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8335
8336 let executor = Arc::new(Executor::new());
8337 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8338 let mut meta = RootMeta::new(
8339 Instant::now()
8340 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8341 .expect("old root timestamp"),
8342 );
8343 meta.unbound_quiesced = true;
8344 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8345 let pending_binds = HashMap::new();
8346 let root_channels = HashMap::new();
8347 let metrics = DispatchPathMetrics::new();
8348
8349 for _ in 0..8 {
8350 reap_idle_roots(
8351 Instant::now(),
8352 &mut live_roots,
8353 &pending_binds,
8354 &root_channels,
8355 &executor,
8356 &metrics,
8357 );
8358 std::thread::sleep(Duration::from_millis(10));
8359 }
8360 assert!(root_dir.path().exists());
8361 assert!(crate::bash_background::process::is_process_alive(pid));
8362 let snapshot = ctx
8363 .bash_background()
8364 .status(
8365 &task_id,
8366 "existing-session",
8367 Some(root.as_path()),
8368 Some(storage.path()),
8369 0,
8370 )
8371 .expect("existing task status");
8372 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8373 let _ = ctx.bash_background().kill(&task_id, "existing-session");
8374 wait_for_background_exit(pid);
8375 }
8376
8377 #[test]
8378 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8379 let (root_dir, root) = test_root("restored-root-background-task");
8380 let storage = tempfile::tempdir().expect("task storage");
8381 let ctx = test_ctx();
8382 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8383
8384 let executor = Arc::new(Executor::new());
8385 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8386 root_dir.close().expect("delete project root");
8387 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8388 let pending_binds = HashMap::new();
8389 let root_channels = HashMap::new();
8390 let metrics = DispatchPathMetrics::new();
8391
8392 let first = reap_idle_roots(
8393 Instant::now(),
8394 &mut live_roots,
8395 &pending_binds,
8396 &root_channels,
8397 &executor,
8398 &metrics,
8399 );
8400 assert!(first.forgotten_deleted_roots.is_empty());
8401 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8402 let second = reap_idle_roots(
8403 Instant::now(),
8404 &mut live_roots,
8405 &pending_binds,
8406 &root_channels,
8407 &executor,
8408 &metrics,
8409 );
8410 assert!(second.forgotten_deleted_roots.is_empty());
8411 assert!(crate::bash_background::process::is_process_alive(pid));
8412 let _ = ctx.bash_background().kill(&task_id, "restored-session");
8413 wait_for_background_exit(pid);
8414 }
8415
8416 #[test]
8417 fn observing_root_again_resets_deleted_sweep_confirmation() {
8418 let (root_dir, root) = test_root("deleted-root-observation-reset");
8419 let ctx = test_ctx();
8420 ctx.mark_subc_unbound();
8421 let executor = Arc::new(Executor::new());
8422 assert!(executor.register_actor(root.clone(), ctx));
8423 root_dir.close().expect("delete project root");
8424
8425 let mut meta = RootMeta::new(Instant::now());
8426 meta.unbound_quiesced = true;
8427 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8428 let pending_binds = HashMap::new();
8429 let root_channels = HashMap::new();
8430 let metrics = DispatchPathMetrics::new();
8431
8432 let first = reap_idle_roots(
8433 Instant::now(),
8434 &mut live_roots,
8435 &pending_binds,
8436 &root_channels,
8437 &executor,
8438 &metrics,
8439 );
8440 assert!(first.forgotten_deleted_roots.is_empty());
8441
8442 std::fs::create_dir_all(root.as_path()).expect("restore project root");
8443 reap_idle_roots(
8444 Instant::now(),
8445 &mut live_roots,
8446 &pending_binds,
8447 &root_channels,
8448 &executor,
8449 &metrics,
8450 );
8451 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8452
8453 let after_reset = reap_idle_roots(
8454 Instant::now(),
8455 &mut live_roots,
8456 &pending_binds,
8457 &root_channels,
8458 &executor,
8459 &metrics,
8460 );
8461 assert!(after_reset.forgotten_deleted_roots.is_empty());
8462 assert!(live_roots.contains_key(&root));
8463 assert!(executor.actor_registered(&root));
8464 }
8465
8466 #[test]
8467 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8468 let (root_dir, root) = test_root("deleted-root-reap");
8469 let app = App::default_shared();
8470 let ctx = Arc::new(AppContext::from_app(
8471 Arc::clone(&app),
8472 Config {
8473 project_root: Some(root.as_path().to_path_buf()),
8474 ..Config::default()
8475 },
8476 ));
8477 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8478 ctx.mark_subc_unbound();
8479 let executor = Arc::new(Executor::new());
8480 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8481 assert_eq!(app.actor_root_count(), 1);
8482 drop(ctx);
8483 root_dir.close().expect("delete project root");
8484
8485 let mut meta = RootMeta::new(Instant::now());
8486 meta.unbound_quiesced = true;
8487 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8488 let outcome = reap_until_forgotten(
8489 &root,
8490 &mut live_roots,
8491 &HashMap::new(),
8492 &HashMap::new(),
8493 &executor,
8494 &DispatchPathMetrics::new(),
8495 );
8496 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8497 assert!(!executor.actor_registered(&root));
8498 assert!(!live_roots.contains_key(&root));
8499 wait_for_actor_root_count(&app, 0);
8500
8501 let status_ctx = AppContext::from_app(app, Config::default());
8502 let status = status_ctx.build_status_snapshot();
8503 assert_eq!(status["runtime"]["live_actor_roots"], 0);
8504 assert_eq!(status["runtime"]["open_routes"], 0);
8505 }
8506
8507 #[test]
8508 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8509 let (root_dir, root) = test_root("deleted-root-reap-census");
8510 let executor = Arc::new(Executor::new());
8511 assert!(executor.register_actor(root.clone(), test_ctx()));
8512 root_dir.close().expect("delete project root");
8513
8514 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8515 let metrics = DispatchPathMetrics::new();
8516 let outcome = reap_idle_roots(
8517 Instant::now(),
8518 &mut live_roots,
8519 &HashMap::new(),
8520 &HashMap::new(),
8521 &executor,
8522 &metrics,
8523 );
8524 assert_eq!(outcome.evicted, 0);
8525
8526 let app = crate::context::App::default_shared();
8527 let health_rollup_cache = HealthRollupCache::new();
8528 health_rollup_cache.refresh(&executor, &app);
8529 let report = build_health_report(
8530 &health_rollup_cache,
8531 &executor,
8532 &HashMap::new(),
8533 &metrics,
8534 &app,
8535 );
8536 let reap = report
8537 .metrics
8538 .as_ref()
8539 .and_then(|metrics| metrics.get("reap"))
8540 .expect("reap health metrics");
8541 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8542 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8543 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8544 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8545 }
8546
8547 #[test]
8548 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8549 let (root_dir, root) = test_root("connection-exit-deleted-root");
8550 let executor = Arc::new(Executor::new());
8551 assert!(executor.register_actor(root.clone(), test_ctx()));
8552
8553 let route = route_key(11, 1);
8554 let mut meta = RootMeta::new(Instant::now());
8555 meta.maintenance_pending = true;
8556 meta.maintenance_queued_kinds
8557 .push_back(MaintenanceDrainKind::CompletionDrains);
8558 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8559 let mut pending_binds = HashMap::new();
8560 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8561 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8562 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8563 let mut route_bash_cancels = HashMap::new();
8564 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8565
8566 quiesce_connection_roots(
8567 &mut live_roots,
8568 &mut pending_binds,
8569 &mut routes,
8570 &mut root_channels,
8571 &mut installed_route_epochs,
8572 &mut route_bash_cancels,
8573 &active_tool_calls,
8574 &executor,
8575 );
8576 assert!(live_roots[&root].unbound_quiesced);
8577 assert!(!live_roots[&root].maintenance_pending);
8578 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8579 assert!(routes.is_empty());
8580 assert!(root_channels.is_empty());
8581
8582 root_dir.close().expect("delete project root");
8583 let metrics = DispatchPathMetrics::new();
8584 let outcome = reap_until_forgotten(
8585 &root,
8586 &mut live_roots,
8587 &pending_binds,
8588 &root_channels,
8589 &executor,
8590 &metrics,
8591 );
8592 let mut session_identity = HashMap::new();
8593 let mut push_buffer = HashMap::new();
8594 let mut bg_subs = HashMap::new();
8595 let mut bg_sub_by_session = HashMap::new();
8596 let mut bg_wake_pending = BgWakePending::new();
8597 let mut bg_wake_epoch = HashMap::new();
8598 let mut pending_bash_asks = HashMap::new();
8599 let mut retry_buffer = HashMap::new();
8600 let mut reclaimed_routes = ReclaimedRoutes::default();
8601 for forgotten in &outcome.forgotten_deleted_roots {
8602 purge_deleted_root_residents(
8603 forgotten,
8604 &mut routes,
8605 &mut root_channels,
8606 &mut installed_route_epochs,
8607 &mut route_bash_cancels,
8608 &active_tool_calls,
8609 executor.as_ref(),
8610 &mut retry_buffer,
8611 &mut reclaimed_routes,
8612 &mut session_identity,
8613 &mut push_buffer,
8614 &mut bg_subs,
8615 &mut bg_sub_by_session,
8616 &mut bg_wake_pending,
8617 &mut bg_wake_epoch,
8618 &mut pending_bash_asks,
8619 &metrics,
8620 );
8621 }
8622
8623 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8624 assert!(!executor.actor_registered(&root));
8625 assert!(!live_roots.contains_key(&root));
8626 }
8627
8628 #[test]
8629 fn unbound_root_quiesces_maintenance_without_removing_actor() {
8630 let (_root_dir, root) = test_root("unbound-root-quiesce");
8631 let ctx = test_ctx();
8632 let executor = Arc::new(Executor::new());
8633 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8634 let mut meta = RootMeta::new(Instant::now());
8635 meta.maintenance_pending = true;
8636 meta.maintenance_jobs_in_flight = 1;
8637 meta.maintenance_queued_kinds
8638 .push_back(MaintenanceDrainKind::ConfigureTail);
8639 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8640 *ctx.search_index()
8642 .write()
8643 .unwrap_or_else(std::sync::PoisonError::into_inner) =
8644 Some(crate::search_index::SearchIndex::new());
8645 ctx.set_cache_writer_capabilities(true, true);
8646 let pending = root.as_path().join("pending.rs");
8647 ctx.add_pending_search_index_paths([pending.clone()]);
8648 let canonical_root = root.as_path().to_path_buf();
8652 let artifact = canonical_root.join("cache.bin");
8653 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8654 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8655 crate::cache_freshness::record_verify_completed(
8656 &canonical_root,
8657 crate::cache_freshness::VerifyArtifact::Search,
8658 seeded_generation,
8659 );
8660 assert!(matches!(
8661 crate::cache_freshness::warm_verify_plan(
8662 &canonical_root,
8663 crate::cache_freshness::VerifyArtifact::Search,
8664 seeded_generation,
8665 ),
8666 crate::cache_freshness::WarmVerifyPlan::Skip
8667 ));
8668 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8671 let _dispatch_tx = dispatch_tx;
8672 let shutdown = Arc::new(AtomicBool::new(false));
8673 let thread_shutdown = Arc::clone(&shutdown);
8674 let join = std::thread::spawn(move || {
8675 while !thread_shutdown.load(Ordering::SeqCst) {
8676 std::thread::yield_now();
8677 }
8678 });
8679 ctx.install_watcher_runtime(
8680 dispatch_rx,
8681 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8682 );
8683 assert!(ctx.watcher_runtime_active());
8684
8685 quiesce_unbound_root(&root, &mut live_roots, &executor);
8686 let meta = &live_roots[&root];
8687 assert!(meta.unbound_quiesced);
8688 assert!(ctx.subc_unbound_quiesced());
8689 assert!(meta.maintenance_pending);
8690 assert!(meta.maintenance_queued_kinds.is_empty());
8691 assert!(executor.actor_registered(&root));
8692 assert!(
8696 ctx.search_index()
8697 .read()
8698 .unwrap_or_else(std::sync::PoisonError::into_inner)
8699 .is_some(),
8700 "quiesce must not evict resident artifacts"
8701 );
8702 assert_eq!(
8703 ctx.pending_callgraph_store_force_token(),
8704 None,
8705 "quiesce must not force a callgraph rebuild"
8706 );
8707 assert_eq!(
8708 ctx.take_pending_search_index_paths(),
8709 vec![pending],
8710 "quiesce must retain pending watcher-derived paths"
8711 );
8712 assert!(
8713 matches!(
8714 crate::cache_freshness::warm_verify_plan(
8715 &canonical_root,
8716 crate::cache_freshness::VerifyArtifact::Search,
8717 seeded_generation,
8718 ),
8719 crate::cache_freshness::WarmVerifyPlan::Skip
8720 ),
8721 "quiesce must not invalidate the warm verify memo"
8722 );
8723 assert!(
8724 ctx.watcher_runtime_active(),
8725 "quiesce must not stop a running watcher"
8726 );
8727 ctx.stop_watcher_runtime();
8728
8729 let meta = live_roots.get_mut(&root).expect("root metadata");
8730 note_maintenance_completion(
8731 meta,
8732 Some(MaintenanceDrainKind::ConfigureTail),
8733 false,
8734 meta.unbound_quiesced,
8735 );
8736 assert!(!meta.maintenance_pending);
8737 assert!(meta.maintenance_queued_kinds.is_empty());
8738 }
8739
8740 #[test]
8741 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8742 let (_dir, root) = test_root("same-root-replacement");
8743 let route = route_key(7, 1);
8744 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8745 let root_channels = HashMap::new();
8746
8747 assert!(!route_removal_will_quiesce_root(
8748 &root,
8749 route,
8750 &installed_channels,
8751 false,
8752 Some(&root),
8753 ));
8754 assert!(route_removal_will_quiesce_root(
8755 &root,
8756 route,
8757 &installed_channels,
8758 false,
8759 None,
8760 ));
8761 assert!(!should_quiesce_removed_root(
8762 &root,
8763 &root_channels,
8764 false,
8765 Some(&root),
8766 ));
8767 assert!(should_quiesce_removed_root(
8768 &root,
8769 &root_channels,
8770 false,
8771 None,
8772 ));
8773 assert!(!should_quiesce_removed_root(
8774 &root,
8775 &root_channels,
8776 true,
8777 None,
8778 ));
8779 }
8780
8781 #[test]
8782 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8783 let (_root_dir, root) = test_root("unbound-root-route-count");
8784 let executor = Arc::new(Executor::new());
8785 assert!(executor.register_actor(root.clone(), test_ctx()));
8786 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8787 let mut root_channels = HashMap::from([(
8788 root.clone(),
8789 HashSet::from([route_key(7, 1), route_key(8, 1)]),
8790 )]);
8791
8792 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8793 if !root_channels.contains_key(&root) {
8794 quiesce_unbound_root(&root, &mut live_roots, &executor);
8795 }
8796 assert!(!live_roots[&root].unbound_quiesced);
8797
8798 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8799 if !root_channels.contains_key(&root) {
8800 quiesce_unbound_root(&root, &mut live_roots, &executor);
8801 }
8802 assert!(live_roots[&root].unbound_quiesced);
8803
8804 live_roots
8805 .get_mut(&root)
8806 .expect("root metadata")
8807 .note_activity();
8808 assert!(
8809 live_roots[&root].unbound_quiesced,
8810 "late asynchronous activity must not reactivate an unbound root"
8811 );
8812
8813 live_roots
8814 .get_mut(&root)
8815 .expect("root metadata")
8816 .reactivate_bound();
8817 assert!(!live_roots[&root].unbound_quiesced);
8818 }
8819
8820 #[test]
8821 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8822 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8823 let (_active_dir, active_root) = test_root("allocator-relief-active");
8824 let now = Instant::now();
8825 let mut live_roots = HashMap::new();
8826 let mut idle = RootMeta::new(now);
8827 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8828 live_roots.insert(idle_root, idle);
8829 let executor = Executor::new();
8830 assert!(process_has_been_idle(now, &live_roots, &executor));
8831
8832 live_roots.insert(active_root.clone(), RootMeta::new(now));
8833 assert!(!process_has_been_idle(now, &live_roots, &executor));
8834
8835 let active = live_roots
8836 .get_mut(&active_root)
8837 .expect("active root metadata");
8838 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8839 active.active_bash_waits = 1;
8840 assert!(!process_has_been_idle(now, &live_roots, &executor));
8841 }
8842
8843 #[test]
8844 fn pressure_relief_log_reports_before_and_after_measurements() {
8845 let allocator = crate::memory::AllocatorMemorySnapshot {
8846 status: "measured",
8847 bytes_in_use: Some(8 * 1024 * 1024),
8848 size_allocated: Some(12 * 1024 * 1024),
8849 retained_slack_bytes: Some(4 * 1024 * 1024),
8850 not_estimated: None,
8851 };
8852 let relief = crate::memory::AllocatorPressureRelief {
8853 bytes_released: 3 * 1024 * 1024,
8854 rss_before_bytes: Some(20 * 1024 * 1024),
8855 rss_after_bytes: Some(17 * 1024 * 1024),
8856 allocator_before: allocator.clone(),
8857 allocator_after: crate::memory::AllocatorMemorySnapshot {
8858 size_allocated: Some(9 * 1024 * 1024),
8859 retained_slack_bytes: Some(1024 * 1024),
8860 ..allocator
8861 },
8862 };
8863 let message = pressure_relief_label(&relief);
8864 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8865 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8866 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8867 assert!(message.contains("reported 3.0 MB released"));
8868 }
8869
8870 #[test]
8871 fn due_maintenance_jobs_skip_poisoned_roots() {
8872 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8873 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8874 let mut live_roots = HashMap::new();
8875 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8876 let mut poisoned_meta = RootMeta::new(Instant::now());
8877 poisoned_meta.maintenance_poisoned = true;
8878 live_roots.insert(poisoned_root.clone(), poisoned_meta);
8879
8880 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8881 &mut live_roots,
8882 MAINTENANCE_SUBMIT_BUDGET,
8883 &HashSet::new(),
8884 );
8885
8886 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8887 assert!(due.iter().all(|(root, _)| root == &healthy_root));
8888 assert!(!deferred);
8889 assert!(live_roots[&healthy_root].maintenance_pending);
8890 assert_eq!(
8891 live_roots[&healthy_root].maintenance_jobs_in_flight,
8892 INITIAL_MAINTENANCE_JOB_COUNT
8893 );
8894 assert!(!live_roots[&poisoned_root].maintenance_pending);
8895 }
8896
8897 #[test]
8898 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8899 let (_dir, root) = test_root("maintenance-unbound");
8900 let mut meta = RootMeta::new(Instant::now());
8901 meta.unbound_quiesced = true;
8902 let mut live_roots = HashMap::from([(root.clone(), meta)]);
8903
8904 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8905 &mut live_roots,
8906 MAINTENANCE_SUBMIT_BUDGET,
8907 &HashSet::new(),
8908 );
8909
8910 assert!(due.is_empty());
8911 assert!(!deferred);
8912 assert!(!live_roots[&root].maintenance_pending);
8913 }
8914
8915 #[test]
8916 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8917 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8918 let ctx = test_ctx();
8919 assert!(!ctx.completion_drains_have_work());
8920
8921 let executor = Executor::new();
8922 assert!(executor.register_actor(root.clone(), ctx));
8923 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8924 let session = "idle-session".to_string();
8925 let channel = route_key(17, 1);
8926 let metrics = DispatchPathMetrics::new();
8927 let bg_sub_by_session =
8928 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8929 let mut bg_wake_pending = BgWakePending::new();
8930
8931 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8932 &mut live_roots,
8933 Some(&executor),
8934 &bg_sub_by_session,
8935 &bg_wake_pending,
8936 MAINTENANCE_SUBMIT_BUDGET,
8937 &HashSet::new(),
8938 );
8939 assert!(idle_tick_jobs.is_empty());
8940 assert!(!deferred);
8941 assert!(!live_roots[&root].maintenance_pending);
8942
8943 let mut bg_wake_epoch = HashMap::new();
8946 push::arm_bg_wake(
8947 root.clone(),
8948 session,
8949 channel,
8950 &mut bg_wake_pending,
8951 &mut bg_wake_epoch,
8952 &metrics,
8953 );
8954 let (next_tick_jobs, deferred) = due_maintenance_jobs(
8955 &mut live_roots,
8956 Some(&executor),
8957 &bg_sub_by_session,
8958 &bg_wake_pending,
8959 MAINTENANCE_SUBMIT_BUDGET,
8960 &HashSet::new(),
8961 );
8962 assert_eq!(
8963 next_tick_jobs,
8964 vec![(root, MaintenanceDrainKind::CompletionDrains)]
8965 );
8966 assert!(!deferred);
8967 }
8968
8969 async fn assert_slow_configure_tail_admission(
8970 config: crate::executor::ExecutorConfig,
8971 shape: &'static str,
8972 ) {
8973 let root_dir = tempfile::tempdir().unwrap();
8974 let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8975 let root = ProjectRootId::from_path(&root_path).unwrap();
8976 let ctx = test_ctx();
8977 ctx.mark_subc_bound();
8978 let storage_root = ctx.storage_dir();
8979 ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8980 generation: ctx.configure_generation(),
8981 root_path: root_path.clone(),
8982 canonical_cache_root: root_path.clone(),
8983 harness: crate::harness::Harness::Opencode,
8984 storage_root: storage_root.clone(),
8985 harness_dir: storage_root.join("opencode"),
8986 session_id: "first-search-admission".to_string(),
8987 home_match: false,
8988 format_tool_cache_clear_needed: false,
8989 run_bash_replay: false,
8990 refresh_project_runtime: false,
8991 sync_bash_compress_flag: false,
8992 reset_filter_registry: false,
8993 clear_failed_spawns: false,
8994 warm_callgraph_store: false,
8995 supersede_search_artifact_persistence: false,
8996 supersede_callgraph_artifact_persistence: false,
8997 supersede_semantic_artifact_persistence: false,
8998 search_artifact_load_start: None,
8999 semantic_artifact_load_start: None,
9000 })
9001 .expect("queue configure maintenance");
9002 let (_gate, maintenance_reached, release_maintenance) =
9003 crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9004 root_path.clone(),
9005 );
9006
9007 let expected_pool_size = config.pool_size;
9008 let expected_actor_cap = config.actor_cap;
9009 let executor = Arc::new(Executor::with_config(config));
9010 assert_eq!(executor.pool_size(), expected_pool_size);
9011 assert_eq!(executor.actor_cap(), expected_actor_cap);
9012 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9013 let metrics = Arc::new(DispatchPathMetrics::new());
9014 let (completion_tx, mut completion_rx) = mpsc::channel(2);
9015 submit_maintenance_job(
9016 &executor,
9017 root.clone(),
9018 MaintenanceDrainKind::ConfigureTail,
9019 Vec::new(),
9020 &completion_tx,
9021 &metrics,
9022 );
9023 maintenance_reached
9024 .recv_timeout(Duration::from_secs(2))
9025 .expect("configure tail reached gate");
9026
9027 let admission_started = std::time::Instant::now();
9028 let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9029 let search = executor.submit_async(
9030 root.clone(),
9031 Lane::HeavyInit,
9032 "first-search".to_string(),
9033 Box::new(move |_ctx| {
9034 search_admitted_tx
9035 .send(())
9036 .expect("signal search admission");
9037 Response::success("first-search", json!({}))
9038 }),
9039 );
9040 let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9041 let mutation = executor.submit_async(
9042 root,
9043 Lane::Mutating,
9044 "queued-mutation".to_string(),
9045 Box::new(move |_ctx| {
9046 mutation_started_tx.send(()).expect("signal mutation start");
9047 Response::success("queued-mutation", json!({}))
9048 }),
9049 );
9050
9051 let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9057 let admission_elapsed = admission_started.elapsed();
9058 let mutation_waited = mutation_started_rx.try_recv().is_err();
9059 release_maintenance
9060 .send(())
9061 .expect("release configure maintenance");
9062 search_admission
9063 .expect("first search must admit while configure maintenance remains gated");
9064 eprintln!(
9065 "first-search admission while configure maintenance is gated ({shape}): {}ms",
9066 admission_elapsed.as_millis()
9067 );
9068 assert!(
9069 mutation_waited,
9070 "mutating work must wait for configure maintenance to release its read epoch"
9071 );
9072 tokio::time::timeout(Duration::from_secs(5), search)
9073 .await
9074 .expect("first search completion timed out")
9075 .expect("first search completion channel closed");
9076 tokio::time::timeout(Duration::from_secs(5), mutation)
9077 .await
9078 .expect("mutation completion timed out")
9079 .expect("mutation completion channel closed");
9080 tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9081 .await
9082 .expect("configure-tail completion timed out")
9083 .expect("configure-tail completion channel closed");
9084 }
9085
9086 #[tokio::test]
9087 async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9088 assert_slow_configure_tail_admission(
9089 crate::executor::ExecutorConfig {
9090 pool_size: 2,
9091 read_cap: 1,
9092 actor_cap: 1,
9093 heavy_permits: 1,
9094 drr_quantum: 1,
9095 },
9096 "pool=2 actor_cap=1",
9097 )
9098 .await;
9099 assert_slow_configure_tail_admission(
9100 crate::executor::ExecutorConfig {
9101 pool_size: 4,
9102 read_cap: 3,
9103 actor_cap: 3,
9104 heavy_permits: 3,
9105 drr_quantum: 1,
9106 },
9107 "pool=4 actor_cap=3",
9108 )
9109 .await;
9110 }
9111
9112 #[tokio::test]
9113 async fn subc_configure_tail_precedes_completed_search_install() {
9114 let root_dir = tempfile::tempdir().unwrap();
9115 let storage = tempfile::tempdir().unwrap();
9116 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9117 let (ctx, ignored_path) =
9118 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9119 let ctx = Arc::new(ctx);
9120 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9121 &ctx,
9122 &ignored_path
9123 ));
9124
9125 let executor = Arc::new(Executor::new());
9126 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9127 let metrics = Arc::new(DispatchPathMetrics::new());
9128 let (completion_tx, mut completion_rx) = mpsc::channel(4);
9129 submit_maintenance_job(
9130 &executor,
9131 root.clone(),
9132 MaintenanceDrainKind::ConfigureTail,
9133 Vec::new(),
9134 &completion_tx,
9135 &metrics,
9136 );
9137 submit_maintenance_job(
9138 &executor,
9139 root,
9140 MaintenanceDrainKind::CompletionDrains,
9141 Vec::new(),
9142 &completion_tx,
9143 &metrics,
9144 );
9145
9146 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9147 .await
9148 .expect("configure-tail completion timed out")
9149 .expect("configure-tail completion channel closed");
9150 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9151 .await
9152 .expect("completion-drains completion timed out")
9153 .expect("completion-drains completion channel closed");
9154 assert!(first.response.id.contains("configure-tail"));
9155 assert!(second.response.id.contains("completion-drains"));
9156 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9157 &ctx,
9158 &ignored_path
9159 ));
9160 assert_eq!(
9161 ctx.search_index()
9162 .read()
9163 .unwrap_or_else(std::sync::PoisonError::into_inner)
9164 .as_ref()
9165 .expect("completed search index installed")
9166 .file_count(),
9167 0,
9168 "configure must install the ignore matcher before pending paths replay"
9169 );
9170 ctx.stop_watcher_runtime();
9171 }
9172
9173 #[test]
9174 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9175 let (_dir, root) = test_root("maintenance-post-bind");
9176 let mut live_roots = HashMap::new();
9177 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9178
9179 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9180 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9181
9182 let meta = live_roots.get(&root).expect("root metadata");
9183 assert!(meta.maintenance_pending);
9184 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9185 assert_eq!(
9186 meta.maintenance_queued_kinds
9187 .iter()
9188 .copied()
9189 .collect::<Vec<_>>(),
9190 vec![
9191 MaintenanceDrainKind::ConfigureTail,
9192 MaintenanceDrainKind::CompletionDrains,
9193 ]
9194 );
9195
9196 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9197 &mut live_roots,
9198 MAINTENANCE_SUBMIT_BUDGET,
9199 &HashSet::new(),
9200 );
9201
9202 assert_eq!(
9203 due,
9204 vec![
9205 (root.clone(), MaintenanceDrainKind::ConfigureTail),
9206 (root.clone(), MaintenanceDrainKind::CompletionDrains),
9207 ]
9208 );
9209 assert!(!deferred);
9210 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9211 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9212 }
9213
9214 #[test]
9215 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9216 let mut live_roots = HashMap::new();
9217 let mut root_ids = Vec::new();
9218 let mut _dirs = Vec::new();
9219 for index in 0..4 {
9220 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9221 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9222 root_ids.push(root_id);
9223 _dirs.push(dir);
9224 }
9225
9226 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9227 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9228 &mut live_roots,
9229 small_budget,
9230 &HashSet::new(),
9231 );
9232
9233 assert_eq!(first_due.len(), small_budget);
9234 assert!(first_deferred);
9235 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9236 assert!(first_due_set
9237 .iter()
9238 .all(|root| live_roots[root].maintenance_pending));
9239 assert!(first_due_set
9240 .iter()
9241 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9242
9243 let all_roots: HashSet<_> = root_ids.into_iter().collect();
9244 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9245 assert!(deferred_roots
9246 .iter()
9247 .all(|root| !live_roots[root].maintenance_pending));
9248 }
9249
9250 #[test]
9251 fn due_maintenance_jobs_defers_pending_bind_roots() {
9252 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9253 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9254 let mut live_roots = HashMap::new();
9255 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9256 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9257 let pending_bind_roots = HashSet::from([bind_root.clone()]);
9258
9259 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9260 &mut live_roots,
9261 usize::MAX,
9262 &pending_bind_roots,
9263 );
9264
9265 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9266 assert!(due.iter().all(|(root, _)| root == &healthy_root));
9267 assert!(!deferred);
9268 assert!(!live_roots[&bind_root].maintenance_pending);
9269 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9270 }
9271
9272 #[test]
9273 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9274 let (_dir, root) = test_root("maintenance-requeue");
9275 let mut live_roots = HashMap::new();
9276 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9277 let (due, deferred) = due_maintenance_jobs_without_actor_context(
9278 &mut live_roots,
9279 usize::MAX,
9280 &HashSet::new(),
9281 );
9282 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9283 assert!(due.iter().all(|(due_root, _)| due_root == &root));
9284 assert!(!deferred);
9285
9286 let meta = live_roots.get_mut(&root).unwrap();
9287 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9288 assert!(meta.maintenance_pending);
9289 assert_eq!(
9290 meta.maintenance_jobs_in_flight,
9291 INITIAL_MAINTENANCE_JOB_COUNT - 1
9292 );
9293 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9294
9295 let (requeued, deferred) =
9296 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9297 assert_eq!(
9298 requeued,
9299 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9300 );
9301 assert!(!deferred);
9302 let meta = live_roots.get_mut(&root).unwrap();
9303 assert_eq!(
9304 meta.maintenance_jobs_in_flight,
9305 INITIAL_MAINTENANCE_JOB_COUNT
9306 );
9307 assert!(meta.maintenance_queued_kinds.is_empty());
9308
9309 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9310 note_maintenance_completion(meta, None, false, false);
9311 }
9312 assert!(!meta.maintenance_pending);
9313 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9314 }
9315
9316 #[test]
9317 fn maintenance_requeue_drops_while_bind_is_pending() {
9318 let (_dir, root) = test_root("maintenance-bind-requeue");
9319 let mut live_roots = HashMap::new();
9320 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9321 let (due, _) = due_maintenance_jobs_without_actor_context(
9322 &mut live_roots,
9323 usize::MAX,
9324 &HashSet::new(),
9325 );
9326 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9327
9328 let meta = live_roots.get_mut(&root).unwrap();
9329 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9330
9331 assert_eq!(
9332 meta.maintenance_jobs_in_flight,
9333 INITIAL_MAINTENANCE_JOB_COUNT - 1
9334 );
9335 assert!(meta.maintenance_queued_kinds.is_empty());
9336 assert!(meta.maintenance_pending);
9337 }
9338
9339 #[test]
9340 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9341 let mut meta = RootMeta::new(Instant::now());
9342 meta.unbound_quiesced = true;
9343
9344 assert!(!should_requiesce_after_maintenance(
9345 &meta,
9346 MaintenanceDrainKind::Lsp,
9347 false,
9348 ));
9349 assert!(!should_requiesce_after_maintenance(
9350 &meta,
9351 MaintenanceDrainKind::ConfigureTail,
9352 true,
9353 ));
9354 assert!(should_requiesce_after_maintenance(
9355 &meta,
9356 MaintenanceDrainKind::ConfigureTail,
9357 false,
9358 ));
9359 }
9360
9361 #[test]
9362 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9363 let (_dir, root) = test_root("maintenance-fatal");
9364 let mut live_roots = HashMap::new();
9365 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9366 let (due, _) = due_maintenance_jobs_without_actor_context(
9367 &mut live_roots,
9368 usize::MAX,
9369 &HashSet::new(),
9370 );
9371 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9372
9373 let meta = live_roots.get_mut(&root).unwrap();
9374 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9375 assert!(meta.maintenance_poisoned);
9376 assert!(meta.maintenance_queued_kinds.is_empty());
9377
9378 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9379 note_maintenance_completion(meta, None, false, false);
9380 }
9381 assert!(!meta.maintenance_pending);
9382 assert_eq!(meta.maintenance_jobs_in_flight, 0);
9383 }
9384
9385 #[test]
9386 fn trust_for_principal_matrix() {
9387 assert_eq!(
9388 trust_for_principal(&Some(Principal::Direct)),
9389 BindTrust::FirstParty
9390 );
9391 for module_id in [
9398 "llm-runner",
9399 "aft",
9400 "broca",
9401 "alfonso-core",
9402 "prefrontal",
9403 "prefrontal-core",
9404 ] {
9405 assert_eq!(
9406 trust_for_principal(&Some(Principal::Reserved {
9407 module_id: module_id.to_string(),
9408 })),
9409 BindTrust::FirstParty,
9410 "reserved module id '{module_id}' must resolve to first-party trust"
9411 );
9412 }
9413 assert_eq!(
9414 trust_for_principal(&Some(Principal::Reserved {
9415 module_id: "subc-mcp".to_string(),
9416 })),
9417 BindTrust::Untrusted
9418 );
9419 assert_eq!(
9420 trust_for_principal(&Some(Principal::Reserved {
9421 module_id: "anything-unknown".to_string(),
9422 })),
9423 BindTrust::Untrusted
9424 );
9425 assert_eq!(
9426 trust_for_principal(&Some(Principal::Unverified)),
9427 BindTrust::Untrusted
9428 );
9429 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9430 }
9431
9432 #[test]
9433 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9434 let principal = Some(Principal::Direct);
9435 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9436 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9437
9438 assert_eq!(
9439 trust_for_bind(fingerprint_a, &principal),
9440 BindTrust::Untrusted
9441 );
9442 assert_eq!(
9443 trust_for_bind(fingerprint_b, &principal),
9444 BindTrust::Untrusted
9445 );
9446 }
9447
9448 #[test]
9455 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9456 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9457 assert_eq!(
9458 trust_for_bind(harness, &Some(Principal::Direct)),
9459 BindTrust::FirstParty,
9460 "a direct principal must stay first-party on {harness}"
9461 );
9462 assert_eq!(
9463 trust_for_bind(harness, &Some(Principal::Unverified)),
9464 BindTrust::Untrusted,
9465 "an unverified principal must stay untrusted on {harness}"
9466 );
9467 assert_eq!(
9468 trust_for_bind(harness, &None),
9469 BindTrust::Untrusted,
9470 "an absent principal must fail closed on {harness}"
9471 );
9472 assert_eq!(
9473 trust_for_bind(
9474 harness,
9475 &Some(Principal::Reserved {
9476 module_id: "subc-mcp".to_string(),
9477 })
9478 ),
9479 BindTrust::Untrusted,
9480 "a non-allowlisted reserved module must stay untrusted on {harness}"
9481 );
9482 }
9483 }
9484
9485 #[tokio::test]
9486 async fn persistent_cancel_resolves_when_fired_before_await() {
9487 let signal = PersistentCancelSignal::new();
9491 signal.cancel();
9492 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9494 .await
9495 .expect("cancelled() must resolve when cancel fired beforehand");
9496
9497 let racing = PersistentCancelSignal::new();
9499 let racing_for_task = racing.clone();
9500 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9501 racing.cancel();
9502 tokio::time::timeout(Duration::from_secs(1), waiter)
9503 .await
9504 .expect("cancelled() must resolve when cancel races the await")
9505 .expect("waiter task panicked");
9506 }
9507
9508 #[test]
9509 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9510 let installed = HashMap::from([(7, 9)]);
9511 let mut reclaimed = ReclaimedRoutes::default();
9512 reclaimed.insert(route_key(8, 1));
9513 for ty in [
9514 FrameType::Request,
9515 FrameType::Response,
9516 FrameType::Error,
9517 FrameType::Push,
9518 FrameType::Cancel,
9519 FrameType::Goodbye,
9520 ] {
9521 let body = if ty.is_pure_header() {
9522 Vec::new()
9523 } else {
9524 br#"{}"#.to_vec()
9525 };
9526 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9527 assert!(
9528 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9529 "{ty:?}"
9530 );
9531 }
9532
9533 let reclaimed_request = Frame::build(
9534 FrameType::Request,
9535 control_flags(),
9536 8,
9537 1,
9538 42,
9539 br#"{}"#.to_vec(),
9540 )
9541 .unwrap();
9542 assert!(ingress_route_should_be_processed(
9543 &installed,
9544 &reclaimed,
9545 &reclaimed_request
9546 ));
9547
9548 let never_installed = Frame::build(
9549 FrameType::Request,
9550 control_flags(),
9551 9,
9552 1,
9553 43,
9554 br#"{}"#.to_vec(),
9555 )
9556 .unwrap();
9557 assert!(!ingress_route_should_be_processed(
9558 &installed,
9559 &reclaimed,
9560 &never_installed
9561 ));
9562
9563 let current = Frame::build(
9564 FrameType::Request,
9565 control_flags(),
9566 7,
9567 9,
9568 43,
9569 br#"{}"#.to_vec(),
9570 )
9571 .unwrap();
9572 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9573 assert!(ingress_route_should_be_processed(
9574 &installed, &reclaimed, ¤t
9575 ));
9576 assert!(ingress_route_should_be_processed(
9577 &installed, &reclaimed, &control
9578 ));
9579 assert_eq!(installed, HashMap::from([(7, 9)]));
9580 }
9581
9582 #[tokio::test]
9583 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9584 let (_dir, root) = test_root("route-bind-b2-ordering");
9585 let route = route_key(7, 3);
9586 let identity = RouteIdentity(Arc::new(RouteIdentityData {
9587 root: root.clone(),
9588 project_root: root.as_path().to_path_buf(),
9589 harness: "opencode".to_string(),
9590 session: "b2-session".to_string(),
9591 trust: BindTrust::FirstParty,
9592 spawn_principal: AuthenticatedPrincipal::FirstParty,
9593 consumer_elicitation_capable: false,
9594 }));
9595 let replay_key = push::ReplayKey::from_identity(&identity);
9596 let completion = RouteBindCompletion {
9597 route,
9598 identity,
9599 bind_root_id: root.clone(),
9600 inserted_new_actor: false,
9601 configure_response: Response::success("subc-bind-7", json!({})),
9602 diagnostics_on_edit: false,
9603 ver: PROTOCOL_VERSION,
9604 corr: 91,
9605 flags: control_flags(),
9606 };
9607 let mut pending_binds = HashMap::from([(
9608 route,
9609 PendingBind {
9610 bind_root_id: root,
9611 inserted_new_actor: false,
9612 cancelled: false,
9613 configure_request_id: "subc-bind-7".to_string(),
9614 started_at: Instant::now(),
9615 warned_half_deadline: false,
9616 deadline_reported: false,
9617 corr: 91,
9618 ver: PROTOCOL_VERSION,
9619 flags: control_flags(),
9620 cancellation: crate::executor::JobCancellation::new(),
9621 },
9622 )]);
9623 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9624 let mut push_buffer =
9625 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9626 let (writer_tx, mut writer_rx) = mpsc::channel(8);
9627 let metrics = Arc::new(DispatchPathMetrics::new());
9628 let executor = Arc::new(Executor::new());
9629 let standing_actor =
9630 standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9631
9632 handle_route_bind_completion(
9633 &writer_tx,
9634 completion,
9635 &mut HashMap::new(),
9636 &mut HashMap::new(),
9637 &mut HashMap::new(),
9638 &mut push_buffer,
9639 &mut HashMap::new(),
9640 &mut pending_binds,
9641 &mut installed_route_epochs,
9642 &executor,
9643 &standing_actor,
9644 &Arc::new(Notify::new()),
9645 &metrics,
9646 None,
9647 )
9648 .await
9649 .unwrap();
9650
9651 let ack = writer_rx.try_recv().expect("RouteBindAck");
9652 assert_eq!(ack.header.ty, FrameType::Response);
9653 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9654 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9655 assert_eq!(route_frame.header.ty, FrameType::Push);
9656 assert_eq!(
9657 (route_frame.header.channel, route_frame.header.epoch),
9658 (route.channel, route.epoch)
9659 );
9660 }
9661}