1use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38 finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39 PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45 Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
46 ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
47};
48use subc_protocol::session::{
49 HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
50 MODULE_CONTROL_OP_HEALTH_CHECK,
51};
52use subc_protocol::{
53 ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
54 PROTOCOL_VERSION,
55};
56use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
57use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
58use tokio::net::TcpStream;
59use tokio::sync::{mpsc, oneshot, Notify};
60use tokio::task::JoinHandle;
61
62const AUTH_DEADLINE: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
66const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
67const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
68const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
69
70const HELLO_CORR: u64 = 1;
72
73const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
76
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
81
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
86
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
91
92const WRITER_QUEUE_CAPACITY: usize = 256;
93
94const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
97
98const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
106const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
107 MaintenanceDrainKind::Watcher,
108 MaintenanceDrainKind::Lsp,
109 MaintenanceDrainKind::ConfigureTail,
110 MaintenanceDrainKind::CompletionDrains,
111];
112#[cfg(test)]
113const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
114
115const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
116const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
117
118const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
119const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
120
121const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
124
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
129
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
132const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
133
134#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
135struct RouteChannel {
136 channel: u16,
137 epoch: u32,
138}
139
140impl fmt::Display for RouteChannel {
141 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142 write!(f, "{}@{}", self.channel, self.epoch)
143 }
144}
145
146type PushEnvelope = (ProjectRootId, PushFrame);
147type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
148type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
149mod bash;
150mod health;
151mod manifest;
152mod push;
153mod standing;
154mod wire;
155
156use self::health::{
157 build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
158 DispatchPathMetrics, HealthRollupCache, ReapBlockerCensus, ResponseTaskGuard,
159 HEALTH_ROLLUP_TTL,
160};
161use self::manifest::{
162 build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163 is_subc_agent_core_tool, is_subc_native_plumbing_tool,
164};
165pub use self::wire::SubcError;
166
167pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
171 manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
172}
173use self::wire::{
174 build_error_frame, build_goodbye_frame, build_tool_response_frame,
175 build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
176 response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
177 send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
178};
179
180struct DecodedFrame {
181 frame: Frame,
182 phase_trace: PhaseTrace,
183}
184
185struct ToolCallCompletion {
186 text: String,
187 phase_trace: PhaseTrace,
188}
189
190#[derive(Clone, Copy, Debug, PartialEq, Eq)]
191enum RouteDetachPolicy {
192 RetainForReplay,
193 CancelOnDetach,
194}
195
196#[derive(Clone)]
197struct ActiveToolCall {
198 root_id: ProjectRootId,
199 cancellation: JobCancellation,
200 detach_policy: RouteDetachPolicy,
201}
202
203type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
204
205struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
206
207impl PendingDeferredSetupGuard {
208 fn new(count: Arc<AtomicUsize>) -> Self {
209 count.fetch_add(1, Ordering::SeqCst);
210 Self(count)
211 }
212}
213
214impl Drop for PendingDeferredSetupGuard {
215 fn drop(&mut self) {
216 self.0.fetch_sub(1, Ordering::SeqCst);
217 }
218}
219
220enum DeferredSetupOutcome {
221 Immediate {
222 text: String,
223 phase_trace: PhaseTrace,
224 },
225 Deferred {
226 pending: PendingResponse,
227 surface_downgraded: bool,
228 phase_trace: PhaseTrace,
229 },
230}
231
232struct PendingSubcResponse {
233 route: RouteChannel,
234 corr: u64,
235 flags: Flags,
236 ver: u8,
237 root: ProjectRootId,
238 session_id: String,
239 bare_name: String,
240 format_context: crate::subc_format::FormatContext,
241 bind_trust: BindTrust,
242 pending: PendingResponse,
243 surface_downgraded: bool,
244 phase_trace: PhaseTrace,
245}
246
247struct ResolvedSubcResponse {
248 entry: PendingSubcResponse,
249 response: Response,
250}
251
252#[derive(Default)]
253struct PendingSubcResponses {
254 entries: Vec<PendingSubcResponse>,
255}
256
257impl PendingSubcResponses {
258 fn register(&mut self, pending: PendingSubcResponse) {
259 self.entries.retain(|entry| {
260 let keep = entry.route != pending.route || entry.corr != pending.corr;
261 if !keep {
262 if let Some(cancellation) = &entry.pending.cancellation {
263 cancellation.request_cancel();
264 }
265 }
266 keep
267 });
268 self.entries.push(pending);
269 }
270
271 fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
272 let mut ready = Vec::new();
273 let mut waiting = Vec::with_capacity(self.entries.len());
274 for mut entry in self.entries.drain(..) {
275 let response = executor
276 .actor_context(&entry.root)
277 .and_then(|ctx| (entry.pending.poll)(&ctx));
278 if let Some(response) = response {
279 ready.push(ResolvedSubcResponse { entry, response });
280 } else {
281 waiting.push(entry);
282 }
283 }
284 self.entries = waiting;
285 ready
286 }
287
288 fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
289 let mut cancelled = false;
290 self.entries.retain(|entry| {
291 let keep = entry.route != route || entry.corr != corr;
292 if !keep {
293 cancelled = true;
294 if let Some(cancellation) = &entry.pending.cancellation {
295 cancellation.request_cancel();
296 }
297 }
298 keep
299 });
300 cancelled
301 }
302
303 fn drain_route(
304 &mut self,
305 route: RouteChannel,
306 executor: &Executor,
307 ) -> Vec<ResolvedSubcResponse> {
308 self.drain_matching(executor, |entry| entry.route == route)
309 }
310
311 fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
312 self.drain_matching(executor, |_| true)
313 }
314
315 fn drain_matching(
316 &mut self,
317 executor: &Executor,
318 matches: impl Fn(&PendingSubcResponse) -> bool,
319 ) -> Vec<ResolvedSubcResponse> {
320 let mut resolved = Vec::new();
321 let mut waiting = Vec::with_capacity(self.entries.len());
322 for mut entry in self.entries.drain(..) {
323 if !matches(&entry) {
324 waiting.push(entry);
325 continue;
326 }
327 if let Some(cancellation) = &entry.pending.cancellation {
328 cancellation.request_cancel();
329 }
330 if let Some(ctx) = executor.actor_context(&entry.root) {
331 if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
332 let response = on_shutdown(&ctx);
333 resolved.push(ResolvedSubcResponse { entry, response });
334 }
335 }
336 }
337 self.entries = waiting;
338 resolved
339 }
340
341 fn is_empty(&self) -> bool {
342 self.entries.is_empty()
343 }
344}
345
346#[derive(Clone)]
347struct PushSenders {
348 lossy_tx: mpsc::Sender<LossyPushEnvelope>,
349 reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
350 lossy_overflow: Arc<push::LossyOverflow>,
351 lossy_seq: Arc<AtomicU64>,
352 fleet_status_client: FleetStatusClient,
353}
354
355#[derive(Clone)]
356struct PersistentCancelSignal {
357 inner: Arc<PersistentCancelInner>,
358}
359
360struct PersistentCancelInner {
361 cancelled: AtomicBool,
362 notify: Notify,
363}
364
365impl PersistentCancelSignal {
366 fn new() -> Self {
367 Self {
368 inner: Arc::new(PersistentCancelInner {
369 cancelled: AtomicBool::new(false),
370 notify: Notify::new(),
371 }),
372 }
373 }
374
375 fn cancel(&self) {
376 if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
377 self.inner.notify.notify_waiters();
378 }
379 }
380
381 fn is_cancelled(&self) -> bool {
382 self.inner.cancelled.load(Ordering::SeqCst)
383 }
384
385 async fn cancelled(&self) {
386 loop {
394 let notified = self.inner.notify.notified();
395 tokio::pin!(notified);
396 notified.as_mut().enable();
397 if self.is_cancelled() {
398 return;
399 }
400 notified.await;
401 }
402 }
403}
404
405fn submit_active_tool_call(
406 executor: &Executor,
407 active: &ActiveToolCalls,
408 route: RouteChannel,
409 corr: u64,
410 root_id: ProjectRootId,
411 lane: Lane,
412 request_id: String,
413 detach_policy: RouteDetachPolicy,
414 job: crate::executor::ExecutorJob,
415) -> oneshot::Receiver<Response> {
416 let (rx, cancellation) =
417 executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
418 active
419 .lock()
420 .unwrap_or_else(std::sync::PoisonError::into_inner)
421 .insert(
422 (route, corr),
423 ActiveToolCall {
424 root_id,
425 cancellation,
426 detach_policy,
427 },
428 );
429 rx
430}
431
432fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
433 active
434 .lock()
435 .unwrap_or_else(std::sync::PoisonError::into_inner)
436 .remove(&(route, corr))
437 .is_some()
438}
439
440fn active_tool_call_is_registered(
441 active: &ActiveToolCalls,
442 route: RouteChannel,
443 corr: u64,
444) -> bool {
445 active
446 .lock()
447 .unwrap_or_else(std::sync::PoisonError::into_inner)
448 .contains_key(&(route, corr))
449}
450
451fn cancel_active_tool_call(
452 active: &ActiveToolCalls,
453 executor: &Executor,
454 route: RouteChannel,
455 corr: u64,
456 reason: &str,
457) -> bool {
458 let call = active
459 .lock()
460 .unwrap_or_else(std::sync::PoisonError::into_inner)
461 .remove(&(route, corr));
462 let Some(call) = call else {
463 return false;
464 };
465 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
466 log::debug!(
467 "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
468 );
469 true
470}
471
472#[derive(Clone, Copy, Debug, PartialEq, Eq)]
473enum RouteWorkDisposition {
474 RetainForReplay,
475 Abandon,
476}
477
478fn apply_route_work_disposition(
479 active: &ActiveToolCalls,
480 executor: &Executor,
481 route: RouteChannel,
482 disposition: RouteWorkDisposition,
483 reason: &str,
484) -> usize {
485 if disposition == RouteWorkDisposition::RetainForReplay {
486 let (retained, cancelled) = {
487 let mut calls = active
488 .lock()
489 .unwrap_or_else(std::sync::PoisonError::into_inner);
490 let mut retained = 0usize;
491 let mut cancelled = Vec::new();
492 calls.retain(|(call_route, _), call| {
493 if *call_route != route {
494 return true;
495 }
496 match call.detach_policy {
497 RouteDetachPolicy::RetainForReplay => {
498 retained += 1;
499 true
500 }
501 RouteDetachPolicy::CancelOnDetach => {
502 cancelled.push(call.clone());
503 false
504 }
505 }
506 });
507 (retained, cancelled)
508 };
509 let cancelled_count = cancelled.len();
510 for call in cancelled {
511 executor.cancel_job(&call.root_id, &call.cancellation);
512 }
513 log::debug!(
514 "subc attach: retained {retained} replayable active tool call(s) and cancelled {cancelled_count} teardown-terminal call(s) route={route} reason={reason}"
515 );
516 return retained;
517 }
518
519 let cancelled = {
520 let mut calls = active
521 .lock()
522 .unwrap_or_else(std::sync::PoisonError::into_inner);
523 let mut cancelled = Vec::new();
524 calls.retain(|(call_route, _), call| {
525 if *call_route == route {
526 cancelled.push(call.clone());
527 false
528 } else {
529 true
530 }
531 });
532 cancelled
533 };
534 for call in &cancelled {
535 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
536 log::debug!(
537 "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
538 );
539 }
540 cancelled.len()
541}
542
543fn cancel_all_active_tool_calls(
544 active: &ActiveToolCalls,
545 executor: &Executor,
546 reason: &str,
547) -> usize {
548 let cancelled = {
549 let mut calls = active
550 .lock()
551 .unwrap_or_else(std::sync::PoisonError::into_inner);
552 calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
553 };
554 for call in &cancelled {
555 let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
556 log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
557 }
558 cancelled.len()
559}
560
561#[derive(Debug, Clone, Copy, PartialEq, Eq)]
562pub(crate) enum BindTrust {
563 FirstParty,
564 Untrusted,
565}
566
567impl BindTrust {
568 fn allows_bash_observation(self) -> bool {
569 matches!(self, Self::FirstParty)
570 }
571
572 fn label(self) -> &'static str {
573 match self {
574 Self::FirstParty => "first_party",
575 Self::Untrusted => "untrusted",
576 }
577 }
578
579 fn sandbox_trust(self) -> PrincipalTrust {
580 match self {
581 Self::FirstParty => PrincipalTrust::FirstParty,
582 Self::Untrusted => PrincipalTrust::Untrusted,
583 }
584 }
585}
586
587pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
588 match principal {
589 Some(Principal::Direct) => BindTrust::FirstParty,
590 Some(Principal::Reserved { module_id })
601 if module_id == "llm-runner"
602 || module_id == "aft"
603 || module_id == "broca"
604 || module_id == "alfonso-core"
605 || module_id == "prefrontal"
606 || module_id == "prefrontal-core" =>
607 {
608 BindTrust::FirstParty
609 }
610 Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
611 BindTrust::Untrusted
612 }
613 }
614}
615
616fn harness_forces_untrusted(harness: &str) -> bool {
617 harness.starts_with("fed:")
618}
619
620pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
621 if harness_forces_untrusted(harness) {
622 BindTrust::Untrusted
623 } else {
624 trust_for_principal(principal)
625 }
626}
627
628fn principal_id(principal: &Option<Principal>) -> Option<String> {
629 match principal {
630 Some(Principal::Direct) => Some("direct".to_string()),
631 Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
632 Some(Principal::Unverified) => Some("unverified".to_string()),
633 None => None,
634 }
635}
636
637fn principal_label(principal: &Option<Principal>) -> String {
638 principal_id(principal).unwrap_or_else(|| "absent".to_string())
639}
640
641#[derive(Debug)]
642struct RootMeta {
648 maintenance_pending: bool,
649 maintenance_jobs_in_flight: usize,
650 maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
651 maintenance_last_submitted: Option<Instant>,
652 maintenance_poisoned: bool,
653 last_touched: Instant,
654 diagnostics_on_edit: bool,
655 active_bash_waits: usize,
656 idle_artifacts_evicted: bool,
657 unbound_quiesced: bool,
658 consecutive_missing_sweeps: u8,
659}
660
661#[derive(Debug)]
662struct PendingBind {
663 bind_root_id: ProjectRootId,
664 inserted_new_actor: bool,
665 cancelled: bool,
666 configure_request_id: String,
667 started_at: Instant,
668 warned_half_deadline: bool,
669 deadline_reported: bool,
670 corr: u64,
671 ver: u8,
672 flags: Flags,
673 cancellation: crate::executor::JobCancellation,
678}
679
680struct RouteBindCompletion {
681 route: RouteChannel,
682 identity: RouteIdentity,
683 bind_root_id: ProjectRootId,
684 inserted_new_actor: bool,
685 configure_response: Response,
686 diagnostics_on_edit: bool,
687 ver: u8,
688 corr: u64,
689 flags: Flags,
690}
691
692#[derive(Debug, Clone)]
693struct RouteIdentity(Arc<RouteIdentityData>);
694
695#[derive(Debug)]
696struct RouteIdentityData {
697 root: ProjectRootId,
698 project_root: PathBuf,
699 harness: String,
700 session: String,
701 trust: BindTrust,
702 spawn_principal: AuthenticatedPrincipal,
703 consumer_elicitation_capable: bool,
704}
705
706impl Deref for RouteIdentity {
707 type Target = RouteIdentityData;
708
709 fn deref(&self) -> &Self::Target {
710 &self.0
711 }
712}
713
714#[derive(Debug, Clone)]
715struct RetainedSessionIdentity {
716 harness: String,
717 trust: BindTrust,
718}
719
720#[derive(Clone)]
721struct BgSub {
722 corr: u64,
723 ver: u8,
724 flags: Flags,
725 root: ProjectRootId,
726 session: String,
727}
728
729type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
732
733struct MaintenanceCompletion {
734 root_id: ProjectRootId,
735 kind: MaintenanceDrainKind,
736 response: Response,
737 empty_bg_sessions: Vec<(String, u64)>,
738 requeue_kind: Option<MaintenanceDrainKind>,
739}
740
741#[derive(Clone, Copy, Debug, PartialEq, Eq)]
742enum MaintenanceDrainKind {
743 Watcher,
744 Lsp,
745 ConfigureTail,
746 CompletionDrains,
747}
748
749impl MaintenanceDrainKind {
750 fn label(self) -> &'static str {
751 match self {
752 Self::Watcher => "watcher",
753 Self::Lsp => "lsp",
754 Self::ConfigureTail => "configure-tail",
755 Self::CompletionDrains => "completion-drains",
756 }
757 }
758}
759
760#[derive(Debug, Default)]
761struct MaintenanceJobOutcome {
762 empty_bg_sessions: Vec<(String, u64)>,
763 requeue_kind: Option<MaintenanceDrainKind>,
764}
765
766#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
767struct ReverseCorrKey {
768 route: RouteChannel,
769 corr: u64,
770}
771
772struct PendingBashAsk {
773 route: RouteChannel,
774 tool_corr: u64,
775 tool_flags: Flags,
776 tool_ver: u8,
777 root: ProjectRootId,
778 project_root: PathBuf,
779 session_id: String,
780 spawn_principal: AuthenticatedPrincipal,
781 edit_slot_survives: Option<bool>,
782 request_id: String,
783 arguments: Value,
784 format_context: crate::subc_format::FormatContext,
785 cancel: bash::BashWaitCancel,
786 grants: Vec<String>,
787 expires_at: Instant,
788}
789
790impl RootMeta {
791 fn new(now: Instant) -> Self {
792 Self {
793 maintenance_pending: false,
794 maintenance_jobs_in_flight: 0,
795 maintenance_queued_kinds: VecDeque::new(),
796 maintenance_last_submitted: None,
797 maintenance_poisoned: false,
798 last_touched: now,
799 diagnostics_on_edit: false,
800 active_bash_waits: 0,
801 idle_artifacts_evicted: false,
802 unbound_quiesced: false,
803 consecutive_missing_sweeps: 0,
804 }
805 }
806
807 fn note_activity(&mut self) {
808 self.last_touched = Instant::now();
809 }
810
811 fn reactivate_bound(&mut self) {
812 self.note_activity();
813 self.idle_artifacts_evicted = false;
814 self.unbound_quiesced = false;
815 }
816}
817
818fn due_maintenance_jobs(
819 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
820 executor: Option<&Executor>,
821 bg_sub_by_session: &BgSubsBySession,
822 bg_wake_pending: &HashSet<RouteChannel>,
823 budget: usize,
824 pending_bind_roots: &HashSet<ProjectRootId>,
825) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
826 let mut jobs = Vec::new();
827 let mut deferred = false;
828 let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
829 roots.sort_by(|left, right| {
830 let left_last = live_roots
831 .get(left)
832 .and_then(|meta| meta.maintenance_last_submitted);
833 let right_last = live_roots
834 .get(right)
835 .and_then(|meta| meta.maintenance_last_submitted);
836 left_last
837 .cmp(&right_last)
838 .then_with(|| left.as_path().cmp(right.as_path()))
839 });
840
841 for root_id in roots {
842 let Some(meta) = live_roots.get_mut(&root_id) else {
843 continue;
844 };
845 if meta.maintenance_poisoned {
846 continue;
847 }
848
849 if pending_bind_roots.contains(&root_id) {
850 if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
851 deferred = true;
852 }
853 continue;
854 }
855
856 if !meta.maintenance_pending {
857 if jobs.len() >= budget {
858 deferred = true;
859 continue;
860 }
861 let executor_actor_context =
865 executor.and_then(|executor| executor.actor_context(&root_id));
866 let root_has_pending_bg_wake =
867 bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
868 sub_root == &root_id
869 && channels
870 .iter()
871 .any(|channel| bg_wake_pending.contains(channel))
872 });
873 let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
874 Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
875 .into_iter()
876 .filter(|kind| {
877 if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
878 return false;
879 }
880 match kind {
881 MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
882 MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
883 MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
884 MaintenanceDrainKind::CompletionDrains => {
890 root_has_pending_bg_wake || ctx.completion_drains_have_work()
891 }
892 }
893 })
894 .collect(),
895 None if meta.unbound_quiesced => Vec::new(),
896 None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
898 };
899 if kinds_with_work.is_empty() {
900 continue;
901 }
902 meta.maintenance_pending = true;
903 meta.maintenance_queued_kinds.extend(kinds_with_work);
904 }
905
906 while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
907 if jobs.len() >= budget {
908 meta.maintenance_queued_kinds.push_front(kind);
909 deferred = true;
910 break;
911 }
912 meta.maintenance_jobs_in_flight += 1;
913 meta.maintenance_last_submitted = Some(Instant::now());
914 jobs.push((root_id.clone(), kind));
915 }
916
917 meta.maintenance_pending =
918 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
919 }
920
921 (jobs, deferred)
922}
923
924fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
925 match estimate.estimated_bytes {
926 Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
927 None if estimate.status == "busy" => "busy".to_string(),
928 None => "not estimated".to_string(),
929 }
930}
931
932fn optional_memory_label(bytes: Option<u64>) -> String {
933 bytes.map_or_else(
934 || "not estimated".to_string(),
935 |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
936 )
937}
938
939fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
940 format!(
941 "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
942 optional_memory_label(relief.rss_before_bytes),
943 optional_memory_label(relief.rss_after_bytes),
944 optional_memory_label(relief.allocator_before.bytes_in_use),
945 optional_memory_label(relief.allocator_after.bytes_in_use),
946 optional_memory_label(relief.allocator_before.size_allocated),
947 optional_memory_label(relief.allocator_after.size_allocated),
948 optional_memory_label(relief.allocator_before.retained_slack_bytes),
949 optional_memory_label(relief.allocator_after.retained_slack_bytes),
950 relief.bytes_released as f64 / (1024.0 * 1024.0),
951 )
952}
953
954fn idle_root_eviction_message(
955 root_id: &ProjectRootId,
956 memory: &crate::memory::RootMemorySnapshot,
957 pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
958) -> String {
959 let freed_bytes = [
962 &memory.semantic,
963 &memory.trigram,
964 &memory.symbols,
965 &memory.callgraph,
966 &memory.inspect,
967 ]
968 .iter()
969 .filter_map(|estimate| estimate.estimated_bytes)
970 .fold(0u64, u64::saturating_add);
971 let mut message = format!(
972 "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
973 root_id.as_path().display(),
974 freed_bytes as f64 / (1024.0 * 1024.0),
975 eviction_estimate_label(&memory.semantic),
976 eviction_estimate_label(&memory.trigram),
977 eviction_estimate_label(&memory.symbols),
978 eviction_estimate_label(&memory.callgraph),
979 eviction_estimate_label(&memory.inspect),
980 eviction_estimate_label(&memory.bash),
981 eviction_estimate_label(&memory.lsp),
982 eviction_estimate_label(&memory.parser_pool),
983 );
984 if let Some(pressure_relief) = pressure_relief {
985 message.push_str(&pressure_relief_label(pressure_relief));
986 }
987 message
988}
989
990fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
991 !live_roots.is_empty()
992 && live_roots.values().all(|meta| {
993 now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
994 && meta.active_bash_waits == 0
995 && !meta.maintenance_pending
996 && meta.maintenance_queued_kinds.is_empty()
997 })
998}
999
1000fn allocator_pressure_relief_after_idle_sweep(
1001 now: Instant,
1002 live_roots: &HashMap<ProjectRootId, RootMeta>,
1003 executor: &Executor,
1004) -> Option<crate::memory::AllocatorPressureRelief> {
1005 if !process_has_been_idle(now, live_roots)
1006 || live_roots.keys().any(|root_id| {
1007 executor
1008 .actor_context(root_id)
1009 .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1010 })
1011 {
1012 return None;
1013 }
1014
1015 #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1016 {
1017 Some(crate::memory::relieve_allocator_pressure())
1018 }
1019 #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1020 {
1021 None
1022 }
1023}
1024
1025fn quiesce_unbound_root(
1026 root_id: &ProjectRootId,
1027 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1028 executor: &Arc<Executor>,
1029) {
1030 let Some(meta) = live_roots.get_mut(root_id) else {
1031 return;
1032 };
1033
1034 let ctx = executor.actor_context(root_id);
1035 if let Some(ctx) = ctx.as_ref() {
1036 ctx.mark_subc_unbound();
1040 }
1041 let cancelled = executor.cancel_queued_maintenance(root_id);
1042 let discarded = ctx
1050 .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1051 .unwrap_or(0);
1052 meta.unbound_quiesced = true;
1053 meta.maintenance_queued_kinds.clear();
1054 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1055 log::info!(
1056 "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1057 root_id.as_path().display(),
1058 cancelled,
1059 discarded
1060 );
1061}
1062
1063#[allow(clippy::too_many_arguments)]
1064fn quiesce_connection_roots(
1065 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1066 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1067 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1068 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1069 installed_route_epochs: &mut HashMap<u16, u32>,
1070 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1071 active_tool_calls: &ActiveToolCalls,
1072 executor: &Arc<Executor>,
1073) {
1074 cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1075 for cancel in route_bash_cancels.values() {
1076 cancel.token.cancel();
1077 }
1078 route_bash_cancels.clear();
1079
1080 let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1081 for pending in pending_binds.values_mut() {
1082 pending.cancelled = true;
1083 roots.insert(pending.bind_root_id.clone());
1084 let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1085 }
1086
1087 for root_id in roots {
1091 if live_roots.contains_key(&root_id) {
1092 quiesce_unbound_root(&root_id, live_roots, executor);
1093 } else if let Some(ctx) = executor.actor_context(&root_id) {
1094 ctx.mark_subc_unbound();
1095 executor.cancel_queued_maintenance(&root_id);
1096 crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1097 }
1098 }
1099
1100 routes.clear();
1101 root_channels.clear();
1102 installed_route_epochs.clear();
1103}
1104
1105#[derive(Debug, Default)]
1109struct ReclaimedRoutes {
1110 highest_epoch_by_channel: HashMap<u16, u32>,
1111}
1112
1113impl ReclaimedRoutes {
1114 fn insert(&mut self, route: RouteChannel) {
1115 self.highest_epoch_by_channel
1116 .entry(route.channel)
1117 .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1118 .or_insert(route.epoch);
1119 }
1120
1121 fn contains(&self, route: RouteChannel) -> bool {
1122 self.highest_epoch_by_channel
1123 .get(&route.channel)
1124 .is_some_and(|epoch| route.epoch <= *epoch)
1125 }
1126}
1127
1128#[derive(Debug, Default)]
1129struct IdleReapOutcome {
1130 evicted: usize,
1131 forgotten_deleted_roots: Vec<ProjectRootId>,
1132}
1133
1134fn reap_idle_roots(
1135 now: Instant,
1136 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1137 pending_binds: &HashMap<RouteChannel, PendingBind>,
1138 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1139 executor: &Arc<Executor>,
1140 metrics: &DispatchPathMetrics,
1141) -> IdleReapOutcome {
1142 let pending_bind_roots = pending_binds
1143 .values()
1144 .map(|pending| pending.bind_root_id.clone())
1145 .collect::<HashSet<_>>();
1146 let mut census = ReapBlockerCensus::default();
1147 let mut candidates = Vec::new();
1148
1149 for (root_id, meta) in live_roots.iter_mut() {
1150 let deleted = !root_id.as_path().exists();
1151 if deleted {
1152 meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1162 } else {
1163 meta.consecutive_missing_sweeps = 0;
1164 }
1165 let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1166 let has_bound_route = root_channels
1167 .get(root_id)
1168 .is_some_and(|channels| !channels.is_empty());
1169 let has_pending_bind = pending_bind_roots.contains(root_id);
1170
1171 if deleted {
1172 let mut retained = false;
1173 if !deletion_confirmed {
1174 census.absence_unconfirmed += 1;
1175 retained = true;
1176 }
1177 if meta.active_bash_waits > 0 {
1181 census.bash_waits += 1;
1182 retained = true;
1183 }
1184 if meta.maintenance_pending {
1185 census.maintenance_pending += 1;
1186 retained = true;
1187 }
1188 if !meta.maintenance_queued_kinds.is_empty() {
1189 census.maintenance_queued += 1;
1190 retained = true;
1191 }
1192 if has_pending_bind {
1193 census.pending_binds += 1;
1194 retained = true;
1195 }
1196 match executor.try_actor_is_idle(root_id) {
1197 Some(true) => {}
1198 Some(false) => {
1199 census.actor_busy += 1;
1200 retained = true;
1201 }
1202 None => {
1203 census.actor_state_busy += 1;
1204 retained = true;
1205 }
1206 }
1207 if retained {
1208 census.deleted_retained += 1;
1209 continue;
1210 }
1211 } else {
1212 if has_bound_route
1216 || !meta.unbound_quiesced
1217 || meta.idle_artifacts_evicted
1218 || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
1219 || meta.active_bash_waits > 0
1220 || meta.maintenance_pending
1221 || !meta.maintenance_queued_kinds.is_empty()
1222 || has_pending_bind
1223 || !executor.actor_is_idle(root_id)
1224 {
1225 continue;
1226 }
1227 }
1228 candidates.push((root_id.clone(), deleted));
1229 }
1230
1231 let mut reaped = Vec::new();
1232 let mut forgotten_deleted_roots = Vec::new();
1233 for (root_id, deleted) in candidates {
1234 let Some(ctx) = executor.actor_context(&root_id) else {
1235 if deleted {
1236 census.deleted_retained += 1;
1237 census.actor_busy += 1;
1238 }
1239 continue;
1240 };
1241 if deleted {
1252 ctx.bash_background()
1253 .kill_running_tasks_for_root(root_id.as_path());
1254 }
1255 let taken_pending = Some(ctx.take_pending_reconciliation_state());
1256 if ctx.artifact_eviction_blocked() {
1257 if let Some(pending) = taken_pending {
1258 ctx.restore_pending_reconciliation_state(pending);
1259 }
1260 if deleted {
1261 census.deleted_retained += 1;
1262 census.artifact_eviction_blocked += 1;
1263 }
1264 continue;
1265 }
1266 let memory_before = ctx.memory_root_snapshot();
1267 if !ctx.evict_idle_artifacts() {
1268 if let Some(pending) = taken_pending {
1269 ctx.restore_pending_reconciliation_state(pending);
1270 }
1271 if deleted {
1272 census.deleted_retained += 1;
1273 census.artifact_eviction_failed += 1;
1274 }
1275 continue;
1276 }
1277 drop(taken_pending);
1278 ctx.stop_watcher_runtime_in_background();
1279 ctx.invalidate_artifacts_after_watcher_gap();
1282
1283 if deleted {
1284 if executor.retire_idle_actor_in_background(&root_id) {
1285 live_roots.remove(&root_id);
1286 forgotten_deleted_roots.push(root_id.clone());
1287 } else {
1288 census.deleted_retained += 1;
1289 census.actor_busy += 1;
1290 }
1291 } else {
1292 if let Some(meta) = live_roots.get_mut(&root_id) {
1293 meta.idle_artifacts_evicted = true;
1294 }
1295 ctx.release_idle_reopenable_resources_in_background();
1296 }
1297 reaped.push((root_id, memory_before));
1298 }
1299
1300 metrics.record_reap(census);
1301 if census.deleted_retained > 0 {
1302 log::info!(
1303 "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1304 census.deleted_retained,
1305 census.blocker_histogram()
1306 );
1307 }
1308
1309 let pressure_relief = (!reaped.is_empty())
1310 .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1311 .flatten();
1312 for (root_id, memory_before) in &reaped {
1313 log::info!(
1314 "{}",
1315 idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1316 );
1317 }
1318 IdleReapOutcome {
1319 evicted: reaped.len(),
1320 forgotten_deleted_roots,
1321 }
1322}
1323
1324#[allow(clippy::too_many_arguments)]
1325fn purge_deleted_root_residents(
1326 root_id: &ProjectRootId,
1327 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1328 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1329 installed_route_epochs: &mut HashMap<u16, u32>,
1330 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1331 active_tool_calls: &ActiveToolCalls,
1332 executor: &Executor,
1333 retry_buffer: &mut RetryBuffer,
1334 reclaimed_routes: &mut ReclaimedRoutes,
1335 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1336 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1337 bg_subs: &mut HashMap<RouteChannel, BgSub>,
1338 bg_sub_by_session: &mut BgSubsBySession,
1339 bg_wake_pending: &mut HashSet<RouteChannel>,
1340 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1341 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1342 metrics: &DispatchPathMetrics,
1343) {
1344 let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1345 stale_routes.extend(
1346 routes
1347 .iter()
1348 .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1349 );
1350 stale_routes.extend(
1351 bg_sub_by_session
1352 .iter()
1353 .filter(|((root, _), _)| root == root_id)
1354 .flat_map(|(_, routes)| routes.iter().copied()),
1355 );
1356 stale_routes.extend(
1357 pending_bash_asks
1358 .values()
1359 .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1360 );
1361
1362 for route in stale_routes {
1363 reclaimed_routes.insert(route);
1364 remove_installed_route(installed_route_epochs, route);
1365 remove_route_channel(routes, root_channels, route);
1366 if let Some(cancel) = route_bash_cancels.remove(&route) {
1367 cancel.token.cancel();
1368 }
1369 apply_route_work_disposition(
1370 active_tool_calls,
1371 executor,
1372 route,
1373 RouteWorkDisposition::Abandon,
1374 "root reclaim",
1375 );
1376 retry_buffer.remove(&route);
1377 if let Some(sub) = bg_subs.remove(&route) {
1378 metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1379 }
1380 bg_wake_pending.remove(&route);
1381 }
1382 root_channels.remove(root_id);
1383 session_identity.retain(|(root, _), _| root != root_id);
1384 push_buffer.retain(|key, _| &key.root != root_id);
1385 bg_wake_epoch.retain(|(root, _), _| root != root_id);
1386 pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1387 bg_sub_by_session.retain(|(root, _), _| root != root_id);
1388
1389 log::info!(
1390 "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1391 root_id.as_path().display()
1392 );
1393}
1394
1395#[allow(clippy::too_many_arguments)]
1396fn submit_due_maintenance_jobs(
1397 executor: &Arc<Executor>,
1398 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1399 pending_binds: &HashMap<RouteChannel, PendingBind>,
1400 bg_sub_by_session: &BgSubsBySession,
1401 bg_wake_pending: &HashSet<RouteChannel>,
1402 bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1403 maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1404 metrics: &Arc<DispatchPathMetrics>,
1405) {
1406 let pending_bind_roots = pending_binds
1407 .values()
1408 .map(|pending| pending.bind_root_id.clone())
1409 .collect::<HashSet<_>>();
1410 let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1411 live_roots,
1412 Some(executor),
1413 bg_sub_by_session,
1414 bg_wake_pending,
1415 MAINTENANCE_SUBMIT_BUDGET,
1416 &pending_bind_roots,
1417 );
1418 if deferred_jobs {
1419 metrics
1420 .maintenance_budget_deferrals
1421 .fetch_add(1, Ordering::Relaxed);
1422 }
1423 for (root_id, kind) in due_jobs {
1424 let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1425 bg_sub_by_session
1426 .iter()
1427 .filter_map(|((root, session), _)| {
1428 if root == &root_id {
1429 Some((
1430 session.clone(),
1431 bg_wake_epoch
1432 .get(&(root_id.clone(), session.clone()))
1433 .copied()
1434 .unwrap_or(0),
1435 ))
1436 } else {
1437 None
1438 }
1439 })
1440 .collect()
1441 } else {
1442 Vec::new()
1443 };
1444 submit_maintenance_job(
1445 executor,
1446 root_id,
1447 kind,
1448 bg_sessions_to_check,
1449 maintenance_tx,
1450 metrics,
1451 );
1452 }
1453}
1454
1455fn should_requiesce_after_maintenance(
1456 meta: &RootMeta,
1457 completed_kind: MaintenanceDrainKind,
1458 bind_pending: bool,
1459) -> bool {
1460 meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1461}
1462
1463fn note_maintenance_completion(
1464 meta: &mut RootMeta,
1465 requeue_kind: Option<MaintenanceDrainKind>,
1466 fatal: bool,
1467 defer_requeue: bool,
1468) {
1469 if fatal {
1470 meta.maintenance_poisoned = true;
1471 }
1472
1473 if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1474 meta.maintenance_queued_kinds.push_back(kind);
1475 }
1476
1477 meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1478 meta.maintenance_pending =
1479 meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1480}
1481
1482fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1483 RouteChannel { channel, epoch }
1484}
1485
1486fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1487 if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1488 installed_epochs.remove(&route.channel);
1489 }
1490}
1491
1492fn ingress_route_should_be_processed(
1493 installed_epochs: &HashMap<u16, u32>,
1494 reclaimed_routes: &ReclaimedRoutes,
1495 frame: &Frame,
1496) -> bool {
1497 if frame.header.channel == 0
1498 || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1499 {
1500 return true;
1501 }
1502
1503 frame.header.ty == FrameType::Request
1508 && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1509}
1510
1511fn bash_elicitation_timeout() -> Duration {
1512 if cfg!(debug_assertions) {
1513 if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1514 if let Ok(ms) = raw.parse::<u64>() {
1515 if ms > 0 {
1516 return Duration::from_millis(ms);
1517 }
1518 }
1519 }
1520 }
1521 BASH_ELICITATION_TIMEOUT
1522}
1523
1524fn allocate_reverse_corr(
1525 pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1526 route: RouteChannel,
1527 next_corr: &mut u64,
1528) -> u64 {
1529 loop {
1530 let corr = *next_corr;
1531 *next_corr = (*next_corr).wrapping_add(1).max(1);
1532 if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1533 return corr;
1534 }
1535 }
1536}
1537
1538fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1539 match kind {
1540 crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1541 crate::bash_permissions::PermissionKind::Bash => "bash",
1542 }
1543}
1544
1545fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1546 let mut patterns = Vec::new();
1547 let mut seen = HashSet::new();
1548 for ask in asks {
1549 for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1550 if seen.insert(pattern.clone()) {
1551 patterns.push(pattern.clone());
1552 }
1553 }
1554 }
1555 patterns
1556}
1557
1558fn bash_elicitation_message(
1559 command: &str,
1560 asks: &[crate::bash_permissions::PermissionAsk],
1561) -> String {
1562 let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1563 let patterns = bash_elicitation_patterns(asks);
1564 let pattern_text = if patterns.is_empty() {
1565 "no matched permission patterns".to_string()
1566 } else {
1567 patterns.join(", ")
1568 };
1569 let ask_kinds = asks
1570 .iter()
1571 .map(|ask| bash_permission_kind_label(&ask.kind))
1572 .collect::<HashSet<_>>()
1573 .into_iter()
1574 .collect::<Vec<_>>()
1575 .join(", ");
1576 if ask_kinds.is_empty() {
1577 format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1578 } else {
1579 format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1580 }
1581}
1582
1583fn bash_elicitation_request_body(
1584 command: &str,
1585 asks: &[crate::bash_permissions::PermissionAsk],
1586) -> Value {
1587 json!({
1588 "method": BASH_ELICITATION_CREATE_METHOD,
1589 "params": {
1590 "mode": "form",
1591 "message": bash_elicitation_message(command, asks),
1592 "requestedSchema": {
1593 "type": "object",
1594 "properties": {
1595 "decision": {
1596 "type": "string",
1597 "enum": ["allow", "deny"],
1598 "description": "Choose allow to run this bash command once, or deny to block it."
1599 }
1600 },
1601 "required": ["decision"],
1602 "additionalProperties": false
1603 },
1604 "_meta": {
1605 "aft": {
1606 "tool": "bash",
1607 "command": command,
1608 "asks": asks
1609 }
1610 }
1611 }
1612 })
1613}
1614
1615fn build_bash_elicitation_request_frame(
1616 ver: u8,
1617 route: RouteChannel,
1618 corr: u64,
1619 flags: Flags,
1620 command: &str,
1621 asks: &[crate::bash_permissions::PermissionAsk],
1622) -> Result<Frame, SubcError> {
1623 let body = bash_elicitation_request_body(command, asks);
1624 Frame::build_with_version(
1625 ver,
1626 FrameType::Request,
1627 flags,
1628 route.channel,
1629 route.epoch,
1630 corr,
1631 serde_json::to_vec(&body).map_err(SubcError::Json)?,
1632 )
1633 .map_err(SubcError::FrameBuild)
1634}
1635
1636fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1637 let Ok(value) = serde_json::from_slice::<Value>(body) else {
1638 return false;
1639 };
1640 flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1641}
1642
1643fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1644 let Some(object) = value.as_object() else {
1645 return false;
1646 };
1647 object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1648}
1649
1650fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1651 let Some(object) = value.as_object() else {
1652 return false;
1653 };
1654 if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1655 return false;
1656 }
1657 let Some(content) = object.get("content").and_then(Value::as_object) else {
1658 return false;
1659 };
1660 content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1661}
1662
1663#[allow(clippy::too_many_arguments)]
1664async fn settle_pending_bash_ask_denied(
1665 tx: &WriterSender,
1666 pending: PendingBashAsk,
1667 routes: &HashMap<RouteChannel, RouteIdentity>,
1668 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1669 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1670 shutdown: &Arc<Notify>,
1671 metrics: &DispatchPathMetrics,
1672) -> Result<(), SubcError> {
1673 let completion = bash::bash_denied_untrusted_completion(
1674 pending.route,
1675 pending.tool_corr,
1676 pending.tool_flags,
1677 pending.tool_ver,
1678 pending.root,
1679 pending.request_id,
1680 pending.format_context,
1681 );
1682 bash::handle_bash_deferred_completion(
1683 tx,
1684 completion,
1685 routes,
1686 live_roots,
1687 route_bash_cancels,
1688 shutdown,
1689 metrics,
1690 )
1691 .await
1692}
1693
1694fn take_pending_bash_asks_for_route(
1695 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1696 route: RouteChannel,
1697) -> Vec<PendingBashAsk> {
1698 let keys = pending_bash_asks
1699 .keys()
1700 .copied()
1701 .filter(|key| key.route == route)
1702 .collect::<Vec<_>>();
1703 keys.into_iter()
1704 .filter_map(|key| pending_bash_asks.remove(&key))
1705 .collect()
1706}
1707
1708#[allow(clippy::too_many_arguments)]
1709async fn settle_pending_bash_asks_for_route(
1710 tx: &WriterSender,
1711 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1712 route: RouteChannel,
1713 routes: &HashMap<RouteChannel, RouteIdentity>,
1714 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1715 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1716 shutdown: &Arc<Notify>,
1717 metrics: &DispatchPathMetrics,
1718) -> Result<(), SubcError> {
1719 for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1720 settle_pending_bash_ask_denied(
1721 tx,
1722 pending,
1723 routes,
1724 live_roots,
1725 route_bash_cancels,
1726 shutdown,
1727 metrics,
1728 )
1729 .await?;
1730 }
1731 Ok(())
1732}
1733
1734#[allow(clippy::too_many_arguments)]
1735async fn settle_all_pending_bash_asks(
1736 tx: &WriterSender,
1737 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1738 routes: &HashMap<RouteChannel, RouteIdentity>,
1739 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1740 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1741 shutdown: &Arc<Notify>,
1742 metrics: &DispatchPathMetrics,
1743) -> Result<(), SubcError> {
1744 let pending = pending_bash_asks
1745 .drain()
1746 .map(|(_, pending)| pending)
1747 .collect::<Vec<_>>();
1748 for pending in pending {
1749 settle_pending_bash_ask_denied(
1750 tx,
1751 pending,
1752 routes,
1753 live_roots,
1754 route_bash_cancels,
1755 shutdown,
1756 metrics,
1757 )
1758 .await?;
1759 }
1760 Ok(())
1761}
1762
1763#[allow(clippy::too_many_arguments)]
1764async fn expire_pending_bash_asks(
1765 tx: &WriterSender,
1766 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1767 routes: &HashMap<RouteChannel, RouteIdentity>,
1768 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1769 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1770 shutdown: &Arc<Notify>,
1771 metrics: &DispatchPathMetrics,
1772) -> Result<(), SubcError> {
1773 let now = Instant::now();
1774 let expired = pending_bash_asks
1775 .iter()
1776 .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1777 .collect::<Vec<_>>();
1778 for key in expired {
1779 if let Some(pending) = pending_bash_asks.remove(&key) {
1780 log::debug!(
1781 "subc attach: bash elicitation request {} on route {} expired fail-closed",
1782 key.corr,
1783 pending.route
1784 );
1785 settle_pending_bash_ask_denied(
1786 tx,
1787 pending,
1788 routes,
1789 live_roots,
1790 route_bash_cancels,
1791 shutdown,
1792 metrics,
1793 )
1794 .await?;
1795 }
1796 }
1797 Ok(())
1798}
1799
1800#[allow(clippy::too_many_arguments)]
1801async fn handle_bash_elicitation_reply(
1802 tx: &WriterSender,
1803 frame: &Frame,
1804 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1805 routes: &HashMap<RouteChannel, RouteIdentity>,
1806 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1807 executor: &Arc<Executor>,
1808 shutdown: &Arc<Notify>,
1809 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1810 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1811 metrics: &Arc<DispatchPathMetrics>,
1812 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1813 dispatch: DispatchFn,
1814) -> Result<(), SubcError> {
1815 let key = ReverseCorrKey {
1816 route: route_key(frame.header.channel, frame.header.epoch),
1817 corr: frame.header.corr,
1818 };
1819 let Some(pending) = pending_bash_asks.remove(&key) else {
1820 return Ok(());
1821 };
1822
1823 if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1824 if routes.contains_key(&key.route) {
1825 bash::submit_deferred_bash(
1826 executor,
1827 bash_deferred_tx,
1828 bash_poll_touch_tx,
1829 metrics,
1830 dispatch,
1831 pending.root,
1832 pending.project_root,
1833 pending.session_id,
1834 pending.request_id,
1835 pending.route,
1836 pending.tool_corr,
1837 pending.tool_flags,
1838 pending.tool_ver,
1839 pending.arguments,
1840 pending.format_context,
1841 pending.cancel,
1842 BindTrust::Untrusted,
1843 pending.spawn_principal,
1844 pending.edit_slot_survives,
1845 Some(pending.grants),
1846 );
1847 return Ok(());
1848 }
1849 log::debug!(
1850 "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1851 key.corr,
1852 pending.route
1853 );
1854 }
1855
1856 settle_pending_bash_ask_denied(
1857 tx,
1858 pending,
1859 routes,
1860 live_roots,
1861 route_bash_cancels,
1862 shutdown,
1863 metrics,
1864 )
1865 .await
1866}
1867
1868#[allow(clippy::too_many_arguments)]
1869async fn cancel_pending_bash_ask_for_tool_call(
1870 tx: &WriterSender,
1871 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1872 route: RouteChannel,
1873 tool_corr: u64,
1874 routes: &HashMap<RouteChannel, RouteIdentity>,
1875 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1876 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1877 shutdown: &Arc<Notify>,
1878 metrics: &DispatchPathMetrics,
1879) -> Result<(), SubcError> {
1880 let keys = pending_bash_asks
1881 .iter()
1882 .filter_map(|(key, pending)| {
1883 (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1884 })
1885 .collect::<Vec<_>>();
1886 for key in keys {
1887 if let Some(pending) = pending_bash_asks.remove(&key) {
1888 settle_pending_bash_ask_denied(
1889 tx,
1890 pending,
1891 routes,
1892 live_roots,
1893 route_bash_cancels,
1894 shutdown,
1895 metrics,
1896 )
1897 .await?;
1898 }
1899 }
1900 Ok(())
1901}
1902
1903fn remove_root_channel(
1904 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1905 root: &ProjectRootId,
1906 channel: RouteChannel,
1907) {
1908 let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1909 channels.remove(&channel);
1910 channels.is_empty()
1911 } else {
1912 false
1913 };
1914 if remove_root {
1915 root_channels.remove(root);
1916 }
1917}
1918
1919fn remove_route_channel(
1920 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1921 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1922 channel: RouteChannel,
1923) -> Option<RouteIdentity> {
1924 let removed = routes.remove(&channel);
1925 if let Some(identity) = &removed {
1926 remove_root_channel(root_channels, &identity.root, channel);
1927 }
1928 removed
1929}
1930
1931fn insert_route_channel(
1932 routes: &mut HashMap<RouteChannel, RouteIdentity>,
1933 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1934 channel: RouteChannel,
1935 identity: RouteIdentity,
1936) {
1937 if let Some(previous) = routes.insert(channel, identity.clone()) {
1938 remove_root_channel(root_channels, &previous.root, channel);
1939 }
1940 root_channels
1941 .entry(identity.root.clone())
1942 .or_default()
1943 .insert(channel);
1944}
1945
1946fn insert_bg_subscription_index(
1947 bg_sub_by_session: &mut BgSubsBySession,
1948 root: ProjectRootId,
1949 session: String,
1950 channel: RouteChannel,
1951) {
1952 bg_sub_by_session
1953 .entry((root, session))
1954 .or_default()
1955 .insert(channel);
1956}
1957
1958fn remove_bg_subscription_index(
1959 bg_sub_by_session: &mut BgSubsBySession,
1960 channel: RouteChannel,
1961 identity: Option<&RouteIdentity>,
1962) {
1963 if let Some(identity) = identity {
1964 let key = (identity.root.clone(), identity.session.clone());
1965 let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1966 channels.remove(&channel);
1967 channels.is_empty()
1968 });
1969 if remove_key {
1970 bg_sub_by_session.remove(&key);
1971 }
1972 } else {
1973 bg_sub_by_session.retain(|_, channels| {
1974 channels.remove(&channel);
1975 !channels.is_empty()
1976 });
1977 }
1978}
1979
1980fn route_removal_will_quiesce_root(
1981 root: &ProjectRootId,
1982 route: RouteChannel,
1983 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1984 has_pending_bind: bool,
1985 replacement_root: Option<&ProjectRootId>,
1986) -> bool {
1987 let removes_last_route = root_channels
1988 .get(root)
1989 .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1990 removes_last_route && !has_pending_bind && replacement_root != Some(root)
1991}
1992
1993fn should_quiesce_removed_root(
1994 root: &ProjectRootId,
1995 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1996 has_pending_bind: bool,
1997 replacement_root: Option<&ProjectRootId>,
1998) -> bool {
1999 !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2000}
2001
2002async fn end_bg_subscription(
2003 writer_tx: &WriterSender,
2004 metrics: &DispatchPathMetrics,
2005 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2006 bg_sub_by_session: &mut BgSubsBySession,
2007 bg_wake_pending: &mut HashSet<RouteChannel>,
2008 channel: RouteChannel,
2009 identity: Option<&RouteIdentity>,
2010 cause: &str,
2011) -> Result<(), SubcError> {
2012 if let Some(sub) = bg_subs.remove(&channel) {
2013 bg_wake_pending.remove(&channel);
2014 remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2015 metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2016 push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2017 }
2018 Ok(())
2019}
2020
2021#[allow(clippy::too_many_arguments)]
2022async fn teardown_installed_route(
2023 tx: &WriterSender,
2024 metrics: &DispatchPathMetrics,
2025 executor: &Arc<Executor>,
2026 channel: RouteChannel,
2027 cancellation_reason: &str,
2028 replacement_root: Option<&ProjectRootId>,
2029 installed_route_epochs: &mut HashMap<u16, u32>,
2030 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2031 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2032 bg_subs: &mut HashMap<RouteChannel, BgSub>,
2033 bg_sub_by_session: &mut BgSubsBySession,
2034 bg_wake_pending: &mut HashSet<RouteChannel>,
2035 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2036 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2037 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2038 active_tool_calls: &ActiveToolCalls,
2039 pending_responses: &mut PendingSubcResponses,
2040 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2041 retry_buffer: &mut RetryBuffer,
2042 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2043 shutdown: &Arc<Notify>,
2044 tool_response_body_limit: usize,
2045) -> Result<(), SubcError> {
2046 remove_installed_route(installed_route_epochs, channel);
2047 let bg_end_cause = match cancellation_reason {
2048 "Goodbye" => "goodbye",
2049 "higher-epoch RouteBind" => "higher-epoch",
2050 other => other,
2051 };
2052 end_bg_subscription(
2053 tx,
2054 metrics,
2055 bg_subs,
2056 bg_sub_by_session,
2057 bg_wake_pending,
2058 channel,
2059 routes.get(&channel),
2060 bg_end_cause,
2061 )
2062 .await?;
2063 settle_pending_bash_asks_for_route(
2064 tx,
2065 pending_bash_asks,
2066 channel,
2067 routes,
2068 live_roots,
2069 route_bash_cancels,
2070 shutdown,
2071 metrics,
2072 )
2073 .await?;
2074 if let Some(cancel) = route_bash_cancels.remove(&channel) {
2075 cancel.token.cancel();
2076 }
2077 for resolved in pending_responses.drain_route(channel, executor) {
2078 deliver_resolved_subc_response(
2079 tx,
2080 resolved,
2081 routes,
2082 live_roots,
2083 executor.as_ref(),
2084 active_tool_calls,
2085 shutdown,
2086 metrics,
2087 tool_response_body_limit,
2088 )
2089 .await?;
2090 }
2091 apply_route_work_disposition(
2095 active_tool_calls,
2096 executor,
2097 channel,
2098 RouteWorkDisposition::RetainForReplay,
2099 cancellation_reason,
2100 );
2101 if let Some(pending) = pending_binds.get_mut(&channel) {
2102 pending.cancelled = true;
2103 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2104 log::debug!(
2105 "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2106 channel.channel
2107 );
2108 }
2109 let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2110 if let Some(identity) = routes.get(&channel) {
2111 let has_pending_bind = pending_binds
2112 .values()
2113 .any(|pending| pending.bind_root_id == identity.root);
2114 if route_removal_will_quiesce_root(
2115 &identity.root,
2116 channel,
2117 root_channels,
2118 has_pending_bind,
2119 replacement_root,
2120 ) {
2121 if let Some(ctx) = executor.actor_context(&identity.root) {
2122 ctx.mark_subc_unbound();
2125 }
2126 }
2127 }
2128 if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2129 let session_still_routed = routes
2130 .values()
2131 .any(|route| route.root == identity.root && route.session == identity.session);
2132 if !session_still_routed {
2133 if let Some(ctx) = executor.actor_context(&identity.root) {
2134 ctx.hashline_bindings()
2135 .teardown(identity.root.as_path(), &identity.session);
2136 }
2137 }
2138 if migrated > 0 {
2139 log::debug!(
2140 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2141 channel.channel
2142 );
2143 }
2144 if let Some(meta) = live_roots.get_mut(&identity.root) {
2145 let idle_for = meta.last_touched.elapsed();
2146 meta.note_activity();
2147 log::debug!(
2148 "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2149 channel.channel,
2150 identity.root.as_path().display(),
2151 identity.harness,
2152 identity.session,
2153 idle_for
2154 );
2155 } else {
2156 log::debug!(
2157 "subc attach: route {} torn down for root {} harness {} session {}",
2158 channel.channel,
2159 identity.root.as_path().display(),
2160 identity.harness,
2161 identity.session
2162 );
2163 }
2164 let has_pending_bind = pending_binds
2165 .values()
2166 .any(|pending| pending.bind_root_id == identity.root);
2167 if should_quiesce_removed_root(
2168 &identity.root,
2169 root_channels,
2170 has_pending_bind,
2171 replacement_root,
2172 ) {
2173 quiesce_unbound_root(&identity.root, live_roots, executor);
2174 }
2175 } else {
2176 if migrated > 0 {
2177 log::debug!(
2178 "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2179 channel.channel
2180 );
2181 }
2182 log::debug!("subc attach: unbound route {} torn down", channel.channel);
2183 }
2184 Ok(())
2185}
2186
2187fn remember_session_identity(
2188 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2189 identity: &RouteIdentity,
2190) {
2191 let key = (identity.root.clone(), identity.session.clone());
2192 if matches!(identity.trust, BindTrust::Untrusted)
2193 && session_identity
2194 .get(&key)
2195 .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2196 {
2197 return;
2198 }
2199
2200 session_identity.insert(
2205 key,
2206 RetainedSessionIdentity {
2207 harness: identity.harness.clone(),
2208 trust: identity.trust,
2209 },
2210 );
2211}
2212
2213fn replay_key_for_session(
2214 session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2215 root: &ProjectRootId,
2216 session: &str,
2217) -> Option<(push::ReplayKey, BindTrust)> {
2218 let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2219 Some((
2220 push::ReplayKey {
2221 root: root.clone(),
2222 harness: retained.harness.clone(),
2223 session: session.to_string(),
2224 },
2225 retained.trust,
2226 ))
2227}
2228pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2231
2232#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2233enum ModuleLoopExit {
2234 Graceful,
2235 SkipSearchFlush,
2236}
2237
2238pub fn run_subc_mode(
2243 connection_file_path: &Path,
2244 ctx: Arc<AppContext>,
2245 executor: Arc<Executor>,
2246 dispatch: DispatchFn,
2247 user_config_path: Option<PathBuf>,
2248) -> Result<(), SubcError> {
2249 run_subc_mode_inner(
2253 connection_file_path,
2254 ctx,
2255 executor,
2256 dispatch,
2257 user_config_path,
2258 false,
2259 MAX_FRAME_BODY_LEN as usize,
2260 )
2261}
2262
2263fn run_subc_mode_inner(
2264 connection_file_path: &Path,
2265 ctx: Arc<AppContext>,
2266 executor: Arc<Executor>,
2267 dispatch: DispatchFn,
2268 user_config_path: Option<PathBuf>,
2269 allow_native_passthrough: bool,
2270 tool_response_body_limit: usize,
2271) -> Result<(), SubcError> {
2272 let runtime = tokio::runtime::Builder::new_current_thread()
2273 .enable_all()
2274 .build()
2275 .map_err(SubcError::Runtime)?;
2276
2277 let executor_for_loop = Arc::clone(&executor);
2278 let loop_result = runtime.block_on(async move {
2279 let shared_app = ctx.app();
2280 drop(ctx);
2281 let stream = connect_and_authenticate(connection_file_path).await?;
2282 log::info!(
2283 "subc attach: authenticated to daemon via {}",
2284 connection_file_path.display()
2285 );
2286 let (read_half, write_half) = tokio::io::split(stream);
2287 run_module_loop(
2288 read_half,
2289 write_half,
2290 connection_file_path,
2291 shared_app,
2292 executor_for_loop,
2293 dispatch,
2294 user_config_path,
2295 allow_native_passthrough,
2296 tool_response_body_limit,
2297 )
2298 .await
2299 });
2300
2301 let actor_contexts = executor.actor_contexts();
2302 if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2303 flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2306 }
2307 for actor_ctx in &actor_contexts {
2308 actor_ctx.lsp().shutdown_all();
2309 actor_ctx.bash_background().detach();
2310 }
2311
2312 loop_result.map(|_| ())
2313}
2314
2315fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2316 for actor_ctx in actor_contexts {
2317 let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2318 }
2319 let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2320}
2321
2322#[doc(hidden)]
2327pub fn run_subc_mode_for_test(
2328 connection_file_path: &Path,
2329 ctx: Arc<AppContext>,
2330 executor: Arc<Executor>,
2331 dispatch: DispatchFn,
2332 user_config_path: Option<PathBuf>,
2333) -> Result<(), SubcError> {
2334 run_subc_mode_inner(
2335 connection_file_path,
2336 ctx,
2337 executor,
2338 dispatch,
2339 user_config_path,
2340 true,
2341 MAX_FRAME_BODY_LEN as usize,
2342 )
2343}
2344
2345#[doc(hidden)]
2348pub fn run_subc_mode_for_test_with_response_body_limit(
2349 connection_file_path: &Path,
2350 ctx: Arc<AppContext>,
2351 executor: Arc<Executor>,
2352 dispatch: DispatchFn,
2353 user_config_path: Option<PathBuf>,
2354 tool_response_body_limit: usize,
2355) -> Result<(), SubcError> {
2356 assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2357 run_subc_mode_inner(
2358 connection_file_path,
2359 ctx,
2360 executor,
2361 dispatch,
2362 user_config_path,
2363 true,
2364 tool_response_body_limit,
2365 )
2366}
2367
2368#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2369enum AttachErrorClass {
2370 Transient,
2371 Permanent,
2372}
2373
2374impl fmt::Display for AttachErrorClass {
2375 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2376 match self {
2377 Self::Transient => f.write_str("transient"),
2378 Self::Permanent => f.write_str("permanent"),
2379 }
2380 }
2381}
2382
2383#[derive(Clone, Copy)]
2384struct AttachRetryPolicy {
2385 budget: Duration,
2386 initial_backoff: Duration,
2387 max_backoff: Duration,
2388 jitter_percent: u64,
2389}
2390
2391const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2392 budget: ATTACH_RETRY_BUDGET,
2393 initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2394 max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2395 jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2396};
2397
2398fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2401 let transient = match error {
2402 SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2403 SubcError::Auth { source, .. } => match source {
2404 subc_transport::AuthError::Timeout { .. }
2405 | subc_transport::AuthError::UnexpectedEof { .. } => true,
2406 subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2407 _ => false,
2408 },
2409 _ => false,
2410 };
2411 if transient {
2412 AttachErrorClass::Transient
2413 } else {
2414 AttachErrorClass::Permanent
2415 }
2416}
2417
2418fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2419 matches!(
2420 kind,
2421 io::ErrorKind::ConnectionRefused
2422 | io::ErrorKind::TimedOut
2423 | io::ErrorKind::ConnectionReset
2424 | io::ErrorKind::ConnectionAborted
2425 | io::ErrorKind::BrokenPipe
2426 | io::ErrorKind::UnexpectedEof
2427 )
2428}
2429
2430async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2434 connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2435}
2436
2437async fn connect_and_authenticate_with_policy(
2438 connection_file_path: &Path,
2439 policy: AttachRetryPolicy,
2440) -> Result<TcpStream, SubcError> {
2441 let started_at = Instant::now();
2442 let deadline = started_at + policy.budget;
2443 let mut attempt = 0_u32;
2444 let mut backoff = policy.initial_backoff;
2445 let mut history = Vec::new();
2446
2447 loop {
2448 attempt = attempt.saturating_add(1);
2449 let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2450 Ok(stream) => return Ok(stream),
2451 Err(error) => error,
2452 };
2453 let class = classify_attach_error(&error);
2454 let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2455 history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2456
2457 if class == AttachErrorClass::Permanent {
2458 log_attach_final_failure(started_at.elapsed(), &history);
2459 return Err(error);
2460 }
2461
2462 let remaining = deadline.saturating_duration_since(Instant::now());
2463 if remaining.is_zero() {
2464 log_attach_final_failure(started_at.elapsed(), &history);
2465 return Err(error);
2466 }
2467
2468 let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2469 log::info!(
2470 "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2471 );
2472 tokio::time::sleep(delay).await;
2473
2474 if Instant::now() >= deadline {
2475 log_attach_final_failure(started_at.elapsed(), &history);
2476 return Err(error);
2477 }
2478 backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2479 }
2480}
2481
2482fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2483 let jitter_percent = jitter_percent.min(100);
2484 if jitter_percent == 0 {
2485 return base;
2486 }
2487
2488 let mut random_bytes = [0_u8; 8];
2489 let random = if getrandom::fill(&mut random_bytes).is_ok() {
2490 u64::from_le_bytes(random_bytes)
2491 } else {
2492 let timestamp = std::time::SystemTime::now()
2493 .duration_since(std::time::UNIX_EPOCH)
2494 .unwrap_or_default()
2495 .subsec_nanos();
2496 u64::from(timestamp) ^ u64::from(attempt)
2497 };
2498 let span = jitter_percent.saturating_mul(2).saturating_add(1);
2499 let multiplier_percent = 100 - jitter_percent + random % span;
2500 let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2501 Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2502}
2503
2504fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2505 log::error!(
2506 "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2507 history.len(),
2508 history.join(" | ")
2509 );
2510}
2511
2512async fn connect_and_authenticate_once(
2513 connection_file_path: &Path,
2514 deadline: Instant,
2515) -> Result<TcpStream, SubcError> {
2516 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2519 SubcError::ConnectionFile {
2520 path: connection_file_path.to_path_buf(),
2521 source,
2522 }
2523 })?;
2524
2525 let endpoint = conn
2526 .endpoints
2527 .first()
2528 .ok_or_else(|| SubcError::NoEndpoint {
2529 path: connection_file_path.to_path_buf(),
2530 })?;
2531 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2532 let ip = endpoint
2533 .host
2534 .parse::<IpAddr>()
2535 .map_err(|_| SubcError::InvalidEndpoint {
2536 path: connection_file_path.to_path_buf(),
2537 endpoint: endpoint_label.clone(),
2538 })?;
2539 let addr = SocketAddr::new(ip, endpoint.port);
2540
2541 let connect_budget = deadline.saturating_duration_since(Instant::now());
2542 let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2543 .await
2544 .map_err(|_| SubcError::Connect {
2545 endpoint: endpoint_label.clone(),
2546 source: io::Error::new(
2547 io::ErrorKind::TimedOut,
2548 "initial subc attach retry budget elapsed during TCP connect",
2549 ),
2550 })?
2551 .map_err(|source| SubcError::Connect {
2552 endpoint: endpoint_label.clone(),
2553 source,
2554 })?;
2555 stream
2556 .set_nodelay(true)
2557 .map_err(|source| SubcError::Connect {
2558 endpoint: endpoint_label.clone(),
2559 source,
2560 })?;
2561
2562 let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2563 authenticate_client(&mut stream, &conn, auth_budget)
2564 .await
2565 .map_err(|source| SubcError::Auth {
2566 endpoint: endpoint_label,
2567 source,
2568 })?;
2569
2570 Ok(stream)
2571}
2572
2573#[allow(clippy::too_many_arguments)]
2574async fn process_route_bind_completion(
2575 writer_tx: &WriterSender,
2576 completion: RouteBindCompletion,
2577 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2578 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2579 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2580 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2581 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2582 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2583 installed_route_epochs: &mut HashMap<u16, u32>,
2584 executor: &Arc<Executor>,
2585 standing_actor: &standing::StandingActor,
2586 shutdown: &Arc<Notify>,
2587 metrics: &Arc<DispatchPathMetrics>,
2588) -> Result<(), SubcError> {
2589 decrement_counted_channel(&metrics.control_completion_queued);
2590 handle_route_bind_completion(
2591 writer_tx,
2592 completion,
2593 routes,
2594 root_channels,
2595 session_identity,
2596 push_buffer,
2597 live_roots,
2598 pending_binds,
2599 installed_route_epochs,
2600 executor,
2601 standing_actor,
2602 shutdown,
2603 metrics,
2604 )
2605 .await
2606}
2607
2608#[allow(clippy::too_many_arguments)]
2609async fn drain_pending_route_bind_completions(
2610 control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2611 writer_tx: &WriterSender,
2612 routes: &mut HashMap<RouteChannel, RouteIdentity>,
2613 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2614 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2615 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2616 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2617 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2618 installed_route_epochs: &mut HashMap<u16, u32>,
2619 executor: &Arc<Executor>,
2620 standing_actor: &standing::StandingActor,
2621 shutdown: &Arc<Notify>,
2622 metrics: &Arc<DispatchPathMetrics>,
2623) -> Result<usize, SubcError> {
2624 let mut drained = 0;
2625 while let Ok(completion) = control_completion_rx.try_recv() {
2626 process_route_bind_completion(
2627 writer_tx,
2628 completion,
2629 routes,
2630 root_channels,
2631 session_identity,
2632 push_buffer,
2633 live_roots,
2634 pending_binds,
2635 installed_route_epochs,
2636 executor,
2637 standing_actor,
2638 shutdown,
2639 metrics,
2640 )
2641 .await?;
2642 drained += 1;
2643 }
2644 Ok(drained)
2645}
2646
2647async fn run_module_loop<R, W>(
2651 mut read: R,
2652 mut write: W,
2653 connection_file_path: &Path,
2654 shared_app: Arc<App>,
2655 executor: Arc<Executor>,
2656 dispatch: DispatchFn,
2657 user_config_path: Option<PathBuf>,
2658 allow_native_passthrough: bool,
2659 tool_response_body_limit: usize,
2660) -> Result<ModuleLoopExit, SubcError>
2661where
2662 R: AsyncRead + Unpin + Send + 'static,
2663 W: AsyncWrite + Unpin + Send + 'static,
2664{
2665 let hello = ModuleHelloBody {
2669 manifest: build_manifest(),
2670 protocol_ver: PROTOCOL_VERSION,
2671 control_ops: control_ops(),
2672 launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2673 };
2674 let hello_frame = Frame::build(
2675 FrameType::Hello,
2676 control_flags(),
2677 0,
2678 0,
2679 HELLO_CORR,
2680 serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2681 )
2682 .map_err(SubcError::FrameBuild)?;
2683 write_frame(&mut write, &hello_frame)
2684 .await
2685 .map_err(SubcError::FrameIo)?;
2686
2687 match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2689 None => return Err(SubcError::ClosedBeforeHelloAck),
2690 Some(frame) => match frame.header.ty {
2691 FrameType::HelloAck => {
2692 log::info!("subc attach: registered (HelloAck received)");
2693 }
2694 FrameType::Error => {
2695 let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2696 return Err(SubcError::HelloRejected { body });
2697 }
2698 other => return Err(SubcError::UnexpectedFrame { ty: other }),
2699 },
2700 }
2701
2702 let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2703 let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2704 let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2705 let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2712 let reader_task = spawn_reader_task(read, reader_tx);
2713 let shutdown = Arc::new(Notify::new());
2714 let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2723 let mut next_maintenance_at = next_drain_at;
2724 let standing_actor =
2725 standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
2726 standing_actor.reconcile_at_startup();
2729 let mut next_standing_pass_at = tokio::time::Instant::now();
2730 #[cfg(any(target_os = "macos", target_os = "linux"))]
2733 let mut last_slack_relief: Option<std::time::Instant> = None;
2734 let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2735 let (bash_deferred_tx, mut bash_deferred_rx) =
2736 mpsc::channel::<bash::BashDeferredCompletion>(256);
2737 let (deferred_response_tx, mut deferred_response_rx) =
2738 mpsc::unbounded_channel::<PendingSubcResponse>();
2739 let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2740 let (control_completion_tx, mut control_completion_rx) =
2741 mpsc::channel::<RouteBindCompletion>(256);
2742 let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2743 let lossy_overflow = Arc::new(push::LossyOverflow::default());
2744 let lossy_seq = Arc::new(AtomicU64::new(0));
2745 let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2746 let (fleet_status_client, fleet_status_task) =
2747 spawn_fleet_status_dial(connection_file_path, 64);
2748 let push_senders = PushSenders {
2749 lossy_tx,
2750 reliable_tx,
2751 lossy_overflow: Arc::clone(&lossy_overflow),
2752 lossy_seq,
2753 fleet_status_client: fleet_status_client.clone(),
2754 };
2755 let connection_cancel = PersistentCancelSignal::new();
2756 let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2757 let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2758 let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2759 let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2760 let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2761 let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2762 let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2763 let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2764 HashMap::new();
2765 let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2766 let mut retry_buffer: RetryBuffer = HashMap::new();
2767 let mut reclaimed_routes = ReclaimedRoutes::default();
2768 let mut completed_tasks = push::CompletedTaskIds::default();
2769 let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2770 let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2771 let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2772 let mut next_bash_ask_corr: u64 = 1;
2773 let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2774 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2775 let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
2776 let mut pending_responses = PendingSubcResponses::default();
2777 let health_rollup_cache = HealthRollupCache::new();
2778 health_rollup_cache.refresh(&executor, &shared_app);
2779 let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2780
2781 let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
2782 shared_app.set_open_route_count(routes.len());
2783 if tokio::time::Instant::now() >= next_health_rollup_at {
2784 health_rollup_cache.refresh(&executor, &shared_app);
2785 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2786 }
2787 crate::logging::perf_tick(Some(&executor));
2788 dispatch_path_metrics.mark_frame_loop_tick();
2789 let ready_inspects = pending_responses.poll_ready(executor.as_ref());
2790 for resolved in ready_inspects {
2791 if let Err(error) = deliver_resolved_subc_response(
2792 &writer_tx,
2793 resolved,
2794 &routes,
2795 &mut live_roots,
2796 executor.as_ref(),
2797 &active_tool_calls,
2798 &shutdown,
2799 &dispatch_path_metrics,
2800 tool_response_body_limit,
2801 )
2802 .await
2803 {
2804 break 'module_loop Err(error);
2805 }
2806 }
2807 if let Err(error) = expire_pending_bash_asks(
2808 &writer_tx,
2809 &mut pending_bash_asks,
2810 &routes,
2811 &mut live_roots,
2812 &mut route_bash_cancels,
2813 &shutdown,
2814 &dispatch_path_metrics,
2815 )
2816 .await
2817 {
2818 break Err(error);
2819 }
2820
2821 match drain_pending_route_bind_completions(
2825 &mut control_completion_rx,
2826 &writer_tx,
2827 &mut routes,
2828 &mut root_channels,
2829 &mut session_identity,
2830 &mut push_buffer,
2831 &mut live_roots,
2832 &mut pending_binds,
2833 &mut installed_route_epochs,
2834 &executor,
2835 &standing_actor,
2836 &shutdown,
2837 &dispatch_path_metrics,
2838 )
2839 .await
2840 {
2841 Ok(drained) => {
2842 if drained > 0 {
2843 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2844 health_rollup_cache.refresh(&executor, &shared_app);
2845 next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2846 }
2847 }
2848 Err(error) => break Err(error),
2849 }
2850
2851 if tokio::time::Instant::now() >= next_drain_at {
2852 push::emit_bg_event_wakes(
2853 &writer_tx,
2854 &dispatch_path_metrics,
2855 &bg_subs,
2856 &mut bg_wake_pending,
2857 );
2858 warn_slow_pending_binds(&mut pending_binds, &executor);
2859 warn_slow_running_interactive_jobs(&executor);
2860 if let Err(error) = expire_overdue_route_binds(
2861 &writer_tx,
2862 &executor,
2863 &mut pending_binds,
2864 &mut installed_route_epochs,
2865 &dispatch_path_metrics,
2866 )
2867 .await
2868 {
2869 break Err(error);
2870 }
2871
2872 let retried = push::drain_retry_buffers_for_bound_routes(
2873 &writer_tx,
2874 &dispatch_path_metrics,
2875 &routes,
2876 &mut retry_buffer,
2877 );
2878 if retried > 0 {
2879 log::debug!(
2880 "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2881 );
2882 }
2883
2884 next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2885 }
2886
2887 let overflow_batch = lossy_overflow.drain();
2893 if !overflow_batch.is_empty() {
2894 let (_, deferred) = push::drain_reliable_push_turn(
2895 &writer_tx,
2896 &dispatch_path_metrics,
2897 &routes,
2898 &root_channels,
2899 &session_identity,
2900 &mut retry_buffer,
2901 &mut push_buffer,
2902 &mut completed_tasks,
2903 &bg_sub_by_session,
2904 &mut bg_wake_pending,
2905 &mut bg_wake_epoch,
2906 &mut reliable_rx,
2907 None,
2908 );
2909 if deferred {
2910 tokio::task::yield_now().await;
2911 }
2912
2913 let mut batch = Vec::new();
2914 while let Ok(item) = lossy_rx.try_recv() {
2915 batch.push(item);
2916 }
2917 batch.extend(overflow_batch);
2918 push::process_lossy_push_envelope_batch(
2919 &writer_tx,
2920 &dispatch_path_metrics,
2921 &routes,
2922 &root_channels,
2923 &completed_tasks,
2924 batch,
2925 );
2926 }
2927
2928 tokio::select! {
2929 biased;
2930 Some(completion) = control_completion_rx.recv() => {
2931 if let Err(error) = process_route_bind_completion(
2932 &writer_tx,
2933 completion,
2934 &mut routes,
2935 &mut root_channels,
2936 &mut session_identity,
2937 &mut push_buffer,
2938 &mut live_roots,
2939 &mut pending_binds,
2940 &mut installed_route_epochs,
2941 &executor,
2942 &standing_actor,
2943 &shutdown,
2944 &dispatch_path_metrics,
2945 )
2946 .await
2947 {
2948 break Err(error);
2949 }
2950 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2951 next_health_rollup_at = tokio::time::Instant::now();
2952 }
2953 _ = shutdown.notified() => {
2954 log::warn!("subc attach: fatal executor response requested teardown");
2955 break Ok(ModuleLoopExit::SkipSearchFlush);
2956 }
2957 maybe_frame = reader_rx.recv() => {
2958 let frame = match maybe_frame {
2959 None => {
2960 log::info!("subc attach: daemon closed connection");
2961 break Ok(ModuleLoopExit::Graceful);
2962 }
2963 Some(Err(error)) => break Err(error),
2964 Some(Ok(frame)) => frame,
2965 };
2966 let phase_trace = frame.phase_trace;
2967 let frame = frame.frame;
2968
2969 if !ingress_route_should_be_processed(
2970 &installed_route_epochs,
2971 &reclaimed_routes,
2972 &frame,
2973 ) {
2974 log::debug!(
2975 "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2976 frame.header.ty,
2977 frame.header.channel,
2978 frame.header.epoch
2979 );
2980 continue;
2981 }
2982
2983 match frame.header.ty {
2984 FrameType::Ping if frame.header.channel == 0 => {
2985 let pong = match Frame::build_with_version(
2986 frame.header.ver,
2987 FrameType::Pong,
2988 frame.header.flags,
2989 0,
2990 0,
2991 frame.header.corr,
2992 Vec::new(),
2993 ) {
2994 Ok(pong) => pong,
2995 Err(error) => break Err(SubcError::FrameBuild(error)),
2996 };
2997 if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2998 break Err(error);
2999 }
3000 }
3001 FrameType::Goodbye if frame.header.channel == 0 => {
3002 log::info!("subc attach: received channel-0 Goodbye");
3003 break Ok(ModuleLoopExit::Graceful);
3004 }
3005 FrameType::Goodbye => {
3006 let channel = route_key(frame.header.channel, frame.header.epoch);
3007 if let Err(error) = teardown_installed_route(
3008 &writer_tx,
3009 &dispatch_path_metrics,
3010 &executor,
3011 channel,
3012 "Goodbye",
3013 None,
3014 &mut installed_route_epochs,
3015 &mut routes,
3016 &mut root_channels,
3017 &mut bg_subs,
3018 &mut bg_sub_by_session,
3019 &mut bg_wake_pending,
3020 &mut pending_bash_asks,
3021 &mut live_roots,
3022 &mut route_bash_cancels,
3023 &active_tool_calls,
3024 &mut pending_responses,
3025 &mut pending_binds,
3026 &mut retry_buffer,
3027 &mut push_buffer,
3028 &shutdown,
3029 tool_response_body_limit,
3030 )
3031 .await
3032 {
3033 break Err(error);
3034 }
3035 }
3036 FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3037 if let Err(error) = handle_bash_elicitation_reply(
3038 &writer_tx,
3039 &frame,
3040 &mut pending_bash_asks,
3041 &routes,
3042 &mut live_roots,
3043 &executor,
3044 &shutdown,
3045 &bash_deferred_tx,
3046 &bash_poll_touch_tx,
3047 &dispatch_path_metrics,
3048 &mut route_bash_cancels,
3049 dispatch,
3050 )
3051 .await
3052 {
3053 break Err(error);
3054 }
3055 }
3056 FrameType::Request if frame.header.channel == 0 => {
3057 if let Err(error) = handle_control_request(
3058 &writer_tx,
3059 &frame,
3060 &shared_app,
3061 &executor,
3062 &mut live_roots,
3063 &mut pending_binds,
3064 &mut installed_route_epochs,
3065 &mut routes,
3066 &mut root_channels,
3067 &mut bg_subs,
3068 &mut bg_sub_by_session,
3069 &mut bg_wake_pending,
3070 &mut pending_bash_asks,
3071 &mut route_bash_cancels,
3072 &active_tool_calls,
3073 &mut pending_responses,
3074 &mut retry_buffer,
3075 &mut push_buffer,
3076 &shutdown,
3077 &control_completion_tx,
3078 &dispatch_path_metrics,
3079 &health_rollup_cache,
3080 &push_senders,
3081 dispatch,
3082 user_config_path.as_deref(),
3083 tool_response_body_limit,
3084 )
3085 .await
3086 {
3087 break Err(error);
3088 }
3089 }
3090 FrameType::Request => {
3091 if let Err(error) = handle_tool_call(
3092 &writer_tx,
3093 &frame,
3094 phase_trace,
3095 &routes,
3096 &pending_binds,
3097 &mut live_roots,
3098 &executor,
3099 &active_tool_calls,
3100 &pending_deferred_setups,
3101 &shutdown,
3102 &connection_cancel,
3103 &bash_deferred_tx,
3104 &bash_poll_touch_tx,
3105 &dispatch_path_metrics,
3106 &mut route_bash_cancels,
3107 &mut pending_bash_asks,
3108 &mut next_bash_ask_corr,
3109 &mut bg_subs,
3110 &mut bg_sub_by_session,
3111 &mut bg_wake_pending,
3112 &mut bg_wake_epoch,
3113 dispatch,
3114 &deferred_response_tx,
3115 allow_native_passthrough,
3116 tool_response_body_limit,
3117 )
3118 .await
3119 {
3120 break Err(error);
3121 }
3122 }
3123 FrameType::Cancel => {
3124 let channel = route_key(frame.header.channel, frame.header.epoch);
3125 cancel_active_tool_call(
3126 &active_tool_calls,
3127 executor.as_ref(),
3128 channel,
3129 frame.header.corr,
3130 "Cancel frame",
3131 );
3132 pending_responses.cancel_request(channel, frame.header.corr);
3133 if bg_subs.contains_key(&channel) {
3134 if let Err(error) = end_bg_subscription(
3135 &writer_tx,
3136 &dispatch_path_metrics,
3137 &mut bg_subs,
3138 &mut bg_sub_by_session,
3139 &mut bg_wake_pending,
3140 channel,
3141 routes.get(&channel),
3142 "cancel",
3143 )
3144 .await
3145 {
3146 break Err(error);
3147 }
3148 }
3149 if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3150 &writer_tx,
3151 &mut pending_bash_asks,
3152 channel,
3153 frame.header.corr,
3154 &routes,
3155 &mut live_roots,
3156 &mut route_bash_cancels,
3157 &shutdown,
3158 &dispatch_path_metrics,
3159 )
3160 .await
3161 {
3162 break Err(error);
3163 }
3164 }
3165 _ => {}
3169 }
3170 }
3171 Some(pending) = deferred_response_rx.recv() => {
3172 if routes.contains_key(&pending.route)
3173 && active_tool_call_is_registered(
3174 &active_tool_calls,
3175 pending.route,
3176 pending.corr,
3177 )
3178 {
3179 pending_responses.register(pending);
3180 } else {
3181 if let Some(cancellation) = &pending.pending.cancellation {
3182 cancellation.request_cancel();
3183 }
3184 finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3185 }
3186 }
3187 Some((root_id, frame)) = reliable_rx.recv() => {
3188 let (_, deferred) = push::drain_reliable_push_turn(
3192 &writer_tx,
3193 &dispatch_path_metrics,
3194 &routes,
3195 &root_channels,
3196 &session_identity,
3197 &mut retry_buffer,
3198 &mut push_buffer,
3199 &mut completed_tasks,
3200 &bg_sub_by_session,
3201 &mut bg_wake_pending,
3202 &mut bg_wake_epoch,
3203 &mut reliable_rx,
3204 Some((root_id, frame)),
3205 );
3206 if deferred {
3207 tokio::task::yield_now().await;
3208 }
3209 }
3210 Some((order, root_id, frame)) = lossy_rx.recv() => {
3211 let (_, deferred) = push::drain_reliable_push_turn(
3215 &writer_tx,
3216 &dispatch_path_metrics,
3217 &routes,
3218 &root_channels,
3219 &session_identity,
3220 &mut retry_buffer,
3221 &mut push_buffer,
3222 &mut completed_tasks,
3223 &bg_sub_by_session,
3224 &mut bg_wake_pending,
3225 &mut bg_wake_epoch,
3226 &mut reliable_rx,
3227 None,
3228 );
3229 if deferred {
3230 tokio::task::yield_now().await;
3231 }
3232
3233 let mut batch = vec![(order, root_id, frame)];
3240 while let Ok(item) = lossy_rx.try_recv() {
3241 batch.push(item);
3242 }
3243 batch.extend(lossy_overflow.drain());
3244 push::process_lossy_push_envelope_batch(
3245 &writer_tx,
3246 &dispatch_path_metrics,
3247 &routes,
3248 &root_channels,
3249 &completed_tasks,
3250 batch,
3251 );
3252 }
3253 Some(done) = bash_deferred_rx.recv() => {
3254 decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3255 if let Err(error) = bash::handle_bash_deferred_completion(
3256 &writer_tx,
3257 done,
3258 &routes,
3259 &mut live_roots,
3260 &mut route_bash_cancels,
3261 &shutdown,
3262 &dispatch_path_metrics,
3263 )
3264 .await
3265 {
3266 break Err(error);
3267 }
3268 }
3269 Some(root_id) = bash_poll_touch_rx.recv() => {
3270 decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3271 if let Some(meta) = live_roots.get_mut(&root_id) {
3272 meta.note_activity();
3273 }
3274 }
3275 Some(completion) = maintenance_rx.recv() => {
3276 decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3277 let root_id = completion.root_id.clone();
3278 let response = completion.response;
3279 let response_is_fatal = response_is_fatal_panic(&response);
3280 let bind_pending = pending_binds
3281 .values()
3282 .any(|pending| pending.bind_root_id == root_id);
3283 let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3284 let defer_requeue = meta.unbound_quiesced || bind_pending;
3285 note_maintenance_completion(
3286 meta,
3287 completion.requeue_kind,
3288 response_is_fatal,
3289 defer_requeue,
3290 );
3291 should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3292 } else {
3293 false
3294 };
3295 if requiesce {
3296 quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3297 }
3298 push::clear_stale_bg_wakes_for_empty_sessions(
3299 &root_id,
3300 &completion.empty_bg_sessions,
3301 &bg_sub_by_session,
3302 &mut bg_wake_pending,
3303 &bg_wake_epoch,
3304 );
3305 if response_is_fatal {
3306 if let Some(meta) = live_roots.get_mut(&root_id) {
3307 meta.maintenance_poisoned = true;
3308 }
3309 log::warn!(
3310 "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3311 );
3312 }
3313 }
3314 _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3315 }
3318 _ = tokio::time::sleep_until(next_drain_at) => {
3319 }
3322 _ = tokio::time::sleep_until(next_maintenance_at) => {
3323 crate::logging::maybe_sweep_logs();
3328 let reaped_lsp_children = shared_app
3329 .lsp_child_registry()
3330 .reap_children_with_gone_cwd_or_reclaimed_root();
3331 if reaped_lsp_children > 0 {
3332 log::warn!(
3333 "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3334 );
3335 }
3336 let reap = reap_idle_roots(
3337 Instant::now(),
3338 &mut live_roots,
3339 &pending_binds,
3340 &root_channels,
3341 &executor,
3342 &dispatch_path_metrics,
3343 );
3344 for root_id in &reap.forgotten_deleted_roots {
3345 purge_deleted_root_residents(
3346 root_id,
3347 &mut routes,
3348 &mut root_channels,
3349 &mut installed_route_epochs,
3350 &mut route_bash_cancels,
3351 &active_tool_calls,
3352 executor.as_ref(),
3353 &mut retry_buffer,
3354 &mut reclaimed_routes,
3355 &mut session_identity,
3356 &mut push_buffer,
3357 &mut bg_subs,
3358 &mut bg_sub_by_session,
3359 &mut bg_wake_pending,
3360 &mut bg_wake_epoch,
3361 &mut pending_bash_asks,
3362 &dispatch_path_metrics,
3363 );
3364 }
3365 if reap.evicted > 0 {
3366 log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3367 }
3368 submit_due_maintenance_jobs(
3369 &executor,
3370 &mut live_roots,
3371 &pending_binds,
3372 &bg_sub_by_session,
3373 &bg_wake_pending,
3374 &bg_wake_epoch,
3375 &maintenance_tx,
3376 &dispatch_path_metrics,
3377 );
3378 if tokio::time::Instant::now() >= next_standing_pass_at {
3379 standing_actor.tick();
3380 next_standing_pass_at = tokio::time::Instant::now()
3381 + standing::STANDING_MAINTENANCE_INTERVAL;
3382 }
3383 #[cfg(any(target_os = "macos", target_os = "linux"))]
3390 {
3391 let now_std = std::time::Instant::now();
3392 if crate::memory::spawn_allocator_slack_relief_if_due(
3393 last_slack_relief,
3394 now_std,
3395 ) {
3396 last_slack_relief = Some(now_std);
3397 }
3398 }
3399 next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3400 }
3401 }
3402 };
3403
3404 shared_app.set_open_route_count(0);
3405
3406 connection_cancel.cancel();
3407 cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3408 let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3409 while pending_deferred_setups.load(Ordering::SeqCst) != 0
3410 && tokio::time::Instant::now() < setup_drain_deadline
3411 {
3412 tokio::select! {
3413 biased;
3414 Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3415 _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3416 }
3417 }
3418 if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3419 log::warn!(
3420 "subc attach: timed out waiting for deferred response setup registration during shutdown"
3421 );
3422 }
3423 while let Ok(pending) = deferred_response_rx.try_recv() {
3424 pending_responses.register(pending);
3425 }
3426 for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3427 if let Err(error) = deliver_resolved_subc_response(
3428 &writer_tx,
3429 resolved,
3430 &routes,
3431 &mut live_roots,
3432 executor.as_ref(),
3433 &active_tool_calls,
3434 &shutdown,
3435 &dispatch_path_metrics,
3436 tool_response_body_limit,
3437 )
3438 .await
3439 {
3440 log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3441 }
3442 }
3443 quiesce_connection_roots(
3446 &mut live_roots,
3447 &mut pending_binds,
3448 &mut routes,
3449 &mut root_channels,
3450 &mut installed_route_epochs,
3451 &mut route_bash_cancels,
3452 &active_tool_calls,
3453 &executor,
3454 );
3455
3456 fleet_status_client.set_route_live(false);
3457 fleet_status_task.abort();
3458 let _ = fleet_status_task.await;
3459
3460 let mut loop_result = loop_result;
3461 if !pending_bash_asks.is_empty() {
3462 let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3463 if let Err(error) = settle_all_pending_bash_asks(
3464 &writer_tx,
3465 &mut pending_bash_asks,
3466 &no_routes,
3467 &mut live_roots,
3468 &mut route_bash_cancels,
3469 &shutdown,
3470 &dispatch_path_metrics,
3471 )
3472 .await
3473 {
3474 loop_result = loop_result.and(Err(error));
3475 }
3476 }
3477
3478 reader_task.abort();
3481 drop(writer_tx);
3482 let writer_result = finish_writer_task(writer_task).await;
3483 loop_result.and_then(|exit| writer_result.map(|_| exit))
3484}
3485
3486fn spawn_writer_task<W>(
3487 mut write: W,
3488 mut rx: mpsc::Receiver<WriterFrame>,
3489 metrics: Arc<DispatchPathMetrics>,
3490) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3491where
3492 W: AsyncWrite + Unpin + Send + 'static,
3493{
3494 tokio::spawn(async move {
3495 let mut write_buffer = Vec::new();
3496 while let Some(mut queued) = rx.recv().await {
3497 let measure = queued.tool_response_trace.is_some();
3498 let dequeued = measure.then(Instant::now);
3499 metrics.writer_active.store(true, Ordering::Relaxed);
3500 decrement_counted_channel(&metrics.writer_queued);
3501 let write_timing = write_frame_contiguous(
3502 &mut write,
3503 queued.frame(),
3504 queued.body(),
3505 &mut write_buffer,
3506 measure,
3507 )
3508 .await;
3509 metrics.writer_active.store(false, Ordering::Relaxed);
3510 let write_timing = write_timing?;
3511
3512 if let (Some(trace), Some(dequeued), Some(write_timing)) =
3513 (queued.tool_response_trace.take(), dequeued, write_timing)
3514 {
3515 if let Some(completed) = trace.finish(
3516 dequeued,
3517 write_timing.write_started,
3518 write_timing.write_finished,
3519 write_timing.frame_bytes,
3520 ) {
3521 log_ctx::with_session(Some(completed.session), || {
3522 crate::logging::note_tool_call_trace(
3523 &completed.name,
3524 &completed.root,
3525 completed.channel,
3526 completed.corr,
3527 completed.phases,
3528 );
3529 });
3530 }
3531 }
3532 }
3533 Ok(())
3534 })
3535}
3536
3537struct FrameWriteTiming {
3538 write_started: Instant,
3539 write_finished: Instant,
3540 frame_bytes: usize,
3541}
3542
3543async fn write_frame_contiguous<W>(
3547 writer: &mut W,
3548 frame: &Frame,
3549 body: &[u8],
3550 buffer: &mut Vec<u8>,
3551 measure: bool,
3552) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3553where
3554 W: AsyncWrite + Unpin,
3555{
3556 if frame.header.len as usize != body.len() {
3557 return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3558 header_len: frame.header.len,
3559 body_len: body.len(),
3560 });
3561 }
3562
3563 let header = frame.header.encode();
3564 buffer.clear();
3565 buffer.reserve(header.len() + body.len());
3566 buffer.extend_from_slice(&header);
3567 buffer.extend_from_slice(body);
3568 let write_started = measure.then(Instant::now);
3569 writer
3570 .write_all(buffer)
3571 .await
3572 .map_err(subc_transport::FrameIoError::Io)?;
3573 Ok(write_started.map(|write_started| FrameWriteTiming {
3574 write_started,
3575 write_finished: Instant::now(),
3576 frame_bytes: buffer.len(),
3577 }))
3578}
3579
3580fn spawn_reader_task<R>(
3581 mut read: R,
3582 tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3583) -> JoinHandle<()>
3584where
3585 R: AsyncRead + Unpin + Send + 'static,
3586{
3587 tokio::spawn(async move {
3588 loop {
3589 match read_frame(&mut read).await {
3590 Ok(Some(frame)) => {
3591 let decoded = DecodedFrame {
3592 frame,
3593 phase_trace: PhaseTrace::new(Instant::now()),
3594 };
3595 if tx.send(Ok(decoded)).await.is_err() {
3596 return;
3597 }
3598 }
3599 Ok(None) => {
3600 return;
3602 }
3603 Err(error) => {
3604 if let subc_transport::FrameIoError::Io(io_error) = &error {
3611 if matches!(
3612 io_error.kind(),
3613 std::io::ErrorKind::ConnectionReset
3614 | std::io::ErrorKind::ConnectionAborted
3615 ) {
3616 log::info!(
3617 "subc attach: connection reset by daemon; treating as close"
3618 );
3619 return;
3620 }
3621 }
3622 let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3623 return;
3624 }
3625 }
3626 }
3627 })
3628}
3629
3630async fn finish_writer_task(
3631 mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3632) -> Result<(), SubcError> {
3633 match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3634 Ok(Ok(Ok(()))) => Ok(()),
3635 Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3636 Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3637 Err(_) => {
3638 writer_task.abort();
3639 Ok(())
3640 }
3641 }
3642}
3643
3644fn register_actor_for_bind(
3645 shared_app: &Arc<App>,
3646 executor: &Arc<Executor>,
3647 push_senders: &PushSenders,
3648 bind_root_id: &ProjectRootId,
3649 route_channel: u16,
3650 root_was_live: bool,
3651) -> bool {
3652 if executor.actor_registered(bind_root_id) {
3653 log::debug!(
3654 "subc attach: reusing actor for route {} root {}",
3655 route_channel,
3656 bind_root_id.as_path().display()
3657 );
3658 return false;
3659 }
3660
3661 if root_was_live {
3662 log::warn!(
3663 "subc attach: recreating missing actor for live root {} on route {}",
3664 bind_root_id.as_path().display(),
3665 route_channel
3666 );
3667 }
3668
3669 let actor_ctx = Arc::new(AppContext::from_app(
3670 Arc::clone(shared_app),
3671 Config::default(),
3672 ));
3673 install_bash_compressor(&actor_ctx);
3674 actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3675 actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3676 push_senders.clone(),
3677 bind_root_id.clone(),
3678 )));
3679 let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3680 drop(actor_ctx);
3681 if inserted {
3682 log::debug!(
3686 "subc attach: registered actor for route {} root {}",
3687 route_channel,
3688 bind_root_id.as_path().display()
3689 );
3690 } else {
3691 log::debug!(
3692 "subc attach: actor appeared while binding route {} root {}; reusing it",
3693 route_channel,
3694 bind_root_id.as_path().display()
3695 );
3696 }
3697 inserted
3698}
3699
3700fn rollback_pending_bind_actor(
3701 executor: &Arc<Executor>,
3702 live_roots: &HashMap<ProjectRootId, RootMeta>,
3703 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3704 root_id: &ProjectRootId,
3705 inserted_new_actor: bool,
3706) {
3707 if !inserted_new_actor || live_roots.contains_key(root_id) {
3708 return;
3709 }
3710
3711 if let Some((route, pending)) = pending_binds
3712 .iter_mut()
3713 .find(|(_, pending)| &pending.bind_root_id == root_id)
3714 {
3715 pending.inserted_new_actor = true;
3716 log::debug!(
3717 "subc attach: transferred rollback ownership for root {} to pending route {}",
3718 root_id.as_path().display(),
3719 route
3720 );
3721 return;
3722 }
3723
3724 executor.remove_actor(root_id);
3725}
3726
3727fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3728 match response.data.get("code").and_then(|code| code.as_str()) {
3729 Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3734 Some("cache_key_probe_failed") => "cache_key_probe_failed",
3738 Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3742 _ => "config_divergence",
3743 }
3744}
3745
3746fn queue_post_bind_configure_and_completion_maintenance(
3747 root_id: &ProjectRootId,
3748 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3749) {
3750 let Some(meta) = live_roots.get_mut(root_id) else {
3751 return;
3752 };
3753 if meta.maintenance_poisoned || meta.maintenance_pending {
3754 return;
3755 }
3756
3757 meta.maintenance_pending = true;
3758 meta.maintenance_queued_kinds
3759 .push_back(MaintenanceDrainKind::ConfigureTail);
3760 meta.maintenance_queued_kinds
3761 .push_back(MaintenanceDrainKind::CompletionDrains);
3762}
3763
3764#[allow(clippy::too_many_arguments)]
3765async fn handle_route_bind_completion(
3766 tx: &WriterSender,
3767 completion: RouteBindCompletion,
3768 routes: &mut HashMap<RouteChannel, RouteIdentity>,
3769 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3770 session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3771 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3772 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3773 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3774 installed_route_epochs: &mut HashMap<u16, u32>,
3775 executor: &Arc<Executor>,
3776 standing_actor: &standing::StandingActor,
3777 shutdown: &Arc<Notify>,
3778 metrics: &Arc<DispatchPathMetrics>,
3779) -> Result<(), SubcError> {
3780 let route_id = completion.route;
3781 let Some(pending) = pending_binds.remove(&route_id) else {
3782 log::warn!(
3783 "subc attach: dropping RouteBind completion for non-pending route {}",
3784 completion.route
3785 );
3786 rollback_pending_bind_actor(
3787 executor,
3788 live_roots,
3789 pending_binds,
3790 &completion.bind_root_id,
3791 completion.inserted_new_actor,
3792 );
3793 let has_pending_bind = pending_binds
3794 .values()
3795 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3796 if !root_channels
3797 .get(&completion.bind_root_id)
3798 .is_some_and(|channels| !channels.is_empty())
3799 && !has_pending_bind
3800 {
3801 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3802 }
3803 remove_installed_route(installed_route_epochs, route_id);
3804 return Ok(());
3805 };
3806
3807 if pending.bind_root_id != completion.bind_root_id {
3808 log::warn!(
3809 "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3810 completion.route,
3811 pending.bind_root_id.as_path().display(),
3812 completion.bind_root_id.as_path().display()
3813 );
3814 }
3815
3816 let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3817 if pending.cancelled {
3818 rollback_pending_bind_actor(
3819 executor,
3820 live_roots,
3821 pending_binds,
3822 &completion.bind_root_id,
3823 inserted_new_actor,
3824 );
3825 let has_pending_bind = pending_binds
3826 .values()
3827 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3828 if !root_channels
3829 .get(&completion.bind_root_id)
3830 .is_some_and(|channels| !channels.is_empty())
3831 && !has_pending_bind
3832 {
3833 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3834 }
3835 log::debug!(
3836 "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3837 completion.route,
3838 completion.bind_root_id.as_path().display()
3839 );
3840 remove_installed_route(installed_route_epochs, route_id);
3841 return Ok(());
3842 }
3843
3844 let failure = if !completion.configure_response.success {
3845 Some((
3846 &completion.configure_response,
3847 "configure failed during route bind",
3848 ))
3849 } else {
3850 None
3851 };
3852
3853 if let Some((response, fallback)) = failure {
3854 rollback_pending_bind_actor(
3855 executor,
3856 live_roots,
3857 pending_binds,
3858 &completion.bind_root_id,
3859 inserted_new_actor,
3860 );
3861 let has_pending_bind = pending_binds
3862 .values()
3863 .any(|pending| pending.bind_root_id == completion.bind_root_id);
3864 if !root_channels
3865 .get(&completion.bind_root_id)
3866 .is_some_and(|channels| !channels.is_empty())
3867 && !has_pending_bind
3868 {
3869 quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3870 }
3871 let message = response_message(response, fallback);
3872 let fatal = response_is_fatal_panic(response);
3873 let error_code = route_bind_error_code_for_configure_response(response);
3874 send_route_bind_error_parts(
3875 tx,
3876 completion.ver,
3877 completion.corr,
3878 completion.flags,
3879 error_code,
3880 &message,
3881 metrics,
3882 )
3883 .await?;
3884 remove_installed_route(installed_route_epochs, route_id);
3885 if fatal {
3886 signal_fatal_teardown(
3887 tx,
3888 Some(completion.route),
3889 completion.ver,
3890 completion.corr,
3891 shutdown,
3892 metrics,
3893 )
3894 .await;
3895 }
3896 return Ok(());
3897 }
3898
3899 remember_session_identity(session_identity, &completion.identity);
3900 let replay_key = push::ReplayKey::from_identity(&completion.identity);
3901 let bind_trust = completion.identity.trust;
3902 insert_route_channel(routes, root_channels, route_id, completion.identity);
3903 let restore_watcher = live_roots
3904 .get(&completion.bind_root_id)
3905 .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3906 live_roots
3907 .entry(completion.bind_root_id.clone())
3908 .and_modify(|meta| {
3909 meta.reactivate_bound();
3910 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3911 meta.maintenance_poisoned = false;
3912 })
3913 .or_insert_with(|| RootMeta::new(Instant::now()));
3914 if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3915 meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3916 meta.maintenance_poisoned = false;
3917 }
3918 if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3919 standing_actor.begin_session_bind(&ctx);
3922 ctx.mark_subc_bound();
3923 if restore_watcher {
3924 crate::commands::configure::ensure_project_watcher(&ctx);
3925 }
3926 }
3927
3928 let ack =
3929 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3930 let response = Frame::build_with_version(
3931 completion.ver,
3932 FrameType::Response,
3933 control_flags(),
3934 0,
3935 0,
3936 completion.corr,
3937 ack,
3938 )
3939 .map_err(SubcError::FrameBuild)?;
3940 send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3941 queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3942 let replayed = push::replay_buffered_push_frames(
3943 tx,
3944 metrics,
3945 route_id,
3946 push_buffer,
3947 &replay_key,
3948 bind_trust,
3949 );
3950 if replayed > 0 {
3951 log::debug!(
3952 "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3953 replayed,
3954 completion.route,
3955 replay_key.root.as_path().display(),
3956 replay_key.harness,
3957 replay_key.session
3958 );
3959 }
3960 log::info!(
3961 "subc attach: route {} bound to root {}",
3962 completion.route,
3963 completion.bind_root_id.as_path().display()
3964 );
3965 Ok(())
3966}
3967
3968async fn expire_overdue_route_binds(
3969 tx: &WriterSender,
3970 executor: &Arc<Executor>,
3971 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3972 installed_route_epochs: &mut HashMap<u16, u32>,
3973 metrics: &DispatchPathMetrics,
3974) -> Result<(), SubcError> {
3975 let now = Instant::now();
3976 let expired: Vec<_> = pending_binds
3977 .iter()
3978 .filter_map(|(route, pending)| {
3979 let age = now.saturating_duration_since(pending.started_at);
3980 (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3981 (
3982 *route,
3983 pending.corr,
3984 pending.ver,
3985 pending.flags,
3986 pending.bind_root_id.clone(),
3987 pending.configure_request_id.clone(),
3988 age,
3989 )
3990 })
3991 })
3992 .collect();
3993
3994 for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3995 if let Some(pending) = pending_binds.get_mut(&route) {
3996 pending.cancelled = true;
3997 pending.deadline_reported = true;
3998 let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3999 log::debug!(
4000 "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4001 );
4002 }
4003 remove_installed_route(installed_route_epochs, route);
4004 let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4005 let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4006 send_route_bind_error_parts(
4007 tx,
4008 ver,
4009 corr,
4010 flags,
4011 "actor_not_ready",
4012 &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4013 metrics,
4014 )
4015 .await?;
4016 log::warn!(
4017 "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4018 route,
4019 root_id.as_path().display(),
4020 deadline_ms,
4021 configure_request_id
4022 );
4023 }
4024
4025 Ok(())
4026}
4027
4028#[allow(clippy::too_many_arguments)]
4033async fn handle_control_request(
4034 tx: &WriterSender,
4035 frame: &Frame,
4036 shared_app: &Arc<App>,
4037 executor: &Arc<Executor>,
4038 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4039 pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4040 installed_route_epochs: &mut HashMap<u16, u32>,
4041 routes: &mut HashMap<RouteChannel, RouteIdentity>,
4042 root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4043 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4044 bg_sub_by_session: &mut BgSubsBySession,
4045 bg_wake_pending: &mut HashSet<RouteChannel>,
4046 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4047 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4048 active_tool_calls: &ActiveToolCalls,
4049 pending_responses: &mut PendingSubcResponses,
4050 retry_buffer: &mut RetryBuffer,
4051 push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4052 shutdown: &Arc<Notify>,
4053 control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4054 metrics: &Arc<DispatchPathMetrics>,
4055 health_rollup_cache: &HealthRollupCache,
4056 push_senders: &PushSenders,
4057 dispatch: DispatchFn,
4058 user_config_path: Option<&Path>,
4059 tool_response_body_limit: usize,
4060) -> Result<(), SubcError> {
4061 let request =
4062 serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4063 match request {
4064 ModuleControlRequest::RouteBind {
4065 route_channel,
4066 epoch,
4067 target: _,
4068 identity,
4069 principal,
4070 consumer_capabilities,
4071 admission_facts: _,
4072 } => {
4073 let route_id = route_key(route_channel, epoch);
4074 if epoch == 0 {
4075 return send_route_bind_error(
4076 tx,
4077 frame,
4078 "config_divergence",
4079 "route bind uses an invalid channel generation",
4080 metrics,
4081 )
4082 .await;
4083 }
4084 let mut bind_root_id = None;
4085 if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4086 if installed_epoch >= epoch {
4087 return send_route_bind_error(
4088 tx,
4089 frame,
4090 "config_divergence",
4091 "route bind generation is not newer than the installed generation",
4092 metrics,
4093 )
4094 .await;
4095 }
4096
4097 let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4098 Ok(root_id) => root_id,
4099 Err(error) => {
4100 return send_route_bind_error(
4101 tx,
4102 frame,
4103 "config_divergence",
4104 &format!("invalid route project root: {error}"),
4105 metrics,
4106 )
4107 .await;
4108 }
4109 };
4110 teardown_installed_route(
4111 tx,
4112 metrics,
4113 executor,
4114 route_key(route_channel, installed_epoch),
4115 "higher-epoch RouteBind",
4116 Some(&replacement_root),
4117 installed_route_epochs,
4118 routes,
4119 root_channels,
4120 bg_subs,
4121 bg_sub_by_session,
4122 bg_wake_pending,
4123 pending_bash_asks,
4124 live_roots,
4125 route_bash_cancels,
4126 active_tool_calls,
4127 pending_responses,
4128 pending_binds,
4129 retry_buffer,
4130 push_buffer,
4131 shutdown,
4132 tool_response_body_limit,
4133 )
4134 .await?;
4135 bind_root_id = Some(replacement_root);
4136 }
4137 if pending_binds.contains_key(&route_id) {
4138 return send_route_bind_error(
4139 tx,
4140 frame,
4141 "config_divergence",
4142 "route bind is already pending for channel",
4143 metrics,
4144 )
4145 .await;
4146 }
4147 let bind_root_id = match bind_root_id {
4148 Some(root_id) => root_id,
4149 None => match ProjectRootId::from_path(&identity.project_root) {
4150 Ok(root_id) => root_id,
4151 Err(error) => {
4152 return send_route_bind_error(
4153 tx,
4154 frame,
4155 "config_divergence",
4156 &format!("invalid route project root: {error}"),
4157 metrics,
4158 )
4159 .await;
4160 }
4161 },
4162 };
4163
4164 let request_id = format!("subc-bind-{route_channel}");
4167 let bind_project_root = identity.project_root.clone();
4168 let bind_harness = identity.harness.clone();
4169 let bind_session = identity.session.clone();
4170 let bind_trust = trust_for_bind(&bind_harness, &principal);
4171 let bind_principal_id = principal_id(&principal);
4172 let consumer_elicitation_capable = consumer_capabilities
4177 .as_ref()
4178 .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4179 log::info!(
4180 "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4181 route_channel,
4182 bind_harness,
4183 principal_label(&principal),
4184 bind_trust.label(),
4185 consumer_elicitation_capable
4186 );
4187
4188 let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4200 user_config_path,
4201 Path::new(&bind_project_root),
4202 );
4203 let config_tiers: Vec<Value> = local_tiers
4204 .iter()
4205 .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4206 .collect();
4207 let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
4208 let configure_json = json!({
4209 "id": request_id,
4210 "command": "configure",
4211 "project_root": bind_project_root,
4212 "harness": bind_harness,
4213 "session_id": bind_session.clone(),
4214 "config": config_tiers,
4215 });
4216 let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4217 Ok(req) => req,
4218 Err(error) => {
4219 return send_route_bind_error(
4220 tx,
4221 frame,
4222 "config_divergence",
4223 &format!("failed to build configure request: {error}"),
4224 metrics,
4225 )
4226 .await;
4227 }
4228 };
4229
4230 let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4231 root: bind_root_id.clone(),
4232 project_root: PathBuf::from(&bind_project_root),
4233 harness: bind_harness.clone(),
4234 session: bind_session.clone(),
4235 trust: bind_trust,
4236 spawn_principal: AuthenticatedPrincipal::RouteBind {
4237 trust: bind_trust.sandbox_trust(),
4238 route_channel,
4239 route_epoch: epoch,
4240 project_root: PathBuf::from(&bind_project_root),
4241 harness: bind_harness.clone(),
4242 session_id: bind_session.clone(),
4243 principal_id: bind_principal_id,
4244 },
4245 consumer_elicitation_capable,
4246 }));
4247 let configure_session = route_identity.session.clone();
4248 let root_was_live = live_roots.contains_key(&bind_root_id);
4249 let inserted_new_actor = register_actor_for_bind(
4250 shared_app,
4251 executor,
4252 push_senders,
4253 &bind_root_id,
4254 route_channel,
4255 root_was_live,
4256 );
4257
4258 let configure_request_id = configure_req.id.clone();
4259 installed_route_epochs.insert(route_channel, epoch);
4260 if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4261 meta.maintenance_queued_kinds.clear();
4262 meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4263 }
4264 let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4265 bind_root_id.clone(),
4266 Lane::Mutating,
4267 configure_request_id.clone(),
4268 Box::new(move |ctx| {
4269 log_ctx::with_session(Some(configure_session.clone()), || {
4270 dispatch(configure_req, ctx)
4271 })
4272 }),
4273 );
4274 pending_binds.insert(
4275 route_id,
4276 PendingBind {
4277 bind_root_id: bind_root_id.clone(),
4278 inserted_new_actor,
4279 cancelled: false,
4280 configure_request_id: configure_request_id.clone(),
4281 started_at: Instant::now(),
4282 warned_half_deadline: false,
4283 deadline_reported: false,
4284 corr: frame.header.corr,
4285 ver: frame.header.ver,
4286 flags: frame.header.flags,
4287 cancellation: configure_cancellation,
4288 },
4289 );
4290
4291 let completion_tx = control_completion_tx.clone();
4292 let completion_identity = route_identity;
4293 let completion_root = bind_root_id.clone();
4294 let completion_route_channel = route_channel;
4295 let completion_ver = frame.header.ver;
4296 let completion_corr = frame.header.corr;
4297 let completion_flags = frame.header.flags;
4298 let completion_metrics = Arc::clone(metrics);
4299 tokio::spawn(async move {
4300 let _response_task = ResponseTaskGuard::new(&completion_metrics);
4301 let configure_response =
4302 await_executor_response(configure_rx, configure_request_id.clone()).await;
4303 let completion = RouteBindCompletion {
4308 route: route_key(completion_route_channel, epoch),
4309 identity: completion_identity,
4310 bind_root_id: completion_root,
4311 inserted_new_actor,
4312 configure_response,
4313 diagnostics_on_edit,
4314 ver: completion_ver,
4315 corr: completion_corr,
4316 flags: completion_flags,
4317 };
4318 if send_counted_channel(
4319 &completion_tx,
4320 &completion_metrics.control_completion_queued,
4321 completion,
4322 )
4323 .await
4324 .is_err()
4325 {
4326 log::debug!(
4327 "subc attach: dropped RouteBind completion for route {} after loop exit",
4328 completion_route_channel
4329 );
4330 }
4331 });
4332
4333 health_rollup_cache.refresh(executor, shared_app);
4334 Ok(())
4335 }
4336 ModuleControlRequest::HealthCheck {} => {
4337 metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
4338 let report = build_health_report(
4339 health_rollup_cache,
4340 executor,
4341 pending_binds,
4342 metrics,
4343 shared_app,
4344 );
4345 let body = serde_json::to_vec(&ModuleControlResponse::from(report))
4346 .map_err(SubcError::Json)?;
4347 let response = Frame::build_with_version(
4348 frame.header.ver,
4349 FrameType::Response,
4350 frame.header.flags,
4351 0,
4352 0,
4353 frame.header.corr,
4354 body,
4355 )
4356 .map_err(SubcError::FrameBuild)?;
4357 send_frame(tx, metrics, response).await
4358 }
4359 }
4360}
4361
4362fn install_bash_compressor(ctx: &AppContext) {
4363 let filter_registry_handle = ctx.shared_filter_registry();
4365 let compress_flag = ctx.bash_compress_flag();
4366 ctx.bash_background().set_compressor_with_exit_code(
4367 move |command: &str, output: String, exit_code: Option<i32>| {
4368 if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
4369 return crate::compress::CompressionResult::new(output);
4370 }
4371 let registry_guard = match filter_registry_handle.read() {
4372 Ok(g) => g,
4373 Err(poisoned) => poisoned.into_inner(),
4374 };
4375 crate::compress::compress_with_registry_exit_code(
4376 command,
4377 &output,
4378 exit_code,
4379 ®istry_guard,
4380 )
4381 },
4382 );
4383}
4384
4385fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4386 let mut diagnostics_on_edit = false;
4387 for tier in tiers {
4388 if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4389 diagnostics_on_edit = value;
4390 }
4391 }
4392 diagnostics_on_edit
4393}
4394
4395fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4396 let stripped = strip_jsonc(doc);
4397 let value = serde_json::from_str::<Value>(&stripped).ok()?;
4398 value
4399 .get("lsp")
4400 .and_then(Value::as_object)?
4401 .get("diagnostics_on_edit")
4402 .and_then(Value::as_bool)
4403}
4404
4405async fn send_route_bind_error(
4406 tx: &WriterSender,
4407 frame: &Frame,
4408 code: &str,
4409 message: &str,
4410 metrics: &DispatchPathMetrics,
4411) -> Result<(), SubcError> {
4412 send_route_bind_error_parts(
4413 tx,
4414 frame.header.ver,
4415 frame.header.corr,
4416 frame.header.flags,
4417 code,
4418 message,
4419 metrics,
4420 )
4421 .await
4422}
4423
4424async fn send_route_bind_error_parts(
4425 tx: &WriterSender,
4426 ver: u8,
4427 corr: u64,
4428 flags: Flags,
4429 code: &str,
4430 message: &str,
4431 metrics: &DispatchPathMetrics,
4432) -> Result<(), SubcError> {
4433 let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4434 send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4435 log_route_bind_rejection(code, message);
4436 Ok(())
4437}
4438
4439fn log_route_bind_rejection(code: &str, message: &str) {
4447 const WINDOW: Duration = Duration::from_secs(60);
4448 static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4449 let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4450 let mut map = match map.try_lock() {
4451 Ok(map) => map,
4452 Err(_) => {
4454 log::warn!("subc attach: route bind rejected ({code}): {message}");
4455 return;
4456 }
4457 };
4458 let now = Instant::now();
4459 if map.len() > 512 {
4463 map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4464 }
4465 match map.get_mut(message) {
4466 Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4467 *suppressed += 1;
4468 }
4469 Some((window_start, suppressed)) => {
4470 if *suppressed > 0 {
4471 log::warn!(
4472 "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4473 *suppressed
4474 );
4475 } else {
4476 log::warn!("subc attach: route bind rejected ({code}): {message}");
4477 }
4478 *window_start = now;
4479 *suppressed = 0;
4480 }
4481 None => {
4482 log::warn!("subc attach: route bind rejected ({code}): {message}");
4483 map.insert(message.to_string(), (now, 0));
4484 }
4485 }
4486}
4487
4488async fn handle_tool_call(
4493 tx: &WriterSender,
4494 frame: &Frame,
4495 mut phase_trace: PhaseTrace,
4496 routes: &HashMap<RouteChannel, RouteIdentity>,
4497 pending_binds: &HashMap<RouteChannel, PendingBind>,
4498 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4499 executor: &Arc<Executor>,
4500 active_tool_calls: &ActiveToolCalls,
4501 pending_deferred_setups: &Arc<AtomicUsize>,
4502 shutdown: &Arc<Notify>,
4503 connection_cancel: &PersistentCancelSignal,
4504 bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4505 bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4506 metrics: &Arc<DispatchPathMetrics>,
4507 route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4508 pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4509 next_bash_ask_corr: &mut u64,
4510 bg_subs: &mut HashMap<RouteChannel, BgSub>,
4511 bg_sub_by_session: &mut BgSubsBySession,
4512 bg_wake_pending: &mut HashSet<RouteChannel>,
4513 bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4514 dispatch: DispatchFn,
4515 deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
4516 allow_native_passthrough: bool,
4517 tool_response_body_limit: usize,
4518) -> Result<(), SubcError> {
4519 let route_id = route_key(frame.header.channel, frame.header.epoch);
4520 if pending_binds.contains_key(&route_id) {
4521 let error = build_error_frame(
4522 frame.header.ver,
4523 frame.header.channel,
4524 frame.header.epoch,
4525 frame.header.corr,
4526 frame.header.flags,
4527 "route_not_bound",
4528 "route is not bound before tool call",
4529 )?;
4530 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4531 }
4532
4533 let Some(identity) = routes.get(&route_id).cloned() else {
4534 let error = build_error_frame(
4535 frame.header.ver,
4536 frame.header.channel,
4537 frame.header.epoch,
4538 frame.header.corr,
4539 frame.header.flags,
4540 "route_not_bound",
4541 "route is not bound before tool call",
4542 )?;
4543 return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4544 };
4545 let restore_watcher = live_roots
4546 .get(&identity.root)
4547 .is_some_and(|meta| meta.idle_artifacts_evicted);
4548 if let Some(meta) = live_roots.get_mut(&identity.root) {
4549 meta.reactivate_bound();
4550 }
4551 if restore_watcher {
4552 if let Some(ctx) = executor.actor_context(&identity.root) {
4553 crate::commands::configure::ensure_project_watcher(&ctx);
4554 }
4555 }
4556
4557 let route_request =
4558 serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4559 if matches!(
4560 route_request,
4561 RouteRequest::BgEvents(BgEventsRequest {
4562 op: BgEventsOp::BgEvents
4563 })
4564 ) {
4565 if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4566 metrics.record_bg_subscription_ended(
4567 &old_sub.root,
4568 &old_sub.session,
4569 route_id,
4570 "resubscribe",
4571 );
4572 push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4573 }
4574 if !identity.trust.allows_bash_observation() {
4575 bg_subs.remove(&route_id);
4576 bg_wake_pending.remove(&route_id);
4577 remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4578 let denied_sub = BgSub {
4579 corr: frame.header.corr,
4580 ver: frame.header.ver,
4581 flags: frame.header.flags,
4582 root: identity.root.clone(),
4583 session: identity.session.clone(),
4584 };
4585 metrics.record_bg_subscription_ended(
4586 &identity.root,
4587 &identity.session,
4588 route_id,
4589 "subscribe-denied",
4590 );
4591 push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4592 return Ok(());
4593 }
4594 bg_subs.insert(
4595 route_id,
4596 BgSub {
4597 corr: frame.header.corr,
4598 ver: frame.header.ver,
4599 flags: frame.header.flags,
4600 root: identity.root.clone(),
4601 session: identity.session.clone(),
4602 },
4603 );
4604 insert_bg_subscription_index(
4605 bg_sub_by_session,
4606 identity.root.clone(),
4607 identity.session.clone(),
4608 route_id,
4609 );
4610 metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4611 push::arm_bg_wake(
4612 identity.root.clone(),
4613 identity.session.clone(),
4614 route_id,
4615 bg_wake_pending,
4616 bg_wake_epoch,
4617 metrics,
4618 );
4619 return Ok(());
4620 }
4621
4622 let RouteRequest::ToolCall(call) = route_request else {
4623 unreachable!("background event subscription returned above")
4624 };
4625 let bare_name = call.name;
4626 let arguments = strip_agent_preview_arg_owned(call.arguments);
4627 let format_context = crate::subc_format::FormatContext::from_tool_call(
4628 &bare_name,
4629 &arguments,
4630 identity.project_root.as_path(),
4631 );
4632
4633 let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4634 let bind_trust = identity.trust;
4635 let diagnostics_on_edit = live_roots
4636 .get(&identity.root)
4637 .map(|meta| meta.diagnostics_on_edit)
4638 .unwrap_or(false);
4639
4640 let requests_host = bare_name == "bash"
4641 && arguments
4642 .get("sandbox")
4643 .or_else(|| {
4644 arguments
4645 .get("params")
4646 .and_then(|params| params.get("sandbox"))
4647 })
4648 .and_then(Value::as_str)
4649 == Some("host");
4650 if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4651 let response = Response::error(
4652 request_id.clone(),
4653 "sandbox_escalation_denied",
4654 "sandbox host escalation is unavailable to untrusted principals",
4655 );
4656 let text = crate::subc_format::format_response_with_context(
4657 &bare_name,
4658 &response,
4659 &format_context,
4660 );
4661 let result = ToolCallResult { text, response };
4662 let response_frame = build_tool_response_frame_with_limit(
4663 frame.header.ver,
4664 route_id,
4665 frame.header.corr,
4666 frame.header.flags,
4667 &result,
4668 bind_trust,
4669 tool_response_body_limit,
4670 )?;
4671 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4672 }
4673
4674 if matches!(bind_trust, BindTrust::Untrusted)
4675 && is_bash_family_tool(&bare_name)
4676 && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4677 {
4678 let response = bash::bash_denied_untrusted_response(request_id.clone());
4679 let text = crate::subc_format::format_response_with_context(
4680 &bare_name,
4681 &response,
4682 &format_context,
4683 );
4684 let result = ToolCallResult { text, response };
4685 let response_frame = build_tool_response_frame_with_limit(
4686 frame.header.ver,
4687 route_id,
4688 frame.header.corr,
4689 frame.header.flags,
4690 &result,
4691 bind_trust,
4692 tool_response_body_limit,
4693 )?;
4694 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4695 }
4696
4697 if !is_subc_agent_core_tool(&bare_name)
4705 && !is_subc_native_plumbing_tool(&bare_name)
4706 && !allow_native_passthrough
4707 {
4708 log::warn!(
4709 "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4710 bare_name,
4711 frame.header.channel
4712 );
4713 let response = Response::error(
4714 request_id.clone(),
4715 "unknown_tool",
4716 format!("tool {:?} is not in the AFT tool manifest", bare_name),
4717 );
4718 let text = crate::subc_format::format_response_with_context(
4719 &bare_name,
4720 &response,
4721 &format_context,
4722 );
4723 let result = ToolCallResult { text, response };
4724 let response_frame = build_tool_response_frame_with_limit(
4725 frame.header.ver,
4726 route_id,
4727 frame.header.corr,
4728 frame.header.flags,
4729 &result,
4730 bind_trust,
4731 tool_response_body_limit,
4732 )?;
4733 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4734 }
4735
4736 if bare_name == "bash" {
4737 if matches!(bind_trust, BindTrust::Untrusted) {
4738 let plan = match bash::prepare_bash_elicitation_plan(
4739 &arguments,
4740 identity.project_root.as_path(),
4741 ) {
4742 Ok(plan) => plan,
4743 Err(error) => {
4744 let response = Response::error(request_id.clone(), error.code, error.message);
4745 let text = crate::subc_format::format_response_with_context(
4746 &bare_name,
4747 &response,
4748 &format_context,
4749 );
4750 let result = ToolCallResult { text, response };
4751 let response_frame = build_tool_response_frame_with_limit(
4752 frame.header.ver,
4753 route_id,
4754 frame.header.corr,
4755 frame.header.flags,
4756 &result,
4757 bind_trust,
4758 tool_response_body_limit,
4759 )?;
4760 return send_reliable_writer_frame(
4761 tx,
4762 metrics,
4763 response_frame,
4764 "tool response",
4765 )
4766 .await;
4767 }
4768 };
4769
4770 let reverse_corr =
4771 allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4772 let ask_frame = build_bash_elicitation_request_frame(
4773 frame.header.ver,
4774 route_id,
4775 reverse_corr,
4776 frame.header.flags,
4777 &plan.command,
4778 &plan.asks,
4779 )?;
4780
4781 let meta = live_roots
4782 .entry(identity.root.clone())
4783 .or_insert_with(|| RootMeta::new(Instant::now()));
4784 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4785 meta.reactivate_bound();
4786
4787 let route_cancel =
4788 route_bash_cancels
4789 .entry(route_id)
4790 .or_insert_with(|| bash::RouteBashCancel {
4791 token: PersistentCancelSignal::new(),
4792 active_waits: 0,
4793 });
4794 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4795 let cancel = bash::BashWaitCancel {
4796 connection: connection_cancel.clone(),
4797 route: route_cancel.token.clone(),
4798 };
4799 pending_bash_asks.insert(
4800 ReverseCorrKey {
4801 route: route_id,
4802 corr: reverse_corr,
4803 },
4804 PendingBashAsk {
4805 route: route_id,
4806 tool_corr: frame.header.corr,
4807 tool_flags: frame.header.flags,
4808 tool_ver: frame.header.ver,
4809 root: identity.root.clone(),
4810 project_root: identity.project_root.clone(),
4811 session_id: identity.session.clone(),
4812 spawn_principal: identity.spawn_principal.clone(),
4813 edit_slot_survives: call.edit_slot_survives,
4814 request_id,
4815 arguments,
4816 format_context,
4817 cancel,
4818 grants: plan.grants,
4819 expires_at: Instant::now() + bash_elicitation_timeout(),
4820 },
4821 );
4822 return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4823 .await;
4824 }
4825
4826 let meta = live_roots
4827 .entry(identity.root.clone())
4828 .or_insert_with(|| RootMeta::new(Instant::now()));
4829 meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4830 meta.reactivate_bound();
4831
4832 let route_cancel =
4833 route_bash_cancels
4834 .entry(route_id)
4835 .or_insert_with(|| bash::RouteBashCancel {
4836 token: PersistentCancelSignal::new(),
4837 active_waits: 0,
4838 });
4839 route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4840 let cancel = bash::BashWaitCancel {
4841 connection: connection_cancel.clone(),
4842 route: route_cancel.token.clone(),
4843 };
4844
4845 bash::submit_deferred_bash(
4846 executor,
4847 bash_deferred_tx,
4848 bash_poll_touch_tx,
4849 metrics,
4850 dispatch,
4851 identity.root.clone(),
4852 identity.project_root.clone(),
4853 identity.session.clone(),
4854 request_id,
4855 route_id,
4856 frame.header.corr,
4857 frame.header.flags,
4858 frame.header.ver,
4859 arguments,
4860 format_context,
4861 cancel,
4862 bind_trust,
4863 identity.spawn_principal.clone(),
4864 call.edit_slot_survives,
4865 None,
4866 );
4867 return Ok(());
4868 }
4869
4870 let lane = command_lane(&bare_name);
4871 let tool_call_context = ToolCallContext {
4872 project_root: identity.project_root.clone(),
4873 session_id: Some(identity.session.clone()),
4874 request_id: request_id.clone(),
4875 diagnostics_on_edit,
4876 preview: call.preview,
4877 edit_slot_survives: call.edit_slot_survives,
4878 report_registration_downgrade: true,
4879 };
4880
4881 let uses_deferred_response_seam = bare_name == "inspect"
4882 || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
4883 if uses_deferred_response_seam {
4884 let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
4885 let response = Response::error(
4886 &request_id,
4887 "actor_not_registered",
4888 "executor actor is not registered",
4889 );
4890 let text = crate::subc_format::format_response_with_context(
4891 &bare_name,
4892 &response,
4893 &format_context,
4894 );
4895 let result = ToolCallResult { text, response };
4896 let response_frame = build_tool_response_frame_with_limit(
4897 frame.header.ver,
4898 route_id,
4899 frame.header.corr,
4900 frame.header.flags,
4901 &result,
4902 bind_trust,
4903 tool_response_body_limit,
4904 )?;
4905 return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4906 };
4907 let identity_for_run = identity.clone();
4908 let request_id_for_force = request_id.clone();
4909 let format_context_for_run = format_context.clone();
4910 let bare_name_for_run = bare_name.clone();
4911 let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
4912 phase_trace.mark_executor_submitted();
4913 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4914 phase_trace.mark_job_admitted();
4915 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4916 let run = || match prepare_tool_call(
4917 &bare_name_for_run,
4918 arguments,
4919 &format_context_for_run,
4920 &tool_call_context,
4921 ctx,
4922 Some(&mut phase_trace),
4923 ) {
4924 Err(result) => {
4925 let response = result.response;
4926 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
4927 text: result.text,
4928 phase_trace,
4929 });
4930 response
4931 }
4932 Ok(prepared) => {
4933 let outcome = if bare_name_for_run == "inspect" {
4934 crate::commands::inspect::handle_inspect_deferred_with_restriction(
4935 &prepared.request,
4936 Arc::clone(&deferred_ctx),
4937 matches!(bind_trust, BindTrust::Untrusted),
4938 )
4939 } else {
4940 crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
4941 &prepared.request,
4942 Arc::clone(&deferred_ctx),
4943 matches!(bind_trust, BindTrust::Untrusted),
4944 )
4945 };
4946 match outcome {
4947 DispatchOutcome::Deferred(pending) => {
4948 let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
4949 pending,
4950 surface_downgraded: prepared.surface_downgraded,
4951 phase_trace,
4952 });
4953 Response::success(
4954 request_id_for_force.clone(),
4955 json!({ "response_deferred": true }),
4956 )
4957 }
4958 DispatchOutcome::Immediate(response) => {
4959 phase_trace.mark_execute_done();
4960 let finalizer = |response: &mut Response| {
4961 crate::response_finalize::finalize_response_with_bg_completions(
4962 response,
4963 ctx,
4964 &identity_for_run.session,
4965 &bare_name_for_run,
4966 bind_trust.allows_bash_observation(),
4967 );
4968 };
4969 let result = finish_tool_call_response(
4970 &bare_name_for_run,
4971 &format_context_for_run,
4972 response,
4973 prepared.surface_downgraded,
4974 Some(&finalizer),
4975 Some(&mut phase_trace),
4976 );
4977 let response = result.response;
4978 let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
4979 text: result.text,
4980 phase_trace,
4981 });
4982 response
4983 }
4984 }
4985 }
4986 };
4987 if matches!(bind_trust, BindTrust::Untrusted) {
4988 ctx.with_force_restrict(&request_id_for_force, run)
4989 } else {
4990 run()
4991 }
4992 })
4993 });
4994 let deferred_setup_guard =
4995 PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
4996 let rx = submit_active_tool_call(
4997 executor.as_ref(),
4998 active_tool_calls,
4999 route_id,
5000 frame.header.corr,
5001 identity.root.clone(),
5002 lane,
5003 request_id.clone(),
5004 RouteDetachPolicy::CancelOnDetach,
5005 job,
5006 );
5007
5008 let completion_tx = tx.clone();
5009 let completion_shutdown = Arc::clone(shutdown);
5010 let completion_metrics = Arc::clone(metrics);
5011 let active_tool_calls = Arc::clone(active_tool_calls);
5012 let deferred_response_tx = deferred_response_tx.clone();
5013 let route = route_id;
5014 let corr = frame.header.corr;
5015 let flags = frame.header.flags;
5016 let ver = frame.header.ver;
5017 let root = identity.root.clone();
5018 let session_id = identity.session.clone();
5019 tokio::spawn(async move {
5020 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5021 let _deferred_setup = deferred_setup_guard;
5022 let response = await_executor_response(rx, request_id.clone()).await;
5023 match setup_rx.await {
5024 Ok(DeferredSetupOutcome::Deferred {
5025 pending,
5026 surface_downgraded,
5027 phase_trace,
5028 }) => {
5029 let pending = PendingSubcResponse {
5030 route,
5031 corr,
5032 flags,
5033 ver,
5034 root,
5035 session_id,
5036 bare_name,
5037 format_context,
5038 bind_trust,
5039 pending,
5040 surface_downgraded,
5041 phase_trace,
5042 };
5043 if let Err(error) = deferred_response_tx.send(pending) {
5044 if let Some(cancellation) = &error.0.pending.cancellation {
5045 cancellation.request_cancel();
5046 }
5047 finish_active_tool_call(&active_tool_calls, route, corr);
5048 }
5049 }
5050 Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5051 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5052 return;
5053 }
5054 let result = ToolCallResult { text, response };
5055 let fatal = response_is_fatal_panic(&result.response);
5056 match build_tool_response_frame_with_limit(
5057 ver,
5058 route,
5059 corr,
5060 flags,
5061 &result,
5062 bind_trust,
5063 tool_response_body_limit,
5064 ) {
5065 Ok(response_frame) => {
5066 let trace = ToolResponseWriteTrace::new(
5067 phase_trace,
5068 bare_name.clone(),
5069 identity.project_root.clone(),
5070 identity.session.clone(),
5071 route.channel,
5072 corr,
5073 );
5074 if let Err(error) = send_traced_tool_response_frame(
5075 &completion_tx,
5076 &completion_metrics,
5077 response_frame,
5078 trace,
5079 )
5080 .await
5081 {
5082 log::warn!(
5083 "subc attach: failed to queue deferred-seam setup response: {error}"
5084 );
5085 }
5086 }
5087 Err(error) => {
5088 log::error!(
5089 "subc attach: failed to build deferred-seam setup response: {error}"
5090 );
5091 }
5092 }
5093 if fatal {
5094 signal_fatal_teardown(
5095 &completion_tx,
5096 Some(route),
5097 ver,
5098 corr,
5099 &completion_shutdown,
5100 &completion_metrics,
5101 )
5102 .await;
5103 }
5104 }
5105 Err(_) => {
5106 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5107 return;
5108 }
5109 let text = crate::subc_format::format_response_with_context(
5110 &bare_name,
5111 &response,
5112 &format_context,
5113 );
5114 let result = ToolCallResult { text, response };
5115 if let Ok(response_frame) = build_tool_response_frame_with_limit(
5116 ver,
5117 route,
5118 corr,
5119 flags,
5120 &result,
5121 bind_trust,
5122 tool_response_body_limit,
5123 ) {
5124 let _ = send_reliable_writer_frame(
5125 &completion_tx,
5126 &completion_metrics,
5127 response_frame,
5128 "deferred setup failure",
5129 )
5130 .await;
5131 }
5132 }
5133 }
5134 });
5135 return Ok(());
5136 }
5137
5138 let bare_name_for_frame = bare_name.clone();
5139 let identity_for_run = identity.clone();
5140 let completion_session = identity.session.clone();
5141 let completion_root = identity.project_root.clone();
5142 let request_id_for_force = request_id.clone();
5143 let format_context_for_frame = format_context.clone();
5144 let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5145 phase_trace.mark_executor_submitted();
5146 let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5147 phase_trace.mark_job_admitted();
5148 log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5149 let run = || {
5150 let finalizer = |response: &mut Response| {
5151 crate::response_finalize::finalize_response_with_bg_completions(
5152 response,
5153 ctx,
5154 &identity_for_run.session,
5155 &bare_name,
5156 bind_trust.allows_bash_observation(),
5157 );
5158 };
5159 match run_tool_call(
5160 &bare_name,
5161 arguments,
5162 &format_context,
5163 &tool_call_context,
5164 ctx,
5165 &dispatch,
5166 Some(&finalizer),
5167 Some(&mut phase_trace),
5168 ) {
5169 ToolCallOutcome::Unary(result) => {
5170 let response = result.response;
5171 let _ = tool_call_tx.send(ToolCallCompletion {
5172 text: result.text,
5173 phase_trace,
5174 });
5175 response
5176 }
5177 }
5178 };
5179 if matches!(bind_trust, BindTrust::Untrusted) {
5180 ctx.with_force_restrict(&request_id_for_force, run)
5181 } else {
5182 run()
5183 }
5184 })
5185 });
5186 let rx = submit_active_tool_call(
5187 executor.as_ref(),
5188 active_tool_calls,
5189 route_id,
5190 frame.header.corr,
5191 identity.root.clone(),
5192 lane,
5193 request_id.clone(),
5194 RouteDetachPolicy::RetainForReplay,
5195 job,
5196 );
5197 let completion_tx = tx.clone();
5198 let completion_shutdown = Arc::clone(shutdown);
5199 let route = route_id;
5200 let corr = frame.header.corr;
5201 let flags = frame.header.flags;
5202 let ver = frame.header.ver;
5203 let completion_metrics = Arc::clone(metrics);
5204 let active_tool_calls = Arc::clone(active_tool_calls);
5205 tokio::spawn(async move {
5206 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5207 let response = await_executor_response(rx, request_id.clone()).await;
5208 let (text, phase_trace) = match tool_call_rx.await {
5209 Ok(completion) => (completion.text, Some(completion.phase_trace)),
5210 Err(_) => (
5211 crate::subc_format::format_response_with_context(
5212 &bare_name_for_frame,
5213 &response,
5214 &format_context_for_frame,
5215 ),
5216 None,
5217 ),
5218 };
5219 if !finish_active_tool_call(&active_tool_calls, route, corr) {
5220 return;
5221 }
5222 let result = ToolCallResult { text, response };
5223 let fatal = response_is_fatal_panic(&result.response);
5224 match build_tool_response_frame_with_limit(
5225 ver,
5226 route,
5227 corr,
5228 flags,
5229 &result,
5230 bind_trust,
5231 tool_response_body_limit,
5232 ) {
5233 Ok(response_frame) => {
5234 let send_result = if let Some(phase_trace) = phase_trace {
5235 let trace = ToolResponseWriteTrace::new(
5236 phase_trace,
5237 bare_name_for_frame,
5238 completion_root,
5239 completion_session,
5240 route.channel,
5241 corr,
5242 );
5243 send_traced_tool_response_frame(
5244 &completion_tx,
5245 &completion_metrics,
5246 response_frame,
5247 trace,
5248 )
5249 .await
5250 } else {
5251 send_reliable_writer_frame(
5252 &completion_tx,
5253 &completion_metrics,
5254 response_frame,
5255 "tool response",
5256 )
5257 .await
5258 };
5259 if let Err(error) = send_result {
5260 log::warn!("subc attach: failed to queue tool response frame: {error}");
5261 }
5262 }
5263 Err(error) => {
5264 log::error!("subc attach: failed to build tool response frame: {error}");
5265 }
5266 }
5267 if fatal {
5268 signal_fatal_teardown(
5269 &completion_tx,
5270 Some(route),
5271 ver,
5272 corr,
5273 &completion_shutdown,
5274 &completion_metrics,
5275 )
5276 .await;
5277 }
5278 });
5279 Ok(())
5280}
5281
5282fn submit_maintenance_job(
5283 executor: &Arc<Executor>,
5284 root_id: ProjectRootId,
5285 kind: MaintenanceDrainKind,
5286 bg_sessions_to_check: Vec<(String, u64)>,
5287 completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5288 metrics: &Arc<DispatchPathMetrics>,
5289) {
5290 let request_id = format!(
5291 "subc-maintenance-drain-{}-{}",
5292 kind.label(),
5293 root_id.as_path().to_string_lossy()
5294 );
5295 let response_id = request_id.clone();
5296 let completion_root_id = root_id.clone();
5297 let maintenance_generation = executor
5298 .actor_context(&root_id)
5299 .map(|ctx| ctx.configure_generation())
5300 .unwrap_or(0);
5301 let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5302 let lane = match kind {
5307 MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
5308 MaintenanceDrainKind::Watcher
5309 | MaintenanceDrainKind::Lsp
5310 | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
5311 };
5312 let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5313 let outcome = match kind {
5314 MaintenanceDrainKind::Watcher => {
5315 let drained = runtime_drain::drain_watcher_events_bounded(
5316 ctx,
5317 runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5318 );
5319 MaintenanceJobOutcome {
5320 empty_bg_sessions: Vec::new(),
5321 requeue_kind: drained.has_more.then_some(kind),
5322 }
5323 }
5324 MaintenanceDrainKind::Lsp => {
5325 let drained = runtime_drain::drain_lsp_events_bounded(
5326 ctx,
5327 runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
5328 );
5329 MaintenanceJobOutcome {
5330 empty_bg_sessions: Vec::new(),
5331 requeue_kind: drained.has_more.then_some(kind),
5332 }
5333 }
5334 MaintenanceDrainKind::ConfigureTail => {
5335 runtime_drain::drain_deferred_configure_maintenance(ctx);
5336 runtime_drain::drain_configure_warning_events(ctx);
5337 MaintenanceJobOutcome::default()
5338 }
5339 MaintenanceDrainKind::CompletionDrains => {
5340 runtime_drain::drain_search_index_events(ctx);
5341 runtime_drain::drain_callgraph_store_events(ctx);
5342 runtime_drain::drain_semantic_index_events(ctx);
5343 runtime_drain::drain_semantic_refresh_events(ctx);
5344 runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
5345 let empty_bg_sessions = bg_sessions_to_check
5346 .into_iter()
5347 .filter(|(session, _)| {
5348 !ctx.bash_background()
5349 .has_completions_for_session(Some(session.as_str()))
5350 })
5351 .collect();
5352 MaintenanceJobOutcome {
5353 empty_bg_sessions,
5354 requeue_kind: None,
5355 }
5356 }
5357 };
5358 let requeued = outcome.requeue_kind.is_some();
5359 let _ = outcome_tx.send(outcome);
5360 Response::success(
5361 response_id,
5362 json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
5363 )
5364 });
5365 let rx = match kind {
5366 MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
5367 root_id,
5368 lane,
5369 request_id.clone(),
5370 crate::executor::MaintenanceCoalesceKey::WatcherDrain,
5371 job,
5372 ),
5373 MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
5374 root_id,
5375 lane,
5376 request_id.clone(),
5377 crate::executor::MaintenanceCoalesceKey::LspDrain,
5378 job,
5379 ),
5380 MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
5381 executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
5382 }
5383 };
5384 let completion_tx = completion_tx.clone();
5385 let completion_metrics = Arc::clone(metrics);
5386 tokio::spawn(async move {
5387 let _response_task = ResponseTaskGuard::new(&completion_metrics);
5388 let response = await_executor_response(rx, request_id).await;
5389 let outcome = outcome_rx.await.unwrap_or_default();
5390 let _ = send_counted_channel(
5391 &completion_tx,
5392 &completion_metrics.maintenance_queued,
5393 MaintenanceCompletion {
5394 root_id: completion_root_id,
5395 kind,
5396 response,
5397 empty_bg_sessions: outcome.empty_bg_sessions,
5398 requeue_kind: outcome.requeue_kind,
5399 },
5400 )
5401 .await;
5402 });
5403}
5404
5405async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
5406 rx.await
5407 .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
5408}
5409
5410async fn deliver_resolved_subc_response(
5411 tx: &WriterSender,
5412 mut resolved: ResolvedSubcResponse,
5413 routes: &HashMap<RouteChannel, RouteIdentity>,
5414 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5415 executor: &Executor,
5416 active_tool_calls: &ActiveToolCalls,
5417 shutdown: &Arc<Notify>,
5418 metrics: &DispatchPathMetrics,
5419 tool_response_body_limit: usize,
5420) -> Result<(), SubcError> {
5421 let entry = &mut resolved.entry;
5422 finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
5423 if let Some(meta) = live_roots.get_mut(&entry.root) {
5424 meta.note_activity();
5425 }
5426
5427 let Some(identity) = routes.get(&entry.route) else {
5428 log::debug!(
5429 "subc attach: dropping deferred {} response {} for unbound route {}",
5430 entry.bare_name,
5431 entry.pending.request_id,
5432 entry.route
5433 );
5434 return Ok(());
5435 };
5436 let Some(ctx) = executor.actor_context(&entry.root) else {
5437 return Ok(());
5438 };
5439 entry.phase_trace.mark_execute_done();
5440 let finalizer = |response: &mut Response| {
5441 crate::response_finalize::finalize_response_with_bg_completions(
5442 response,
5443 &ctx,
5444 &entry.session_id,
5445 &entry.bare_name,
5446 entry.bind_trust.allows_bash_observation(),
5447 );
5448 };
5449 let result = finish_tool_call_response(
5450 &entry.bare_name,
5451 &entry.format_context,
5452 resolved.response,
5453 entry.surface_downgraded,
5454 Some(&finalizer),
5455 Some(&mut entry.phase_trace),
5456 );
5457 let fatal = response_is_fatal_panic(&result.response);
5458 let response_frame = build_tool_response_frame_with_limit(
5459 entry.ver,
5460 entry.route,
5461 entry.corr,
5462 entry.flags,
5463 &result,
5464 identity.trust,
5465 tool_response_body_limit,
5466 )?;
5467 let trace = ToolResponseWriteTrace::new(
5468 std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
5469 entry.bare_name.clone(),
5470 identity.project_root.clone(),
5471 entry.session_id.clone(),
5472 entry.route.channel,
5473 entry.corr,
5474 );
5475 send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
5476 if fatal {
5477 signal_fatal_teardown(
5478 tx,
5479 Some(entry.route),
5480 entry.ver,
5481 entry.corr,
5482 shutdown,
5483 metrics,
5484 )
5485 .await;
5486 }
5487 Ok(())
5488}
5489
5490async fn signal_fatal_teardown(
5491 tx: &WriterSender,
5492 route: Option<RouteChannel>,
5493 ver: u8,
5494 corr: u64,
5495 shutdown: &Arc<Notify>,
5496 metrics: &DispatchPathMetrics,
5497) {
5498 if let Some(route) = route {
5499 if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
5500 if let Err(error) = send_frame(tx, metrics, frame).await {
5501 log::warn!(
5502 "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
5503 );
5504 }
5505 }
5506 }
5507 if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
5508 if let Err(error) = send_frame(tx, metrics, frame).await {
5509 log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
5510 }
5511 }
5512 shutdown.notify_one();
5513}
5514#[derive(Debug, Deserialize)]
5515#[serde(untagged)]
5516enum RouteRequest {
5517 BgEvents(BgEventsRequest),
5518 ToolCall(ToolCallRequest),
5519}
5520
5521#[derive(Debug, Deserialize)]
5522struct BgEventsRequest {
5523 op: BgEventsOp,
5524}
5525
5526#[derive(Debug, Deserialize)]
5527#[serde(rename_all = "snake_case")]
5528enum BgEventsOp {
5529 BgEvents,
5530}
5531
5532#[derive(Debug, Deserialize)]
5533struct ToolCallRequest {
5534 name: String,
5535 #[serde(default)]
5536 arguments: Value,
5537 #[serde(default)]
5539 edit_slot_survives: Option<bool>,
5540 #[serde(default)]
5545 preview: bool,
5546}
5547
5548#[cfg(test)]
5549pub(crate) mod test_support {
5550 use super::*;
5551 use crate::bash_background::BgTaskStatus;
5552 use crate::protocol::{
5553 BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
5554 ProgressFrame, StatusChangedFrame,
5555 };
5556 use serde_json::json;
5557
5558 pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
5559 let dir = tempfile::Builder::new()
5560 .prefix(name)
5561 .tempdir()
5562 .expect("temp root");
5563 let root = ProjectRootId::from_path(dir.path()).expect("project root id");
5564 (dir, root)
5565 }
5566
5567 pub(super) fn test_ctx() -> Arc<AppContext> {
5568 Arc::new(AppContext::new(
5569 Box::new(crate::parser::TreeSitterProvider::new()),
5570 crate::config::Config::default(),
5571 ))
5572 }
5573
5574 fn inspect_context(root: &Path) -> Arc<AppContext> {
5575 let mut config = crate::config::Config::default();
5576 config.project_root = Some(root.to_path_buf());
5577 let ctx = Arc::new(AppContext::new(
5578 Box::new(crate::parser::TreeSitterProvider::new()),
5579 config,
5580 ));
5581 ctx.set_harness(crate::harness::Harness::Opencode);
5582 ctx
5583 }
5584
5585 fn inspect_request(id: &str) -> RawRequest {
5586 serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
5587 }
5588
5589 fn submit_deferred_inspect_setup(
5590 executor: &Arc<Executor>,
5591 root: &ProjectRootId,
5592 ctx: &Arc<AppContext>,
5593 request_id: &str,
5594 ) -> (PendingResponse, JobCancellation) {
5595 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5596 let request = inspect_request(request_id);
5597 let inspect_ctx = Arc::clone(ctx);
5598 let (_rx, cancellation) = executor.submit_cancellable_async(
5599 root.clone(),
5600 Lane::SerialLspStatus,
5601 request_id.to_string(),
5602 Box::new(move |_| {
5603 let DispatchOutcome::Deferred(pending) =
5604 crate::commands::inspect::handle_inspect_deferred_with_restriction(
5605 &request,
5606 inspect_ctx,
5607 true,
5608 )
5609 else {
5610 panic!("inspect setup must defer")
5611 };
5612 pending_tx.send(pending).expect("send pending inspect");
5613 Response::success("inspect-setup", json!({}))
5614 }),
5615 );
5616 let pending = pending_rx
5617 .recv_timeout(Duration::from_secs(1))
5618 .expect("inspect setup leaves the executor lane");
5619 let deadline = Instant::now() + Duration::from_secs(1);
5620 while !executor.actor_is_idle(root) {
5621 assert!(
5622 Instant::now() < deadline,
5623 "inspect setup kept lane counters live"
5624 );
5625 std::thread::sleep(Duration::from_millis(5));
5626 }
5627 (pending, cancellation)
5628 }
5629
5630 fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5631 let deadline = Instant::now() + Duration::from_secs(60);
5632 loop {
5633 if let Some(response) = (pending.poll)(ctx) {
5634 return response;
5635 }
5636 assert!(Instant::now() < deadline, "inspect terminal timed out");
5637 std::thread::sleep(Duration::from_millis(5));
5638 }
5639 }
5640
5641 fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
5642 let source_dir = root.join("src");
5643 std::fs::create_dir_all(&source_dir).expect("create source dir");
5644 std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
5645 .expect("write Cargo manifest");
5646 let source = source_dir.join("main.rs");
5647 std::fs::write(&source, "fn main() {}\n").expect("write source");
5648 let binary = root.join("cold-navigation-server");
5649 std::fs::write(&binary, b"fixture").expect("write server placeholder");
5650
5651 let ctx = inspect_context(root);
5652 ctx.lsp()
5653 .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
5654 (ctx, source)
5655 }
5656
5657 fn navigation_request(id: &str, source: &Path) -> RawRequest {
5658 serde_json::from_value(json!({
5659 "id": id,
5660 "command": "lsp_hover",
5661 "file": source.display().to_string(),
5662 "line": 1,
5663 "character": 1,
5664 }))
5665 .expect("navigation request")
5666 }
5667
5668 fn submit_deferred_navigation_setup(
5669 executor: &Arc<Executor>,
5670 root: &ProjectRootId,
5671 ctx: &Arc<AppContext>,
5672 source: &Path,
5673 request_id: &str,
5674 ) -> (PendingResponse, JobCancellation) {
5675 let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5676 let request = navigation_request(request_id, source);
5677 let navigation_ctx = Arc::clone(ctx);
5678 let (_rx, cancellation) = executor.submit_cancellable_async(
5679 root.clone(),
5680 Lane::SerialLspStatus,
5681 request_id.to_string(),
5682 Box::new(move |_| {
5683 let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
5684 handle_lsp_navigation_deferred_with_restriction(
5685 &request,
5686 navigation_ctx,
5687 true,
5688 )
5689 else {
5690 panic!("cold navigation setup must defer")
5691 };
5692 pending_tx.send(pending).expect("send pending navigation");
5693 Response::success("navigation-setup", json!({}))
5694 }),
5695 );
5696 let pending = pending_rx
5697 .recv_timeout(Duration::from_secs(1))
5698 .expect("navigation setup leaves the executor lane");
5699 let deadline = Instant::now() + Duration::from_secs(1);
5700 while !executor.actor_is_idle(root) {
5701 assert!(
5702 Instant::now() < deadline,
5703 "navigation setup kept lane counters live"
5704 );
5705 std::thread::sleep(Duration::from_millis(5));
5706 }
5707 (pending, cancellation)
5708 }
5709
5710 fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5711 let deadline = Instant::now() + Duration::from_secs(1);
5712 loop {
5713 if let Some(response) = (pending.poll)(ctx) {
5714 return response;
5715 }
5716 assert!(Instant::now() < deadline, "navigation terminal timed out");
5717 std::thread::sleep(Duration::from_millis(5));
5718 }
5719 }
5720
5721 #[test]
5722 fn deferred_inspect_releases_lane_for_bind_and_mutation() {
5723 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5724 let executor = Arc::new(Executor::new());
5725 let (dir, root) = test_root("deferred-inspect-storm");
5726 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5727 let ctx = inspect_context(dir.path());
5728 executor.register_actor(root.clone(), Arc::clone(&ctx));
5729 let (started_rx, release_tx) =
5730 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5731 let (mut pending, _cancellation) =
5732 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
5733 started_rx
5734 .recv_timeout(Duration::from_secs(1))
5735 .expect("deferred inspect body starts");
5736
5737 for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
5738 let response = executor.submit(
5739 root.clone(),
5740 Lane::Mutating,
5741 request_id.to_string(),
5742 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5743 );
5744 assert!(
5745 response
5746 .recv_timeout(Duration::from_secs(1))
5747 .expect("writer admits while inspect remains deferred")
5748 .success
5749 );
5750 }
5751 assert_eq!(
5752 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5753 1,
5754 "writer admissions must not finish the detached inspect"
5755 );
5756
5757 release_tx.send(()).expect("release inspect body");
5758 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5759 assert!(
5760 terminal.data.get("inspect_terminal").is_some(),
5761 "inspect must still produce its terminal: {:?}",
5762 terminal.data
5763 );
5764 }
5765
5766 #[test]
5767 fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
5768 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
5769 let executor = Arc::new(Executor::new());
5770 let (dir, root) = test_root("deferred-cold-navigation");
5771 let (ctx, source) = cold_navigation_context(dir.path());
5772 executor.register_actor(root.clone(), Arc::clone(&ctx));
5773 let (started_rx, _release_tx) =
5774 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
5775 let (mut pending, cancellation) = submit_deferred_navigation_setup(
5776 &executor,
5777 &root,
5778 &ctx,
5779 &source,
5780 "subc-cold-navigation",
5781 );
5782 started_rx
5783 .recv_timeout(Duration::from_secs(1))
5784 .expect("detached navigation reaches its cold-start gate");
5785
5786 for (lane, request_id) in [
5787 (Lane::PureRead, "subc-navigation-read"),
5788 (Lane::Mutating, "subc-navigation-write"),
5789 ] {
5790 let response = executor.submit(
5791 root.clone(),
5792 lane,
5793 request_id.to_string(),
5794 Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5795 );
5796 assert!(
5797 response
5798 .recv_timeout(Duration::from_secs(1))
5799 .expect("unrelated work admits while navigation remains deferred")
5800 .success
5801 );
5802 }
5803 assert_eq!(
5804 crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
5805 1,
5806 "lane admissions must not finish the detached navigation"
5807 );
5808
5809 cancellation.request_cancel();
5810 let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
5811 assert!(!terminal.success);
5812 let deadline = Instant::now() + Duration::from_secs(1);
5813 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
5814 assert!(
5815 Instant::now() < deadline,
5816 "cancelled navigation worker did not settle"
5817 );
5818 std::thread::sleep(Duration::from_millis(5));
5819 }
5820 assert!(executor.actor_is_idle(&root));
5821 }
5822
5823 #[test]
5824 fn cancelling_pending_navigation_removes_entry_without_reply() {
5825 let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
5826 let executor = Arc::new(Executor::new());
5827 let (dir, root) = test_root("cancelled-pending-navigation");
5828 let (ctx, source) = cold_navigation_context(dir.path());
5829 executor.register_actor(root.clone(), Arc::clone(&ctx));
5830 let (started_rx, _release_tx) =
5831 crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
5832 let (pending, _cancellation) = submit_deferred_navigation_setup(
5833 &executor,
5834 &root,
5835 &ctx,
5836 &source,
5837 "subc-cancel-navigation",
5838 );
5839 started_rx
5840 .recv_timeout(Duration::from_secs(1))
5841 .expect("detached navigation reaches its cancellation gate");
5842
5843 let route = RouteChannel {
5844 channel: 17,
5845 epoch: 1,
5846 };
5847 let mut registry = PendingSubcResponses::default();
5848 registry.register(PendingSubcResponse {
5849 route,
5850 corr: 71,
5851 flags: Flags::new(false, Priority::Passive, false),
5852 ver: PROTOCOL_VERSION,
5853 root: root.clone(),
5854 session_id: "navigation-cancel-session".to_string(),
5855 bare_name: "lsp_hover".to_string(),
5856 format_context: crate::subc_format::FormatContext::from_tool_call(
5857 "lsp_hover",
5858 &json!({}),
5859 dir.path(),
5860 ),
5861 bind_trust: BindTrust::FirstParty,
5862 pending,
5863 surface_downgraded: false,
5864 phase_trace: PhaseTrace::new(Instant::now()),
5865 });
5866
5867 assert!(registry.cancel_request(route, 71));
5868 assert!(registry.is_empty());
5869 let deadline = Instant::now() + Duration::from_secs(1);
5870 while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
5871 assert!(
5872 Instant::now() < deadline,
5873 "pending cancellation did not settle the detached worker"
5874 );
5875 std::thread::sleep(Duration::from_millis(5));
5876 }
5877 assert!(
5878 registry.poll_ready(executor.as_ref()).is_empty(),
5879 "a cancelled navigation must not leak a reply"
5880 );
5881 }
5882
5883 #[test]
5884 fn same_root_deferred_inspects_are_single_flight() {
5885 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5886 let executor = Arc::new(Executor::new());
5887 let (dir, root) = test_root("single-flight-deferred-inspect");
5888 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5889 let ctx = inspect_context(dir.path());
5890 executor.register_actor(root.clone(), Arc::clone(&ctx));
5891 let (started_rx, release_tx) =
5892 crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5893 let (mut first, _first_cancellation) =
5894 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
5895 started_rx
5896 .recv_timeout(Duration::from_secs(1))
5897 .expect("first inspect owns the root flight");
5898 let (mut second, second_cancellation) =
5899 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
5900
5901 assert_eq!(
5902 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5903 1,
5904 "only one detached body may run for a root"
5905 );
5906 assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
5907 second_cancellation.request_cancel();
5908 let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
5909 assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
5910 assert_eq!(
5911 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5912 1,
5913 "cancelling the queued request must not release the active flight"
5914 );
5915
5916 release_tx.send(()).expect("release first inspect");
5917 let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
5918 assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
5919 assert_eq!(
5920 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5921 0
5922 );
5923 }
5924
5925 #[test]
5926 fn route_abandonment_cancels_detached_inspect_thread() {
5927 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5928 let executor = Arc::new(Executor::new());
5929 let (dir, root) = test_root("cancelled-deferred-inspect");
5930 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5931 let ctx = inspect_context(dir.path());
5932 executor.register_actor(root.clone(), Arc::clone(&ctx));
5933 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5934 let route = RouteChannel {
5935 channel: 7,
5936 epoch: 1,
5937 };
5938 let (started_rx, _release_tx) =
5939 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5940 let (mut pending, cancellation) =
5941 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
5942 active.lock().expect("active tool call map").insert(
5943 (route, 41),
5944 ActiveToolCall {
5945 root_id: root.clone(),
5946 cancellation,
5947 detach_policy: RouteDetachPolicy::CancelOnDetach,
5948 },
5949 );
5950 started_rx
5951 .recv_timeout(Duration::from_secs(1))
5952 .expect("detached inspect reaches cancellation gate");
5953 assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
5954
5955 assert!(cancel_active_tool_call(
5956 &active,
5957 executor.as_ref(),
5958 route,
5959 41,
5960 "test route abandonment"
5961 ));
5962 let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5963 assert_eq!(terminal.data["inspect_terminal"], "interrupted");
5964 assert_eq!(
5965 crate::commands::inspect::deferred_inspect_root_count_for_test(),
5966 0
5967 );
5968 assert!(executor.actor_is_idle(&root));
5969 assert!(active.lock().expect("active tool call map").is_empty());
5970 let restriction_deadline = Instant::now() + Duration::from_secs(1);
5971 while ctx.request_force_restrict("subc-inspect-abandoned") {
5972 assert!(
5973 Instant::now() < restriction_deadline,
5974 "detached force-restrict guard leaked"
5975 );
5976 std::thread::sleep(Duration::from_millis(5));
5977 }
5978 }
5979
5980 #[test]
5981 fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
5982 let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
5987 pool_size: 3,
5988 read_cap: 2,
5989 actor_cap: 2,
5990 heavy_permits: 1,
5991 drr_quantum: 1,
5992 }));
5993 let (_dir, root) = test_root("cancelled-interactive-search");
5994 executor.register_actor(root.clone(), test_ctx());
5995 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5996 let route = RouteChannel {
5997 channel: 9,
5998 epoch: 1,
5999 };
6000
6001 let disabled_iterations = Arc::new(AtomicUsize::new(0));
6002 let disabled_probe = Arc::clone(&disabled_iterations);
6003 let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6004 let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6005 root.clone(),
6006 Lane::PureRead,
6007 "untracked-search".to_string(),
6008 Box::new(move |_| {
6009 disabled_started_tx
6010 .send(())
6011 .expect("signal untracked search");
6012 let deadline = Instant::now() + Duration::from_secs(5);
6013 while !crate::commands::semantic_search::search_cancellation_requested() {
6014 if Instant::now() >= deadline {
6015 return Response::error(
6016 "untracked-search",
6017 "test_timeout",
6018 "untracked search did not receive cancellation",
6019 );
6020 }
6021 disabled_probe.fetch_add(1, Ordering::Relaxed);
6022 std::thread::yield_now();
6023 }
6024 Response::error(
6025 "untracked-search",
6026 "request_cancelled",
6027 "cancelled at search checkpoint",
6028 )
6029 }),
6030 );
6031 disabled_started_rx
6032 .recv_timeout(Duration::from_secs(1))
6033 .expect("untracked search starts");
6034 assert_eq!(
6035 apply_route_work_disposition(
6036 &active,
6037 executor.as_ref(),
6038 route,
6039 RouteWorkDisposition::Abandon,
6040 "disabled cancellation wiring",
6041 ),
6042 0
6043 );
6044 let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6045 std::thread::sleep(Duration::from_millis(10));
6046 assert!(
6047 disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6048 "without route registration the search keeps computing"
6049 );
6050 disabled_cancellation.request_cancel();
6051 let disabled_response = disabled_rx
6052 .blocking_recv()
6053 .expect("untracked search settles after explicit cleanup");
6054 assert_eq!(disabled_response.data["code"], "request_cancelled");
6055
6056 let tracked_iterations = Arc::new(AtomicUsize::new(0));
6057 let tracked_probe = Arc::clone(&tracked_iterations);
6058 let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6059 let tracked_rx = submit_active_tool_call(
6060 executor.as_ref(),
6061 &active,
6062 route,
6063 42,
6064 root.clone(),
6065 Lane::PureRead,
6066 "tracked-search".to_string(),
6067 RouteDetachPolicy::RetainForReplay,
6068 Box::new(move |_| {
6069 tracked_started_tx.send(()).expect("signal tracked search");
6070 let deadline = Instant::now() + Duration::from_secs(5);
6071 while !crate::commands::semantic_search::search_cancellation_requested() {
6072 if Instant::now() >= deadline {
6073 return Response::error(
6074 "tracked-search",
6075 "test_timeout",
6076 "tracked search did not receive cancellation",
6077 );
6078 }
6079 tracked_probe.fetch_add(1, Ordering::Relaxed);
6080 std::thread::yield_now();
6081 }
6082 Response::error(
6083 "tracked-search",
6084 "request_cancelled",
6085 "cancelled at search checkpoint",
6086 )
6087 }),
6088 );
6089 tracked_started_rx
6090 .recv_timeout(Duration::from_secs(1))
6091 .expect("tracked search starts");
6092
6093 let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6094 let terminal_rx = submit_active_tool_call(
6095 executor.as_ref(),
6096 &active,
6097 route,
6098 43,
6099 root.clone(),
6100 Lane::PureRead,
6101 "teardown-terminal".to_string(),
6102 RouteDetachPolicy::CancelOnDetach,
6103 Box::new(move |_| {
6104 terminal_started_tx
6105 .send(())
6106 .expect("signal teardown-terminal call");
6107 let deadline = Instant::now() + Duration::from_secs(5);
6108 while !crate::executor::current_job_cancelled() {
6109 if Instant::now() >= deadline {
6110 return Response::error(
6111 "teardown-terminal",
6112 "test_timeout",
6113 "terminal call did not receive cancellation",
6114 );
6115 }
6116 std::thread::yield_now();
6117 }
6118 Response::error(
6119 "teardown-terminal",
6120 "request_cancelled",
6121 "cancelled for terminal-emitting teardown",
6122 )
6123 }),
6124 );
6125 terminal_started_rx
6126 .recv_timeout(Duration::from_secs(1))
6127 .expect("teardown-terminal call starts");
6128 assert_eq!(
6129 apply_route_work_disposition(
6130 &active,
6131 executor.as_ref(),
6132 route,
6133 RouteWorkDisposition::RetainForReplay,
6134 "test route detach",
6135 ),
6136 1,
6137 "only the replayable search remains active after route detach"
6138 );
6139 let terminal_response = terminal_rx
6140 .blocking_recv()
6141 .expect("teardown-terminal call stops at cancellation checkpoint");
6142 assert_eq!(terminal_response.data["code"], "request_cancelled");
6143 let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6144 std::thread::sleep(Duration::from_millis(10));
6145 assert!(
6146 tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6147 "route detach must retain work whose response can replay after rebind"
6148 );
6149 assert_eq!(
6150 apply_route_work_disposition(
6151 &active,
6152 executor.as_ref(),
6153 route,
6154 RouteWorkDisposition::Abandon,
6155 "test session purge",
6156 ),
6157 1
6158 );
6159 let tracked_response = tracked_rx
6160 .blocking_recv()
6161 .expect("tracked search stops at cancellation checkpoint");
6162 assert_eq!(tracked_response.data["code"], "request_cancelled");
6163 assert!(active.lock().expect("active tool calls").is_empty());
6164
6165 let deadline = Instant::now() + Duration::from_secs(1);
6166 while !executor.actor_is_idle(&root) {
6167 assert!(
6168 Instant::now() < deadline,
6169 "cancelled search must release the PureRead lane"
6170 );
6171 std::thread::sleep(Duration::from_millis(2));
6172 }
6173 }
6174
6175 #[test]
6176 fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6177 let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6178 let executor = Arc::new(Executor::new());
6179 let (dir, root) = test_root("shutdown-deferred-inspect");
6180 std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6181 let ctx = inspect_context(dir.path());
6182 executor.register_actor(root.clone(), Arc::clone(&ctx));
6183 let (started_rx, _release_tx) =
6184 crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6185 let (pending, cancellation) =
6186 submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6187 let route = RouteChannel {
6188 channel: 8,
6189 epoch: 1,
6190 };
6191 started_rx
6192 .recv_timeout(Duration::from_secs(1))
6193 .expect("detached inspect reaches shutdown gate");
6194 let mut registry = PendingSubcResponses::default();
6195 registry.register(PendingSubcResponse {
6196 route,
6197 corr: 42,
6198 flags: Flags::new(false, Priority::Passive, false),
6199 ver: PROTOCOL_VERSION,
6200 root: root.clone(),
6201 session_id: "shutdown-session".to_string(),
6202 bare_name: "inspect".to_string(),
6203 format_context: crate::subc_format::FormatContext::from_tool_call(
6204 "inspect",
6205 &json!({}),
6206 dir.path(),
6207 ),
6208 bind_trust: BindTrust::FirstParty,
6209 pending,
6210 surface_downgraded: false,
6211 phase_trace: PhaseTrace::new(Instant::now()),
6212 });
6213 let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6214 (route, 42),
6215 ActiveToolCall {
6216 root_id: root.clone(),
6217 cancellation,
6218 detach_policy: RouteDetachPolicy::CancelOnDetach,
6219 },
6220 )])));
6221
6222 let resolved = registry.drain_on_shutdown(executor.as_ref());
6223 assert!(registry.is_empty());
6224 assert_eq!(resolved.len(), 1);
6225 assert_eq!(
6226 resolved[0].response.data["failure_reason"],
6227 "daemon_shutdown"
6228 );
6229 finish_active_tool_call(&active, route, 42);
6230 let deadline = Instant::now() + Duration::from_secs(1);
6231 while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6232 assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6233 std::thread::sleep(Duration::from_millis(5));
6234 }
6235 assert!(active.lock().expect("active calls").is_empty());
6236 assert!(executor.actor_is_idle(&root));
6237 }
6238
6239 pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6240 let deadline = Instant::now() + Duration::from_secs(30);
6241 loop {
6242 let observed = ctx.watcher_registry_count();
6243 if observed == expected {
6244 return;
6245 }
6246 assert!(
6247 Instant::now() < deadline,
6248 "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6249 );
6250 std::thread::sleep(Duration::from_millis(50));
6251 }
6252 }
6253
6254 pub(super) fn reap_until_forgotten(
6264 root: &ProjectRootId,
6265 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6266 pending_binds: &HashMap<RouteChannel, PendingBind>,
6267 root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6268 executor: &Arc<Executor>,
6269 metrics: &DispatchPathMetrics,
6270 ) -> IdleReapOutcome {
6271 let deadline = Instant::now() + Duration::from_secs(30);
6272 loop {
6273 let outcome = reap_idle_roots(
6274 Instant::now(),
6275 live_roots,
6276 pending_binds,
6277 root_channels,
6278 executor,
6279 metrics,
6280 );
6281 if outcome.forgotten_deleted_roots.contains(root) {
6282 return outcome;
6283 }
6284 assert!(
6285 Instant::now() < deadline,
6286 "deleted root was never forgotten: {root:?}"
6287 );
6288 std::thread::sleep(Duration::from_millis(10));
6289 }
6290 }
6291
6292 pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
6293 let deadline = Instant::now() + Duration::from_secs(30);
6294 loop {
6295 let observed = app.actor_root_count();
6296 if observed == expected {
6297 return;
6298 }
6299 assert!(
6300 Instant::now() < deadline,
6301 "actor root count did not settle before deadline: expected={expected}, observed={observed}"
6302 );
6303 std::thread::sleep(Duration::from_millis(50));
6304 }
6305 }
6306
6307 pub(super) fn status_frame(seq: u64) -> PushFrame {
6308 status_frame_with_session(seq, None)
6309 }
6310
6311 pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
6312 PushFrame::StatusChanged(StatusChangedFrame {
6313 frame_type: "status_changed",
6314 session_id: session_id.map(str::to_string),
6315 snapshot: json!({ "seq": seq }),
6316 })
6317 }
6318
6319 pub(super) fn completion_frame(task_id: &str) -> PushFrame {
6320 completion_frame_with_session(task_id, "session-1")
6321 }
6322
6323 pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
6324 PushFrame::BashCompleted(BashCompletedFrame {
6325 frame_type: "bash_completed",
6326 task_id: task_id.to_string(),
6327 session_id: session_id.to_string(),
6328 status: BgTaskStatus::Completed,
6329 exit_code: Some(0),
6330 command: format!("echo {task_id}"),
6331 output_preview: String::new(),
6332 output_truncated: false,
6333 original_tokens: None,
6334 compressed_tokens: None,
6335 tokens_skipped: false,
6336 status_reason: None,
6337 })
6338 }
6339
6340 pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
6341 long_running_frame_with_session(task_id, "session-1", elapsed_ms)
6342 }
6343
6344 pub(super) fn long_running_frame_with_session(
6345 task_id: &str,
6346 session_id: &str,
6347 elapsed_ms: u64,
6348 ) -> PushFrame {
6349 PushFrame::BashLongRunning(BashLongRunningFrame {
6350 frame_type: "bash_long_running",
6351 task_id: task_id.to_string(),
6352 session_id: session_id.to_string(),
6353 command: format!("sleep {elapsed_ms}"),
6354 elapsed_ms,
6355 })
6356 }
6357
6358 pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
6359 PushFrame::BashPatternMatch(BashPatternMatchFrame {
6360 frame_type: "bash_pattern_match",
6361 task_id: "task-pattern".to_string(),
6362 session_id: session_id.to_string(),
6363 watch_id: "watch-1".to_string(),
6364 match_text: "needle".to_string(),
6365 match_offset: 7,
6366 context: "haystack needle".to_string(),
6367 once: true,
6368 reason: "pattern_match",
6369 })
6370 }
6371
6372 pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
6373 PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
6374 frame_type: "configure_warnings",
6375 session_id: session_id.map(str::to_string),
6376 project_root: "/tmp/subc-test".to_string(),
6377 warnings: Vec::new(),
6378 })
6379 }
6380
6381 pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
6382 route_identity_with_trust(root, session_id, BindTrust::FirstParty)
6383 }
6384
6385 pub(super) fn route_identity_with_trust(
6386 root: &ProjectRootId,
6387 session_id: &str,
6388 trust: BindTrust,
6389 ) -> RouteIdentity {
6390 RouteIdentity(Arc::new(RouteIdentityData {
6391 root: root.clone(),
6392 project_root: root.as_path().to_path_buf(),
6393 harness: "opencode".to_string(),
6394 session: session_id.to_string(),
6395 trust,
6396 spawn_principal: AuthenticatedPrincipal::RouteBind {
6397 trust: trust.sandbox_trust(),
6398 route_channel: 0,
6399 route_epoch: 0,
6400 project_root: root.as_path().to_path_buf(),
6401 harness: "opencode".to_string(),
6402 session_id: session_id.to_string(),
6403 principal_id: Some(match trust {
6404 BindTrust::FirstParty => "direct".to_string(),
6405 BindTrust::Untrusted => "unverified".to_string(),
6406 }),
6407 },
6408 consumer_elicitation_capable: false,
6409 }))
6410 }
6411
6412 pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
6413 PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
6414 }
6415
6416 pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
6417 match frame {
6418 PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
6419 _ => None,
6420 }
6421 }
6422
6423 pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
6424 match frame {
6425 PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
6426 _ => None,
6427 }
6428 }
6429
6430 pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
6431 let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
6432 body.get("task_id")
6433 .and_then(serde_json::Value::as_str)
6434 .map(str::to_string)
6435 }
6436}
6437
6438#[cfg(test)]
6439mod tests {
6440 use super::test_support::{
6441 completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
6442 wait_for_actor_root_count, wait_for_watcher_count,
6443 };
6444 use super::*;
6445 use crate::bash_background::BgTaskStatus;
6446
6447 fn attach_error(kind: io::ErrorKind) -> SubcError {
6448 SubcError::Connect {
6449 endpoint: "127.0.0.1:1".to_string(),
6450 source: io::Error::new(kind, "constructed attach failure"),
6451 }
6452 }
6453
6454 fn auth_io_error(kind: io::ErrorKind) -> SubcError {
6455 SubcError::Auth {
6456 endpoint: "127.0.0.1:1".to_string(),
6457 source: subc_transport::AuthError::Io {
6458 stage: subc_transport::AuthStage::ServerProof,
6459 source: io::Error::new(kind, "constructed auth failure"),
6460 },
6461 }
6462 }
6463
6464 #[test]
6465 fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
6466 let transient_errors = vec![
6467 attach_error(io::ErrorKind::ConnectionRefused),
6468 attach_error(io::ErrorKind::TimedOut),
6469 attach_error(io::ErrorKind::ConnectionReset),
6470 auth_io_error(io::ErrorKind::ConnectionAborted),
6471 auth_io_error(io::ErrorKind::BrokenPipe),
6472 SubcError::Auth {
6473 endpoint: "127.0.0.1:1".to_string(),
6474 source: subc_transport::AuthError::UnexpectedEof {
6475 stage: subc_transport::AuthStage::ServerProof,
6476 expected: 4,
6477 actual: 0,
6478 },
6479 },
6480 SubcError::Auth {
6481 endpoint: "127.0.0.1:1".to_string(),
6482 source: subc_transport::AuthError::Timeout {
6483 stage: subc_transport::AuthStage::ServerProof,
6484 deadline: AUTH_DEADLINE,
6485 },
6486 },
6487 ];
6488 for error in &transient_errors {
6489 assert_eq!(
6490 classify_attach_error(error),
6491 AttachErrorClass::Transient,
6492 "expected transient: {error}"
6493 );
6494 }
6495
6496 let permanent_errors = vec![
6497 attach_error(io::ErrorKind::PermissionDenied),
6498 auth_io_error(io::ErrorKind::InvalidData),
6499 SubcError::Auth {
6500 endpoint: "127.0.0.1:1".to_string(),
6501 source: subc_transport::AuthError::InvalidServerProof,
6502 },
6503 SubcError::Auth {
6504 endpoint: "127.0.0.1:1".to_string(),
6505 source: subc_transport::AuthError::DaemonIdMismatch,
6506 },
6507 SubcError::ConnectionFile {
6508 path: PathBuf::from("subc-connection.json"),
6509 source: subc_transport::ConnectionFileError::Invalid {
6510 reason: "constructed invalid file".to_string(),
6511 },
6512 },
6513 SubcError::NoEndpoint {
6514 path: PathBuf::from("subc-connection.json"),
6515 },
6516 SubcError::InvalidEndpoint {
6517 path: PathBuf::from("subc-connection.json"),
6518 endpoint: "not-an-ip:1234".to_string(),
6519 },
6520 ];
6521 for error in &permanent_errors {
6522 assert_eq!(
6523 classify_attach_error(error),
6524 AttachErrorClass::Permanent,
6525 "expected permanent: {error}"
6526 );
6527 }
6528 }
6529
6530 #[test]
6531 fn incompatible_wire_version_is_rejected_before_tcp_connect() {
6532 let conn_dir = tempfile::tempdir().expect("connection tempdir");
6533 let conn_path = conn_dir.path().join("subc-connection.json");
6534 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
6535 listener
6536 .set_nonblocking(true)
6537 .expect("set listener nonblocking");
6538 let port = listener.local_addr().expect("listener addr").port();
6539 connection_file::write_atomic(
6540 &conn_path,
6541 &connection_file::ConnectionInfo {
6542 schema: connection_file::SCHEMA_VERSION,
6543 wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
6544 endpoints: vec![connection_file::Endpoint {
6545 host: "127.0.0.1".to_string(),
6546 port,
6547 }],
6548 key: vec![0x42; subc_transport::KEY_LEN],
6549 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
6550 pid: std::process::id(),
6551 daemon_ver: "subc-test".to_string(),
6552 },
6553 )
6554 .expect("write connection file");
6555
6556 let runtime = tokio::runtime::Builder::new_current_thread()
6557 .enable_all()
6558 .build()
6559 .expect("test runtime");
6560 let result = runtime.block_on(connect_and_authenticate_with_policy(
6561 &conn_path,
6562 AttachRetryPolicy {
6563 budget: Duration::from_secs(1),
6564 initial_backoff: Duration::from_millis(5),
6565 max_backoff: Duration::from_millis(10),
6566 jitter_percent: 0,
6567 },
6568 ));
6569 assert!(matches!(
6570 result,
6571 Err(SubcError::ConnectionFile {
6572 source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
6573 ..
6574 })
6575 ));
6576 assert!(matches!(
6577 listener.accept(),
6578 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
6579 ));
6580 }
6581
6582 #[test]
6583 fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
6584 let conn_dir = tempfile::tempdir().expect("connection tempdir");
6585 let conn_path = conn_dir.path().join("subc-connection.json");
6586 let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
6587 let port = listener.local_addr().expect("reserved addr").port();
6588 drop(listener);
6589 connection_file::write_atomic(
6590 &conn_path,
6591 &connection_file::ConnectionInfo {
6592 schema: connection_file::SCHEMA_VERSION,
6593 wire_version: Some(PROTOCOL_VERSION),
6594 endpoints: vec![connection_file::Endpoint {
6595 host: "127.0.0.1".to_string(),
6596 port,
6597 }],
6598 key: vec![0x42; subc_transport::KEY_LEN],
6599 daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
6600 pid: std::process::id(),
6601 daemon_ver: "subc-test".to_string(),
6602 },
6603 )
6604 .expect("write connection file");
6605
6606 let policy = AttachRetryPolicy {
6607 budget: Duration::from_millis(40),
6608 initial_backoff: Duration::from_millis(5),
6609 max_backoff: Duration::from_millis(10),
6610 jitter_percent: 0,
6611 };
6612 let runtime = tokio::runtime::Builder::new_current_thread()
6613 .enable_all()
6614 .build()
6615 .expect("test runtime");
6616 let started_at = Instant::now();
6617 let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
6618 let elapsed = started_at.elapsed();
6619 let error = match result {
6620 Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
6621 Err(error) => error,
6622 };
6623
6624 assert!(matches!(error, SubcError::Connect { .. }), "{error}");
6625 assert!(
6626 elapsed >= Duration::from_millis(35),
6627 "retry budget ended too early: {elapsed:?}"
6628 );
6629 assert!(
6630 elapsed < Duration::from_secs(1),
6631 "retry budget was not bounded: {elapsed:?}"
6632 );
6633 }
6634
6635 fn due_maintenance_jobs_without_actor_context(
6636 live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6637 budget: usize,
6638 pending_bind_roots: &HashSet<ProjectRootId>,
6639 ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
6640 due_maintenance_jobs(
6641 live_roots,
6642 None,
6643 &HashMap::new(),
6644 &HashSet::new(),
6645 budget,
6646 pending_bind_roots,
6647 )
6648 }
6649
6650 fn actor_ctx_with_dirty_search_index(
6651 root: &Path,
6652 storage: &Path,
6653 file_name: &str,
6654 old_contents: &str,
6655 new_contents: &str,
6656 ) -> (Arc<AppContext>, PathBuf, PathBuf) {
6657 let file = root.join(file_name);
6658 std::fs::write(&file, old_contents).expect("write source");
6659 let canonical_root = std::fs::canonicalize(root).expect("canonical root");
6660 let ctx = Arc::new(AppContext::new(
6661 Box::new(crate::parser::TreeSitterProvider::new()),
6662 Config {
6663 project_root: Some(root.to_path_buf()),
6664 storage_dir: Some(storage.to_path_buf()),
6665 ..Config::default()
6666 },
6667 ));
6668 ctx.set_canonical_cache_root(canonical_root.clone());
6669
6670 let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
6671 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6672 let git_head = index.stored_git_head().map(str::to_owned);
6673 index.write_to_disk(&cache_dir, git_head.as_deref());
6674
6675 std::fs::write(&file, new_contents).expect("edit source");
6676 index.update_file(&file);
6677 *ctx.search_index()
6678 .write()
6679 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6680 (ctx, canonical_root, cache_dir)
6681 }
6682
6683 #[test]
6684 fn graceful_shutdown_flushes_every_actor_search_index() {
6685 let storage = tempfile::tempdir().expect("storage tempdir");
6686 let (root1_dir, root1) = test_root("shutdown-flush-root-1");
6687 let (root2_dir, root2) = test_root("shutdown-flush-root-2");
6688 let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
6689 root1_dir.path(),
6690 storage.path(),
6691 "alpha.txt",
6692 "old actor one token\n",
6693 "new actor one token\n",
6694 );
6695 let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
6696 root2_dir.path(),
6697 storage.path(),
6698 "beta.txt",
6699 "old actor two token\n",
6700 "new actor two token\n",
6701 );
6702
6703 let executor = Executor::new();
6704 assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
6705 assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
6706
6707 flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
6708
6709 let mut restored1 =
6710 crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
6711 .expect("load flushed root one index");
6712 restored1.ready = true;
6713 assert_eq!(
6714 restored1
6715 .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
6716 .matches
6717 .len(),
6718 1,
6719 "graceful subc shutdown should flush the first root's trigram delta"
6720 );
6721
6722 let mut restored2 =
6723 crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
6724 .expect("load flushed root two index");
6725 restored2.ready = true;
6726 assert_eq!(
6727 restored2
6728 .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
6729 .matches
6730 .len(),
6731 1,
6732 "graceful subc shutdown should flush every registered root"
6733 );
6734 }
6735
6736 #[test]
6737 fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
6738 let _ = env_logger::builder().is_test(true).try_init();
6739 let (root_dir, root) = test_root("idle-root-reaper");
6740 let storage = tempfile::tempdir().expect("storage tempdir");
6741 std::fs::write(
6742 root_dir.path().join("main.rs"),
6743 "fn entry() { leaf(); }\nfn leaf() {}\n",
6744 )
6745 .expect("source file");
6746 let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
6747 let app = App::default_shared();
6748 let ctx = Arc::new(AppContext::from_app(
6749 Arc::clone(&app),
6750 Config {
6751 project_root: Some(canonical_root.clone()),
6752 storage_dir: Some(storage.path().to_path_buf()),
6753 callgraph_store: true,
6754 search_index: true,
6755 ..Config::default()
6756 },
6757 ));
6758 ctx.set_canonical_cache_root(canonical_root.clone());
6759 let project_key = crate::search_index::artifact_cache_key(&canonical_root);
6760 crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
6761 assert!(ctx
6762 .ensure_callgraph_store()
6763 .expect("build callgraph store")
6764 .is_some());
6765
6766 let cache_dir =
6767 crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
6768 let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6769 let git_head = index.stored_git_head().map(str::to_owned);
6770 index.write_to_disk(&cache_dir, git_head.as_deref());
6771 *ctx.search_index()
6772 .write()
6773 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6774 let seeded_generation =
6777 crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
6778 .expect("seeded artifact generation");
6779 crate::cache_freshness::record_verify_completed(
6780 &canonical_root,
6781 crate::cache_freshness::VerifyArtifact::Search,
6782 Some(seeded_generation),
6783 );
6784 assert!(
6785 matches!(
6786 crate::cache_freshness::warm_verify_plan(
6787 canonical_root.as_path(),
6788 crate::cache_freshness::VerifyArtifact::Search,
6789 Some(seeded_generation),
6790 ),
6791 crate::cache_freshness::WarmVerifyPlan::Skip
6792 ),
6793 "memo must be warm before eviction for the downgrade assertion to bite"
6794 );
6795
6796 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6797 let _dispatch_tx = dispatch_tx;
6798 let shutdown = Arc::new(AtomicBool::new(false));
6799 let thread_shutdown = Arc::clone(&shutdown);
6800 let join = std::thread::spawn(move || {
6801 while !thread_shutdown.load(Ordering::SeqCst) {
6802 std::thread::yield_now();
6803 }
6804 });
6805 ctx.install_watcher_runtime(
6806 dispatch_rx,
6807 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6808 );
6809 wait_for_watcher_count(&ctx, 1);
6810
6811 let executor = Arc::new(Executor::new());
6812 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6813 ctx.mark_subc_unbound();
6814 let mut live_roots = HashMap::new();
6815 let mut meta = RootMeta::new(Instant::now());
6816 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6817 meta.unbound_quiesced = true;
6818 live_roots.insert(root.clone(), meta);
6819
6820 let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
6821 assert!(message.contains("evicted idle root"));
6822 assert!(message.contains("freed ~"));
6823 assert!(message.contains("semantic"));
6824 assert!(!message.contains("semantic not estimated retained"));
6825 assert!(message.contains("trigram"));
6826 assert!(message.contains("retained: bash"));
6827 assert!(message.contains("parser_pool"));
6828
6829 assert_eq!(
6830 reap_idle_roots(
6831 Instant::now(),
6832 &mut live_roots,
6833 &HashMap::new(),
6834 &HashMap::new(),
6835 &executor,
6836 &DispatchPathMetrics::new(),
6837 )
6838 .evicted,
6839 1
6840 );
6841 assert!(ctx.search_index().read().unwrap().is_none());
6842 wait_for_watcher_count(&ctx, 0);
6843 assert!(
6848 matches!(
6849 crate::cache_freshness::warm_verify_plan(
6850 canonical_root.as_path(),
6851 crate::cache_freshness::VerifyArtifact::Search,
6852 Some(seeded_generation),
6853 ),
6854 crate::cache_freshness::WarmVerifyPlan::Strict
6855 ),
6856 "idle eviction must force strict re-verification"
6857 );
6858 assert!(
6859 crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
6860 );
6861 ctx.mark_subc_bound();
6862 assert!(ctx
6863 .ensure_callgraph_store()
6864 .expect("reopen callgraph store")
6865 .is_some());
6866 assert!(live_roots[&root].idle_artifacts_evicted);
6867 }
6868
6869 #[test]
6870 fn idle_root_reaper_applies_ttl_to_unbound_roots() {
6871 let (_root_dir, root) = test_root("idle-root-ttl-gate");
6872 let ctx = test_ctx();
6873 let executor = Arc::new(Executor::new());
6874 assert!(executor.register_actor(root.clone(), ctx));
6875 let ctx = executor.actor_context(&root).expect("actor context");
6876 ctx.mark_subc_unbound();
6877 let now = Instant::now();
6878 let mut meta = RootMeta::new(now);
6879 meta.unbound_quiesced = true;
6880 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6881
6882 assert_eq!(
6886 reap_idle_roots(
6887 now,
6888 &mut live_roots,
6889 &HashMap::new(),
6890 &HashMap::new(),
6891 &executor,
6892 &DispatchPathMetrics::new(),
6893 )
6894 .evicted,
6895 0
6896 );
6897 assert!(!live_roots[&root].idle_artifacts_evicted);
6898
6899 ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
6905 assert_eq!(
6906 reap_idle_roots(
6907 now + IDLE_ROOT_TTL,
6908 &mut live_roots,
6909 &HashMap::new(),
6910 &HashMap::new(),
6911 &executor,
6912 &DispatchPathMetrics::new(),
6913 )
6914 .evicted,
6915 1
6916 );
6917 assert!(live_roots[&root].idle_artifacts_evicted);
6918 assert!(
6919 ctx.take_pending_search_index_paths().is_empty(),
6920 "TTL eviction must dispose retained pending reconciliation paths"
6921 );
6922 }
6923
6924 #[test]
6925 fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
6926 let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
6927 let ctx = test_ctx();
6928 let executor = Arc::new(Executor::new());
6929 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6930 ctx.mark_subc_unbound();
6931
6932 let pending = root.as_path().join("edited-while-unbound.rs");
6938 ctx.add_pending_search_index_paths([pending.clone()]);
6939 let dirty_source = root.as_path().join("dirty.rs");
6940 std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
6941 let mut dirty = crate::search_index::SearchIndex::new();
6942 dirty.ready = true;
6943 dirty.update_file(&dirty_source);
6944 *ctx.search_index()
6945 .write()
6946 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
6947 assert!(ctx.artifact_eviction_blocked());
6948
6949 let mut live_roots = HashMap::new();
6950 let mut meta = RootMeta::new(Instant::now());
6951 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6952 meta.unbound_quiesced = true;
6953 live_roots.insert(root.clone(), meta);
6954
6955 assert_eq!(
6956 reap_idle_roots(
6957 Instant::now(),
6958 &mut live_roots,
6959 &HashMap::new(),
6960 &HashMap::new(),
6961 &executor,
6962 &DispatchPathMetrics::new(),
6963 )
6964 .evicted,
6965 0,
6966 "the dirty index must still block this eviction"
6967 );
6968 assert_eq!(
6969 ctx.take_pending_search_index_paths(),
6970 vec![pending],
6971 "a blocked eviction must restore the taken pending paths"
6972 );
6973 }
6974
6975 #[test]
6976 fn idle_reap_with_bound_route_keeps_watcher_running() {
6977 let (_root_dir, root) = test_root("bound-root-reap-gate");
6978 let ctx = test_ctx();
6979 let executor = Arc::new(Executor::new());
6980 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6981
6982 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6983 let _dispatch_tx = dispatch_tx;
6984 let shutdown = Arc::new(AtomicBool::new(false));
6985 let thread_shutdown = Arc::clone(&shutdown);
6986 let join = std::thread::spawn(move || {
6987 while !thread_shutdown.load(Ordering::SeqCst) {
6988 std::thread::yield_now();
6989 }
6990 });
6991 ctx.install_watcher_runtime(
6992 dispatch_rx,
6993 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6994 );
6995
6996 let mut meta = RootMeta::new(Instant::now());
6997 meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6998 let mut live_roots = HashMap::from([(root.clone(), meta)]);
6999 let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7000 assert_eq!(
7001 reap_idle_roots(
7002 Instant::now(),
7003 &mut live_roots,
7004 &HashMap::new(),
7005 &bound,
7006 &executor,
7007 &DispatchPathMetrics::new(),
7008 )
7009 .evicted,
7010 0
7011 );
7012 wait_for_watcher_count(&ctx, 1);
7013 ctx.stop_watcher_runtime_in_background();
7014 wait_for_watcher_count(&ctx, 0);
7015 }
7016
7017 #[test]
7018 fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
7019 let (root_dir, root) = test_root("deleted-bound-root-reap");
7020 let executor = Arc::new(Executor::new());
7021 assert!(executor.register_actor(root.clone(), test_ctx()));
7022 root_dir.close().expect("delete project root");
7023
7024 let route = route_key(19, 3);
7025 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7026 let cancel_signal = PersistentCancelSignal::new();
7027 let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
7028 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7029 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7030 let mut route_bash_cancels = HashMap::from([(
7031 route,
7032 bash::RouteBashCancel {
7033 token: cancel_signal.clone(),
7034 active_waits: 0,
7035 },
7036 )]);
7037 let metrics = DispatchPathMetrics::new();
7038
7039 let first = reap_idle_roots(
7040 Instant::now(),
7041 &mut live_roots,
7042 &HashMap::new(),
7043 &root_channels,
7044 &executor,
7045 &metrics,
7046 );
7047 assert!(first.forgotten_deleted_roots.is_empty());
7048 assert!(executor.actor_registered(&root));
7049
7050 let mut forgotten = Vec::new();
7051 for _ in 0..100 {
7052 let outcome = reap_idle_roots(
7053 Instant::now(),
7054 &mut live_roots,
7055 &HashMap::new(),
7056 &root_channels,
7057 &executor,
7058 &metrics,
7059 );
7060 if !outcome.forgotten_deleted_roots.is_empty() {
7061 forgotten = outcome.forgotten_deleted_roots;
7062 break;
7063 }
7064 std::thread::sleep(Duration::from_millis(10));
7065 }
7066 assert_eq!(forgotten, vec![root.clone()]);
7067 assert!(!executor.actor_registered(&root));
7068
7069 let mut retry_buffer = HashMap::new();
7070 let mut reclaimed_routes = ReclaimedRoutes::default();
7071 let mut session_identity = HashMap::new();
7072 let mut push_buffer = HashMap::new();
7073 let mut bg_subs = HashMap::from([(
7074 route,
7075 BgSub {
7076 corr: 77,
7077 ver: PROTOCOL_VERSION,
7078 flags: control_flags(),
7079 root: root.clone(),
7080 session: "deleted-route".to_string(),
7081 },
7082 )]);
7083 let mut bg_sub_by_session = HashMap::from([(
7084 (root.clone(), "deleted-route".to_string()),
7085 HashSet::from([route]),
7086 )]);
7087 let mut bg_wake_pending = HashSet::from([route]);
7088 let mut bg_wake_epoch = HashMap::new();
7089 let mut pending_bash_asks = HashMap::new();
7090 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
7091 health::take_bg_observability_logs_for_test();
7092 purge_deleted_root_residents(
7093 &root,
7094 &mut routes,
7095 &mut root_channels,
7096 &mut installed_route_epochs,
7097 &mut route_bash_cancels,
7098 &active_tool_calls,
7099 executor.as_ref(),
7100 &mut retry_buffer,
7101 &mut reclaimed_routes,
7102 &mut session_identity,
7103 &mut push_buffer,
7104 &mut bg_subs,
7105 &mut bg_sub_by_session,
7106 &mut bg_wake_pending,
7107 &mut bg_wake_epoch,
7108 &mut pending_bash_asks,
7109 &metrics,
7110 );
7111
7112 assert!(routes.is_empty());
7113 assert!(root_channels.is_empty());
7114 assert!(installed_route_epochs.is_empty());
7115 assert!(route_bash_cancels.is_empty());
7116 assert!(reclaimed_routes.contains(route));
7117 assert!(cancel_signal.is_cancelled());
7118 assert_eq!(
7119 health::take_bg_observability_logs_for_test(),
7120 vec![format!(
7121 "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
7122 root.as_path().display()
7123 )]
7124 );
7125 }
7126
7127 #[test]
7134 fn live_root_with_bound_route_is_never_reclaimed() {
7135 let (_root_dir, root) = test_root("live-bound-root-retained");
7136 let ctx = test_ctx();
7137 ctx.mark_subc_unbound();
7138 let executor = Arc::new(Executor::new());
7139 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7140
7141 let route = RouteChannel {
7142 channel: 7,
7143 epoch: 1,
7144 };
7145 let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
7146 meta.unbound_quiesced = true;
7147 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7148 let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7149
7150 for _ in 0..3 {
7155 let outcome = reap_idle_roots(
7156 Instant::now(),
7157 &mut live_roots,
7158 &HashMap::new(),
7159 &root_channels,
7160 &executor,
7161 &DispatchPathMetrics::new(),
7162 );
7163 assert!(
7164 outcome.forgotten_deleted_roots.is_empty(),
7165 "a root whose directory exists must never be forgotten"
7166 );
7167 }
7168
7169 assert!(live_roots.contains_key(&root), "live root must be retained");
7170 assert!(
7171 executor.actor_registered(&root),
7172 "live root's actor must survive"
7173 );
7174 assert!(
7175 root.as_path().exists(),
7176 "test vehicle must keep the directory alive; otherwise this control proves nothing"
7177 );
7178 }
7179
7180 #[test]
7181 fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
7182 let (root_dir, root) = test_root("deleted-root-first-observation");
7183 let ctx = test_ctx();
7184 ctx.mark_subc_unbound();
7185 let executor = Arc::new(Executor::new());
7186 assert!(executor.register_actor(root.clone(), ctx));
7187 root_dir.close().expect("delete project root");
7188
7189 let mut meta = RootMeta::new(Instant::now());
7190 meta.unbound_quiesced = true;
7191 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7192 let outcome = reap_idle_roots(
7193 Instant::now(),
7194 &mut live_roots,
7195 &HashMap::new(),
7196 &HashMap::new(),
7197 &executor,
7198 &DispatchPathMetrics::new(),
7199 );
7200
7201 assert!(outcome.forgotten_deleted_roots.is_empty());
7202 assert!(live_roots.contains_key(&root));
7203 assert!(executor.actor_registered(&root));
7204 }
7205
7206 fn spawn_background_for_root(
7207 ctx: &AppContext,
7208 root: &ProjectRootId,
7209 storage: &tempfile::TempDir,
7210 session_id: &str,
7211 ) -> (String, u32) {
7212 let command = if cfg!(windows) {
7218 "ping -n 31 127.0.0.1 > nul"
7220 } else {
7221 "sleep 30"
7222 };
7223 let task_id = ctx
7224 .bash_background()
7225 .spawn(
7226 crate::sandbox_spawn::SpawnPlan::Unsandboxed,
7227 command,
7228 session_id.to_string(),
7229 storage.path().to_path_buf(),
7230 HashMap::new(),
7231 Some(Duration::from_secs(60)),
7232 storage.path().to_path_buf(),
7233 8,
7234 true,
7235 false,
7236 Some(root.as_path().to_path_buf()),
7237 )
7238 .expect("spawn background task");
7239 let snapshot = ctx
7240 .bash_background()
7241 .status(
7242 &task_id,
7243 session_id,
7244 Some(root.as_path()),
7245 Some(storage.path()),
7246 0,
7247 )
7248 .expect("background task status");
7249 (task_id, snapshot.child_pid.expect("background child pid"))
7250 }
7251
7252 fn wait_for_background_exit(pid: u32) {
7253 let deadline = Instant::now() + Duration::from_secs(5);
7254 while crate::bash_background::process::is_process_alive(pid) {
7255 assert!(
7256 Instant::now() < deadline,
7257 "background task process survived kill"
7258 );
7259 std::thread::sleep(Duration::from_millis(20));
7260 }
7261 }
7262
7263 #[test]
7264 fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
7265 let (root_dir, root) = test_root("deleted-root-background-task");
7266 let storage = tempfile::tempdir().expect("task storage");
7267 let ctx = test_ctx();
7268 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
7269 assert!(crate::bash_background::process::is_process_alive(pid));
7270
7271 let executor = Arc::new(Executor::new());
7272 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7273 assert!(executor.actor_is_idle(&root));
7274 root_dir.close().expect("delete project root");
7275 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7276 let pending_binds = HashMap::new();
7277 let root_channels = HashMap::new();
7278 let metrics = DispatchPathMetrics::new();
7279
7280 let first = reap_idle_roots(
7281 Instant::now(),
7282 &mut live_roots,
7283 &pending_binds,
7284 &root_channels,
7285 &executor,
7286 &metrics,
7287 );
7288 assert!(first.forgotten_deleted_roots.is_empty());
7289 assert!(crate::bash_background::process::is_process_alive(pid));
7290
7291 let outcome = reap_until_forgotten(
7292 &root,
7293 &mut live_roots,
7294 &pending_binds,
7295 &root_channels,
7296 &executor,
7297 &metrics,
7298 );
7299 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7300 wait_for_background_exit(pid);
7301
7302 let snapshot = ctx
7303 .bash_background()
7304 .status(
7305 &task_id,
7306 "reclaim-session",
7307 Some(root.as_path()),
7308 Some(storage.path()),
7309 0,
7310 )
7311 .expect("reclaimed task status");
7312 assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
7313 assert_eq!(
7314 snapshot.info.status_reason.as_deref(),
7315 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
7316 );
7317 assert_eq!(
7318 serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
7319 crate::bash_background::registry::ROOT_RECLAIMED_REASON
7320 );
7321 let completion = ctx
7322 .bash_background()
7323 .drain_completions_for_session(Some("reclaim-session"))
7324 .pop()
7325 .expect("reclaimed task completion");
7326 assert_eq!(
7327 completion.status_reason.as_deref(),
7328 Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
7329 );
7330 }
7331
7332 #[test]
7333 fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
7334 let (root_dir, root) = test_root("existing-root-background-task");
7335 let storage = tempfile::tempdir().expect("task storage");
7336 let ctx = test_ctx();
7337 ctx.mark_subc_unbound();
7338 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
7339
7340 let executor = Arc::new(Executor::new());
7341 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7342 let mut meta = RootMeta::new(
7343 Instant::now()
7344 .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
7345 .expect("old root timestamp"),
7346 );
7347 meta.unbound_quiesced = true;
7348 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7349 let pending_binds = HashMap::new();
7350 let root_channels = HashMap::new();
7351 let metrics = DispatchPathMetrics::new();
7352
7353 for _ in 0..8 {
7354 reap_idle_roots(
7355 Instant::now(),
7356 &mut live_roots,
7357 &pending_binds,
7358 &root_channels,
7359 &executor,
7360 &metrics,
7361 );
7362 std::thread::sleep(Duration::from_millis(10));
7363 }
7364 assert!(root_dir.path().exists());
7365 assert!(crate::bash_background::process::is_process_alive(pid));
7366 let snapshot = ctx
7367 .bash_background()
7368 .status(
7369 &task_id,
7370 "existing-session",
7371 Some(root.as_path()),
7372 Some(storage.path()),
7373 0,
7374 )
7375 .expect("existing task status");
7376 assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7377 let _ = ctx.bash_background().kill(&task_id, "existing-session");
7378 wait_for_background_exit(pid);
7379 }
7380
7381 #[test]
7382 fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
7383 let (root_dir, root) = test_root("restored-root-background-task");
7384 let storage = tempfile::tempdir().expect("task storage");
7385 let ctx = test_ctx();
7386 let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
7387
7388 let executor = Arc::new(Executor::new());
7389 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7390 root_dir.close().expect("delete project root");
7391 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7392 let pending_binds = HashMap::new();
7393 let root_channels = HashMap::new();
7394 let metrics = DispatchPathMetrics::new();
7395
7396 let first = reap_idle_roots(
7397 Instant::now(),
7398 &mut live_roots,
7399 &pending_binds,
7400 &root_channels,
7401 &executor,
7402 &metrics,
7403 );
7404 assert!(first.forgotten_deleted_roots.is_empty());
7405 std::fs::create_dir_all(root.as_path()).expect("restore project root");
7406 let second = reap_idle_roots(
7407 Instant::now(),
7408 &mut live_roots,
7409 &pending_binds,
7410 &root_channels,
7411 &executor,
7412 &metrics,
7413 );
7414 assert!(second.forgotten_deleted_roots.is_empty());
7415 assert!(crate::bash_background::process::is_process_alive(pid));
7416 let _ = ctx.bash_background().kill(&task_id, "restored-session");
7417 wait_for_background_exit(pid);
7418 }
7419
7420 #[test]
7421 fn observing_root_again_resets_deleted_sweep_confirmation() {
7422 let (root_dir, root) = test_root("deleted-root-observation-reset");
7423 let ctx = test_ctx();
7424 ctx.mark_subc_unbound();
7425 let executor = Arc::new(Executor::new());
7426 assert!(executor.register_actor(root.clone(), ctx));
7427 root_dir.close().expect("delete project root");
7428
7429 let mut meta = RootMeta::new(Instant::now());
7430 meta.unbound_quiesced = true;
7431 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7432 let pending_binds = HashMap::new();
7433 let root_channels = HashMap::new();
7434 let metrics = DispatchPathMetrics::new();
7435
7436 let first = reap_idle_roots(
7437 Instant::now(),
7438 &mut live_roots,
7439 &pending_binds,
7440 &root_channels,
7441 &executor,
7442 &metrics,
7443 );
7444 assert!(first.forgotten_deleted_roots.is_empty());
7445
7446 std::fs::create_dir_all(root.as_path()).expect("restore project root");
7447 reap_idle_roots(
7448 Instant::now(),
7449 &mut live_roots,
7450 &pending_binds,
7451 &root_channels,
7452 &executor,
7453 &metrics,
7454 );
7455 std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
7456
7457 let after_reset = reap_idle_roots(
7458 Instant::now(),
7459 &mut live_roots,
7460 &pending_binds,
7461 &root_channels,
7462 &executor,
7463 &metrics,
7464 );
7465 assert!(after_reset.forgotten_deleted_roots.is_empty());
7466 assert!(live_roots.contains_key(&root));
7467 assert!(executor.actor_registered(&root));
7468 }
7469
7470 #[test]
7471 fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
7472 let (root_dir, root) = test_root("deleted-root-reap");
7473 let app = App::default_shared();
7474 let ctx = Arc::new(AppContext::from_app(
7475 Arc::clone(&app),
7476 Config {
7477 project_root: Some(root.as_path().to_path_buf()),
7478 ..Config::default()
7479 },
7480 ));
7481 ctx.set_canonical_cache_root(root.as_path().to_path_buf());
7482 ctx.mark_subc_unbound();
7483 let executor = Arc::new(Executor::new());
7484 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7485 assert_eq!(app.actor_root_count(), 1);
7486 drop(ctx);
7487 root_dir.close().expect("delete project root");
7488
7489 let mut meta = RootMeta::new(Instant::now());
7490 meta.unbound_quiesced = true;
7491 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7492 let outcome = reap_until_forgotten(
7493 &root,
7494 &mut live_roots,
7495 &HashMap::new(),
7496 &HashMap::new(),
7497 &executor,
7498 &DispatchPathMetrics::new(),
7499 );
7500 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7501 assert!(!executor.actor_registered(&root));
7502 assert!(!live_roots.contains_key(&root));
7503 wait_for_actor_root_count(&app, 0);
7504
7505 let status_ctx = AppContext::from_app(app, Config::default());
7506 let status = status_ctx.build_status_snapshot();
7507 assert_eq!(status["runtime"]["live_actor_roots"], 0);
7508 assert_eq!(status["runtime"]["open_routes"], 0);
7509 }
7510
7511 #[test]
7512 fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
7513 let (root_dir, root) = test_root("deleted-root-reap-census");
7514 let executor = Arc::new(Executor::new());
7515 assert!(executor.register_actor(root.clone(), test_ctx()));
7516 root_dir.close().expect("delete project root");
7517
7518 let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
7519 let metrics = DispatchPathMetrics::new();
7520 let outcome = reap_idle_roots(
7521 Instant::now(),
7522 &mut live_roots,
7523 &HashMap::new(),
7524 &HashMap::new(),
7525 &executor,
7526 &metrics,
7527 );
7528 assert_eq!(outcome.evicted, 0);
7529
7530 let app = crate::context::App::default_shared();
7531 let health_rollup_cache = HealthRollupCache::new();
7532 health_rollup_cache.refresh(&executor, &app);
7533 let report = build_health_report(
7534 &health_rollup_cache,
7535 &executor,
7536 &HashMap::new(),
7537 &metrics,
7538 &app,
7539 );
7540 let reap = report
7541 .metrics
7542 .as_ref()
7543 .and_then(|metrics| metrics.get("reap"))
7544 .expect("reap health metrics");
7545 assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
7546 assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
7547 assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
7548 assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
7549 }
7550
7551 #[test]
7552 fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
7553 let (root_dir, root) = test_root("connection-exit-deleted-root");
7554 let executor = Arc::new(Executor::new());
7555 assert!(executor.register_actor(root.clone(), test_ctx()));
7556
7557 let route = route_key(11, 1);
7558 let mut meta = RootMeta::new(Instant::now());
7559 meta.maintenance_pending = true;
7560 meta.maintenance_queued_kinds
7561 .push_back(MaintenanceDrainKind::CompletionDrains);
7562 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7563 let mut pending_binds = HashMap::new();
7564 let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
7565 let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7566 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7567 let mut route_bash_cancels = HashMap::new();
7568 let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
7569
7570 quiesce_connection_roots(
7571 &mut live_roots,
7572 &mut pending_binds,
7573 &mut routes,
7574 &mut root_channels,
7575 &mut installed_route_epochs,
7576 &mut route_bash_cancels,
7577 &active_tool_calls,
7578 &executor,
7579 );
7580 assert!(live_roots[&root].unbound_quiesced);
7581 assert!(!live_roots[&root].maintenance_pending);
7582 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
7583 assert!(routes.is_empty());
7584 assert!(root_channels.is_empty());
7585
7586 root_dir.close().expect("delete project root");
7587 let metrics = DispatchPathMetrics::new();
7588 let outcome = reap_until_forgotten(
7589 &root,
7590 &mut live_roots,
7591 &pending_binds,
7592 &root_channels,
7593 &executor,
7594 &metrics,
7595 );
7596 let mut session_identity = HashMap::new();
7597 let mut push_buffer = HashMap::new();
7598 let mut bg_subs = HashMap::new();
7599 let mut bg_sub_by_session = HashMap::new();
7600 let mut bg_wake_pending = HashSet::new();
7601 let mut bg_wake_epoch = HashMap::new();
7602 let mut pending_bash_asks = HashMap::new();
7603 let mut retry_buffer = HashMap::new();
7604 let mut reclaimed_routes = ReclaimedRoutes::default();
7605 for forgotten in &outcome.forgotten_deleted_roots {
7606 purge_deleted_root_residents(
7607 forgotten,
7608 &mut routes,
7609 &mut root_channels,
7610 &mut installed_route_epochs,
7611 &mut route_bash_cancels,
7612 &active_tool_calls,
7613 executor.as_ref(),
7614 &mut retry_buffer,
7615 &mut reclaimed_routes,
7616 &mut session_identity,
7617 &mut push_buffer,
7618 &mut bg_subs,
7619 &mut bg_sub_by_session,
7620 &mut bg_wake_pending,
7621 &mut bg_wake_epoch,
7622 &mut pending_bash_asks,
7623 &metrics,
7624 );
7625 }
7626
7627 assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7628 assert!(!executor.actor_registered(&root));
7629 assert!(!live_roots.contains_key(&root));
7630 }
7631
7632 #[test]
7633 fn unbound_root_quiesces_maintenance_without_removing_actor() {
7634 let (_root_dir, root) = test_root("unbound-root-quiesce");
7635 let ctx = test_ctx();
7636 let executor = Arc::new(Executor::new());
7637 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7638 let mut meta = RootMeta::new(Instant::now());
7639 meta.maintenance_pending = true;
7640 meta.maintenance_jobs_in_flight = 1;
7641 meta.maintenance_queued_kinds
7642 .push_back(MaintenanceDrainKind::ConfigureTail);
7643 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7644 *ctx.search_index()
7646 .write()
7647 .unwrap_or_else(std::sync::PoisonError::into_inner) =
7648 Some(crate::search_index::SearchIndex::new());
7649 ctx.set_cache_writer_capabilities(true, true);
7650 let pending = root.as_path().join("pending.rs");
7651 ctx.add_pending_search_index_paths([pending.clone()]);
7652 let canonical_root = root.as_path().to_path_buf();
7656 let artifact = canonical_root.join("cache.bin");
7657 std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
7658 let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
7659 crate::cache_freshness::record_verify_completed(
7660 &canonical_root,
7661 crate::cache_freshness::VerifyArtifact::Search,
7662 seeded_generation,
7663 );
7664 assert!(matches!(
7665 crate::cache_freshness::warm_verify_plan(
7666 &canonical_root,
7667 crate::cache_freshness::VerifyArtifact::Search,
7668 seeded_generation,
7669 ),
7670 crate::cache_freshness::WarmVerifyPlan::Skip
7671 ));
7672 let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7675 let _dispatch_tx = dispatch_tx;
7676 let shutdown = Arc::new(AtomicBool::new(false));
7677 let thread_shutdown = Arc::clone(&shutdown);
7678 let join = std::thread::spawn(move || {
7679 while !thread_shutdown.load(Ordering::SeqCst) {
7680 std::thread::yield_now();
7681 }
7682 });
7683 ctx.install_watcher_runtime(
7684 dispatch_rx,
7685 crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7686 );
7687 assert!(ctx.watcher_runtime_active());
7688
7689 quiesce_unbound_root(&root, &mut live_roots, &executor);
7690 let meta = &live_roots[&root];
7691 assert!(meta.unbound_quiesced);
7692 assert!(ctx.subc_unbound_quiesced());
7693 assert!(meta.maintenance_pending);
7694 assert!(meta.maintenance_queued_kinds.is_empty());
7695 assert!(executor.actor_registered(&root));
7696 assert!(
7700 ctx.search_index()
7701 .read()
7702 .unwrap_or_else(std::sync::PoisonError::into_inner)
7703 .is_some(),
7704 "quiesce must not evict resident artifacts"
7705 );
7706 assert_eq!(
7707 ctx.pending_callgraph_store_force_token(),
7708 None,
7709 "quiesce must not force a callgraph rebuild"
7710 );
7711 assert_eq!(
7712 ctx.take_pending_search_index_paths(),
7713 vec![pending],
7714 "quiesce must retain pending watcher-derived paths"
7715 );
7716 assert!(
7717 matches!(
7718 crate::cache_freshness::warm_verify_plan(
7719 &canonical_root,
7720 crate::cache_freshness::VerifyArtifact::Search,
7721 seeded_generation,
7722 ),
7723 crate::cache_freshness::WarmVerifyPlan::Skip
7724 ),
7725 "quiesce must not invalidate the warm verify memo"
7726 );
7727 assert!(
7728 ctx.watcher_runtime_active(),
7729 "quiesce must not stop a running watcher"
7730 );
7731 ctx.stop_watcher_runtime();
7732
7733 let meta = live_roots.get_mut(&root).expect("root metadata");
7734 note_maintenance_completion(
7735 meta,
7736 Some(MaintenanceDrainKind::ConfigureTail),
7737 false,
7738 meta.unbound_quiesced,
7739 );
7740 assert!(!meta.maintenance_pending);
7741 assert!(meta.maintenance_queued_kinds.is_empty());
7742 }
7743
7744 #[test]
7745 fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
7746 let (_dir, root) = test_root("same-root-replacement");
7747 let route = route_key(7, 1);
7748 let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7749 let root_channels = HashMap::new();
7750
7751 assert!(!route_removal_will_quiesce_root(
7752 &root,
7753 route,
7754 &installed_channels,
7755 false,
7756 Some(&root),
7757 ));
7758 assert!(route_removal_will_quiesce_root(
7759 &root,
7760 route,
7761 &installed_channels,
7762 false,
7763 None,
7764 ));
7765 assert!(!should_quiesce_removed_root(
7766 &root,
7767 &root_channels,
7768 false,
7769 Some(&root),
7770 ));
7771 assert!(should_quiesce_removed_root(
7772 &root,
7773 &root_channels,
7774 false,
7775 None,
7776 ));
7777 assert!(!should_quiesce_removed_root(
7778 &root,
7779 &root_channels,
7780 true,
7781 None,
7782 ));
7783 }
7784
7785 #[test]
7786 fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
7787 let (_root_dir, root) = test_root("unbound-root-route-count");
7788 let executor = Arc::new(Executor::new());
7789 assert!(executor.register_actor(root.clone(), test_ctx()));
7790 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7791 let mut root_channels = HashMap::from([(
7792 root.clone(),
7793 HashSet::from([route_key(7, 1), route_key(8, 1)]),
7794 )]);
7795
7796 remove_root_channel(&mut root_channels, &root, route_key(7, 1));
7797 if !root_channels.contains_key(&root) {
7798 quiesce_unbound_root(&root, &mut live_roots, &executor);
7799 }
7800 assert!(!live_roots[&root].unbound_quiesced);
7801
7802 remove_root_channel(&mut root_channels, &root, route_key(8, 1));
7803 if !root_channels.contains_key(&root) {
7804 quiesce_unbound_root(&root, &mut live_roots, &executor);
7805 }
7806 assert!(live_roots[&root].unbound_quiesced);
7807
7808 live_roots
7809 .get_mut(&root)
7810 .expect("root metadata")
7811 .note_activity();
7812 assert!(
7813 live_roots[&root].unbound_quiesced,
7814 "late asynchronous activity must not reactivate an unbound root"
7815 );
7816
7817 live_roots
7818 .get_mut(&root)
7819 .expect("root metadata")
7820 .reactivate_bound();
7821 assert!(!live_roots[&root].unbound_quiesced);
7822 }
7823
7824 #[test]
7825 fn allocator_pressure_relief_requires_every_root_to_be_idle() {
7826 let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
7827 let (_active_dir, active_root) = test_root("allocator-relief-active");
7828 let now = Instant::now();
7829 let mut live_roots = HashMap::new();
7830 let mut idle = RootMeta::new(now);
7831 idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7832 live_roots.insert(idle_root, idle);
7833 assert!(process_has_been_idle(now, &live_roots));
7834
7835 live_roots.insert(active_root.clone(), RootMeta::new(now));
7836 assert!(!process_has_been_idle(now, &live_roots));
7837
7838 let active = live_roots
7839 .get_mut(&active_root)
7840 .expect("active root metadata");
7841 active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7842 active.active_bash_waits = 1;
7843 assert!(!process_has_been_idle(now, &live_roots));
7844 }
7845
7846 #[test]
7847 fn pressure_relief_log_reports_before_and_after_measurements() {
7848 let allocator = crate::memory::AllocatorMemorySnapshot {
7849 status: "measured",
7850 bytes_in_use: Some(8 * 1024 * 1024),
7851 size_allocated: Some(12 * 1024 * 1024),
7852 retained_slack_bytes: Some(4 * 1024 * 1024),
7853 not_estimated: None,
7854 };
7855 let relief = crate::memory::AllocatorPressureRelief {
7856 bytes_released: 3 * 1024 * 1024,
7857 rss_before_bytes: Some(20 * 1024 * 1024),
7858 rss_after_bytes: Some(17 * 1024 * 1024),
7859 allocator_before: allocator.clone(),
7860 allocator_after: crate::memory::AllocatorMemorySnapshot {
7861 size_allocated: Some(9 * 1024 * 1024),
7862 retained_slack_bytes: Some(1024 * 1024),
7863 ..allocator
7864 },
7865 };
7866 let message = pressure_relief_label(&relief);
7867 assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
7868 assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
7869 assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
7870 assert!(message.contains("reported 3.0 MB released"));
7871 }
7872
7873 #[test]
7874 fn due_maintenance_jobs_skip_poisoned_roots() {
7875 let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
7876 let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
7877 let mut live_roots = HashMap::new();
7878 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7879 let mut poisoned_meta = RootMeta::new(Instant::now());
7880 poisoned_meta.maintenance_poisoned = true;
7881 live_roots.insert(poisoned_root.clone(), poisoned_meta);
7882
7883 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7884 &mut live_roots,
7885 MAINTENANCE_SUBMIT_BUDGET,
7886 &HashSet::new(),
7887 );
7888
7889 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7890 assert!(due.iter().all(|(root, _)| root == &healthy_root));
7891 assert!(!deferred);
7892 assert!(live_roots[&healthy_root].maintenance_pending);
7893 assert_eq!(
7894 live_roots[&healthy_root].maintenance_jobs_in_flight,
7895 INITIAL_MAINTENANCE_JOB_COUNT
7896 );
7897 assert!(!live_roots[&poisoned_root].maintenance_pending);
7898 }
7899
7900 #[test]
7901 fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
7902 let (_dir, root) = test_root("maintenance-unbound");
7903 let mut meta = RootMeta::new(Instant::now());
7904 meta.unbound_quiesced = true;
7905 let mut live_roots = HashMap::from([(root.clone(), meta)]);
7906
7907 let (due, deferred) = due_maintenance_jobs_without_actor_context(
7908 &mut live_roots,
7909 MAINTENANCE_SUBMIT_BUDGET,
7910 &HashSet::new(),
7911 );
7912
7913 assert!(due.is_empty());
7914 assert!(!deferred);
7915 assert!(!live_roots[&root].maintenance_pending);
7916 }
7917
7918 #[test]
7919 fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
7920 let (_dir, root) = test_root("maintenance-idle-bg-subscription");
7921 let ctx = test_ctx();
7922 assert!(!ctx.completion_drains_have_work());
7923
7924 let executor = Executor::new();
7925 assert!(executor.register_actor(root.clone(), ctx));
7926 let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7927 let session = "idle-session".to_string();
7928 let channel = route_key(17, 1);
7929 let metrics = DispatchPathMetrics::new();
7930 let bg_sub_by_session =
7931 HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
7932 let mut bg_wake_pending = HashSet::new();
7933
7934 let (idle_tick_jobs, deferred) = due_maintenance_jobs(
7935 &mut live_roots,
7936 Some(&executor),
7937 &bg_sub_by_session,
7938 &bg_wake_pending,
7939 MAINTENANCE_SUBMIT_BUDGET,
7940 &HashSet::new(),
7941 );
7942 assert!(idle_tick_jobs.is_empty());
7943 assert!(!deferred);
7944 assert!(!live_roots[&root].maintenance_pending);
7945
7946 let mut bg_wake_epoch = HashMap::new();
7949 push::arm_bg_wake(
7950 root.clone(),
7951 session,
7952 channel,
7953 &mut bg_wake_pending,
7954 &mut bg_wake_epoch,
7955 &metrics,
7956 );
7957 let (next_tick_jobs, deferred) = due_maintenance_jobs(
7958 &mut live_roots,
7959 Some(&executor),
7960 &bg_sub_by_session,
7961 &bg_wake_pending,
7962 MAINTENANCE_SUBMIT_BUDGET,
7963 &HashSet::new(),
7964 );
7965 assert_eq!(
7966 next_tick_jobs,
7967 vec![(root, MaintenanceDrainKind::CompletionDrains)]
7968 );
7969 assert!(!deferred);
7970 }
7971
7972 #[tokio::test]
7973 async fn subc_configure_tail_precedes_completed_search_install() {
7974 let root_dir = tempfile::tempdir().unwrap();
7975 let storage = tempfile::tempdir().unwrap();
7976 let root = ProjectRootId::from_path(root_dir.path()).unwrap();
7977 let (ctx, ignored_path) =
7978 runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
7979 let ctx = Arc::new(ctx);
7980 assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
7981 &ctx,
7982 &ignored_path
7983 ));
7984
7985 let executor = Arc::new(Executor::new());
7986 assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7987 let metrics = Arc::new(DispatchPathMetrics::new());
7988 let (completion_tx, mut completion_rx) = mpsc::channel(4);
7989 submit_maintenance_job(
7990 &executor,
7991 root.clone(),
7992 MaintenanceDrainKind::ConfigureTail,
7993 Vec::new(),
7994 &completion_tx,
7995 &metrics,
7996 );
7997 submit_maintenance_job(
7998 &executor,
7999 root,
8000 MaintenanceDrainKind::CompletionDrains,
8001 Vec::new(),
8002 &completion_tx,
8003 &metrics,
8004 );
8005
8006 let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
8007 .await
8008 .expect("configure-tail completion timed out")
8009 .expect("configure-tail completion channel closed");
8010 let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
8011 .await
8012 .expect("completion-drains completion timed out")
8013 .expect("completion-drains completion channel closed");
8014 assert!(first.response.id.contains("configure-tail"));
8015 assert!(second.response.id.contains("completion-drains"));
8016 assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
8017 &ctx,
8018 &ignored_path
8019 ));
8020 assert_eq!(
8021 ctx.search_index()
8022 .read()
8023 .unwrap_or_else(std::sync::PoisonError::into_inner)
8024 .as_ref()
8025 .expect("completed search index installed")
8026 .file_count(),
8027 0,
8028 "configure must install the ignore matcher before pending paths replay"
8029 );
8030 ctx.stop_watcher_runtime();
8031 }
8032
8033 #[test]
8034 fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
8035 let (_dir, root) = test_root("maintenance-post-bind");
8036 let mut live_roots = HashMap::new();
8037 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8038
8039 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
8040 queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
8041
8042 let meta = live_roots.get(&root).expect("root metadata");
8043 assert!(meta.maintenance_pending);
8044 assert_eq!(meta.maintenance_jobs_in_flight, 0);
8045 assert_eq!(
8046 meta.maintenance_queued_kinds
8047 .iter()
8048 .copied()
8049 .collect::<Vec<_>>(),
8050 vec![
8051 MaintenanceDrainKind::ConfigureTail,
8052 MaintenanceDrainKind::CompletionDrains,
8053 ]
8054 );
8055
8056 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8057 &mut live_roots,
8058 MAINTENANCE_SUBMIT_BUDGET,
8059 &HashSet::new(),
8060 );
8061
8062 assert_eq!(
8063 due,
8064 vec![
8065 (root.clone(), MaintenanceDrainKind::ConfigureTail),
8066 (root.clone(), MaintenanceDrainKind::CompletionDrains),
8067 ]
8068 );
8069 assert!(!deferred);
8070 assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
8071 assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8072 }
8073
8074 #[test]
8075 fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
8076 let mut live_roots = HashMap::new();
8077 let mut root_ids = Vec::new();
8078 let mut _dirs = Vec::new();
8079 for index in 0..4 {
8080 let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
8081 live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
8082 root_ids.push(root_id);
8083 _dirs.push(dir);
8084 }
8085
8086 let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
8087 let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
8088 &mut live_roots,
8089 small_budget,
8090 &HashSet::new(),
8091 );
8092
8093 assert_eq!(first_due.len(), small_budget);
8094 assert!(first_deferred);
8095 let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
8096 assert!(first_due_set
8097 .iter()
8098 .all(|root| live_roots[root].maintenance_pending));
8099 assert!(first_due_set
8100 .iter()
8101 .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
8102
8103 let all_roots: HashSet<_> = root_ids.into_iter().collect();
8104 let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
8105 assert!(deferred_roots
8106 .iter()
8107 .all(|root| !live_roots[root].maintenance_pending));
8108 }
8109
8110 #[test]
8111 fn due_maintenance_jobs_defers_pending_bind_roots() {
8112 let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
8113 let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
8114 let mut live_roots = HashMap::new();
8115 live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
8116 live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8117 let pending_bind_roots = HashSet::from([bind_root.clone()]);
8118
8119 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8120 &mut live_roots,
8121 usize::MAX,
8122 &pending_bind_roots,
8123 );
8124
8125 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8126 assert!(due.iter().all(|(root, _)| root == &healthy_root));
8127 assert!(!deferred);
8128 assert!(!live_roots[&bind_root].maintenance_pending);
8129 assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
8130 }
8131
8132 #[test]
8133 fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
8134 let (_dir, root) = test_root("maintenance-requeue");
8135 let mut live_roots = HashMap::new();
8136 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8137 let (due, deferred) = due_maintenance_jobs_without_actor_context(
8138 &mut live_roots,
8139 usize::MAX,
8140 &HashSet::new(),
8141 );
8142 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8143 assert!(due.iter().all(|(due_root, _)| due_root == &root));
8144 assert!(!deferred);
8145
8146 let meta = live_roots.get_mut(&root).unwrap();
8147 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
8148 assert!(meta.maintenance_pending);
8149 assert_eq!(
8150 meta.maintenance_jobs_in_flight,
8151 INITIAL_MAINTENANCE_JOB_COUNT - 1
8152 );
8153 assert_eq!(meta.maintenance_queued_kinds.len(), 1);
8154
8155 let (requeued, deferred) =
8156 due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
8157 assert_eq!(
8158 requeued,
8159 vec![(root.clone(), MaintenanceDrainKind::Watcher)]
8160 );
8161 assert!(!deferred);
8162 let meta = live_roots.get_mut(&root).unwrap();
8163 assert_eq!(
8164 meta.maintenance_jobs_in_flight,
8165 INITIAL_MAINTENANCE_JOB_COUNT
8166 );
8167 assert!(meta.maintenance_queued_kinds.is_empty());
8168
8169 for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
8170 note_maintenance_completion(meta, None, false, false);
8171 }
8172 assert!(!meta.maintenance_pending);
8173 assert_eq!(meta.maintenance_jobs_in_flight, 0);
8174 }
8175
8176 #[test]
8177 fn maintenance_requeue_drops_while_bind_is_pending() {
8178 let (_dir, root) = test_root("maintenance-bind-requeue");
8179 let mut live_roots = HashMap::new();
8180 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8181 let (due, _) = due_maintenance_jobs_without_actor_context(
8182 &mut live_roots,
8183 usize::MAX,
8184 &HashSet::new(),
8185 );
8186 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8187
8188 let meta = live_roots.get_mut(&root).unwrap();
8189 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
8190
8191 assert_eq!(
8192 meta.maintenance_jobs_in_flight,
8193 INITIAL_MAINTENANCE_JOB_COUNT - 1
8194 );
8195 assert!(meta.maintenance_queued_kinds.is_empty());
8196 assert!(meta.maintenance_pending);
8197 }
8198
8199 #[test]
8200 fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
8201 let mut meta = RootMeta::new(Instant::now());
8202 meta.unbound_quiesced = true;
8203
8204 assert!(!should_requiesce_after_maintenance(
8205 &meta,
8206 MaintenanceDrainKind::Lsp,
8207 false,
8208 ));
8209 assert!(!should_requiesce_after_maintenance(
8210 &meta,
8211 MaintenanceDrainKind::ConfigureTail,
8212 true,
8213 ));
8214 assert!(should_requiesce_after_maintenance(
8215 &meta,
8216 MaintenanceDrainKind::ConfigureTail,
8217 false,
8218 ));
8219 }
8220
8221 #[test]
8222 fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
8223 let (_dir, root) = test_root("maintenance-fatal");
8224 let mut live_roots = HashMap::new();
8225 live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8226 let (due, _) = due_maintenance_jobs_without_actor_context(
8227 &mut live_roots,
8228 usize::MAX,
8229 &HashSet::new(),
8230 );
8231 assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8232
8233 let meta = live_roots.get_mut(&root).unwrap();
8234 note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
8235 assert!(meta.maintenance_poisoned);
8236 assert!(meta.maintenance_queued_kinds.is_empty());
8237
8238 for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
8239 note_maintenance_completion(meta, None, false, false);
8240 }
8241 assert!(!meta.maintenance_pending);
8242 assert_eq!(meta.maintenance_jobs_in_flight, 0);
8243 }
8244
8245 #[test]
8246 fn trust_for_principal_matrix() {
8247 assert_eq!(
8248 trust_for_principal(&Some(Principal::Direct)),
8249 BindTrust::FirstParty
8250 );
8251 for module_id in [
8258 "llm-runner",
8259 "aft",
8260 "broca",
8261 "alfonso-core",
8262 "prefrontal",
8263 "prefrontal-core",
8264 ] {
8265 assert_eq!(
8266 trust_for_principal(&Some(Principal::Reserved {
8267 module_id: module_id.to_string(),
8268 })),
8269 BindTrust::FirstParty,
8270 "reserved module id '{module_id}' must resolve to first-party trust"
8271 );
8272 }
8273 assert_eq!(
8274 trust_for_principal(&Some(Principal::Reserved {
8275 module_id: "subc-mcp".to_string(),
8276 })),
8277 BindTrust::Untrusted
8278 );
8279 assert_eq!(
8280 trust_for_principal(&Some(Principal::Reserved {
8281 module_id: "anything-unknown".to_string(),
8282 })),
8283 BindTrust::Untrusted
8284 );
8285 assert_eq!(
8286 trust_for_principal(&Some(Principal::Unverified)),
8287 BindTrust::Untrusted
8288 );
8289 assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
8290 }
8291
8292 #[test]
8293 fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
8294 let principal = Some(Principal::Direct);
8295 let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
8296 let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
8297
8298 assert_eq!(
8299 trust_for_bind(fingerprint_a, &principal),
8300 BindTrust::Untrusted
8301 );
8302 assert_eq!(
8303 trust_for_bind(fingerprint_b, &principal),
8304 BindTrust::Untrusted
8305 );
8306 }
8307
8308 #[test]
8315 fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
8316 for harness in ["opencode", "pi", "runner", "mcp:claude"] {
8317 assert_eq!(
8318 trust_for_bind(harness, &Some(Principal::Direct)),
8319 BindTrust::FirstParty,
8320 "a direct principal must stay first-party on {harness}"
8321 );
8322 assert_eq!(
8323 trust_for_bind(harness, &Some(Principal::Unverified)),
8324 BindTrust::Untrusted,
8325 "an unverified principal must stay untrusted on {harness}"
8326 );
8327 assert_eq!(
8328 trust_for_bind(harness, &None),
8329 BindTrust::Untrusted,
8330 "an absent principal must fail closed on {harness}"
8331 );
8332 assert_eq!(
8333 trust_for_bind(
8334 harness,
8335 &Some(Principal::Reserved {
8336 module_id: "subc-mcp".to_string(),
8337 })
8338 ),
8339 BindTrust::Untrusted,
8340 "a non-allowlisted reserved module must stay untrusted on {harness}"
8341 );
8342 }
8343 }
8344
8345 #[tokio::test]
8346 async fn persistent_cancel_resolves_when_fired_before_await() {
8347 let signal = PersistentCancelSignal::new();
8351 signal.cancel();
8352 tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
8354 .await
8355 .expect("cancelled() must resolve when cancel fired beforehand");
8356
8357 let racing = PersistentCancelSignal::new();
8359 let racing_for_task = racing.clone();
8360 let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
8361 racing.cancel();
8362 tokio::time::timeout(Duration::from_secs(1), waiter)
8363 .await
8364 .expect("cancelled() must resolve when cancel races the await")
8365 .expect("waiter task panicked");
8366 }
8367
8368 #[test]
8369 fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
8370 let installed = HashMap::from([(7, 9)]);
8371 let mut reclaimed = ReclaimedRoutes::default();
8372 reclaimed.insert(route_key(8, 1));
8373 for ty in [
8374 FrameType::Request,
8375 FrameType::Response,
8376 FrameType::Error,
8377 FrameType::Push,
8378 FrameType::Cancel,
8379 FrameType::Goodbye,
8380 ] {
8381 let body = if ty.is_pure_header() {
8382 Vec::new()
8383 } else {
8384 br#"{}"#.to_vec()
8385 };
8386 let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
8387 assert!(
8388 !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
8389 "{ty:?}"
8390 );
8391 }
8392
8393 let reclaimed_request = Frame::build(
8394 FrameType::Request,
8395 control_flags(),
8396 8,
8397 1,
8398 42,
8399 br#"{}"#.to_vec(),
8400 )
8401 .unwrap();
8402 assert!(ingress_route_should_be_processed(
8403 &installed,
8404 &reclaimed,
8405 &reclaimed_request
8406 ));
8407
8408 let never_installed = Frame::build(
8409 FrameType::Request,
8410 control_flags(),
8411 9,
8412 1,
8413 43,
8414 br#"{}"#.to_vec(),
8415 )
8416 .unwrap();
8417 assert!(!ingress_route_should_be_processed(
8418 &installed,
8419 &reclaimed,
8420 &never_installed
8421 ));
8422
8423 let current = Frame::build(
8424 FrameType::Request,
8425 control_flags(),
8426 7,
8427 9,
8428 43,
8429 br#"{}"#.to_vec(),
8430 )
8431 .unwrap();
8432 let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
8433 assert!(ingress_route_should_be_processed(
8434 &installed, &reclaimed, ¤t
8435 ));
8436 assert!(ingress_route_should_be_processed(
8437 &installed, &reclaimed, &control
8438 ));
8439 assert_eq!(installed, HashMap::from([(7, 9)]));
8440 }
8441
8442 #[tokio::test]
8443 async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
8444 let (_dir, root) = test_root("route-bind-b2-ordering");
8445 let route = route_key(7, 3);
8446 let identity = RouteIdentity(Arc::new(RouteIdentityData {
8447 root: root.clone(),
8448 project_root: root.as_path().to_path_buf(),
8449 harness: "opencode".to_string(),
8450 session: "b2-session".to_string(),
8451 trust: BindTrust::FirstParty,
8452 spawn_principal: AuthenticatedPrincipal::FirstParty,
8453 consumer_elicitation_capable: false,
8454 }));
8455 let replay_key = push::ReplayKey::from_identity(&identity);
8456 let completion = RouteBindCompletion {
8457 route,
8458 identity,
8459 bind_root_id: root.clone(),
8460 inserted_new_actor: false,
8461 configure_response: Response::success("subc-bind-7", json!({})),
8462 diagnostics_on_edit: false,
8463 ver: PROTOCOL_VERSION,
8464 corr: 91,
8465 flags: control_flags(),
8466 };
8467 let mut pending_binds = HashMap::from([(
8468 route,
8469 PendingBind {
8470 bind_root_id: root,
8471 inserted_new_actor: false,
8472 cancelled: false,
8473 configure_request_id: "subc-bind-7".to_string(),
8474 started_at: Instant::now(),
8475 warned_half_deadline: false,
8476 deadline_reported: false,
8477 corr: 91,
8478 ver: PROTOCOL_VERSION,
8479 flags: control_flags(),
8480 cancellation: crate::executor::JobCancellation::new(),
8481 },
8482 )]);
8483 let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8484 let mut push_buffer =
8485 HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
8486 let (writer_tx, mut writer_rx) = mpsc::channel(8);
8487 let metrics = Arc::new(DispatchPathMetrics::new());
8488 let executor = Arc::new(Executor::new());
8489 let standing_actor =
8490 standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
8491
8492 handle_route_bind_completion(
8493 &writer_tx,
8494 completion,
8495 &mut HashMap::new(),
8496 &mut HashMap::new(),
8497 &mut HashMap::new(),
8498 &mut push_buffer,
8499 &mut HashMap::new(),
8500 &mut pending_binds,
8501 &mut installed_route_epochs,
8502 &executor,
8503 &standing_actor,
8504 &Arc::new(Notify::new()),
8505 &metrics,
8506 )
8507 .await
8508 .unwrap();
8509
8510 let ack = writer_rx.try_recv().expect("RouteBindAck");
8511 assert_eq!(ack.header.ty, FrameType::Response);
8512 assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
8513 let route_frame = writer_rx.try_recv().expect("post-ack route frame");
8514 assert_eq!(route_frame.header.ty, FrameType::Push);
8515 assert_eq!(
8516 (route_frame.header.channel, route_frame.header.epoch),
8517 (route.channel, route.epoch)
8518 );
8519 }
8520}